diff options
Diffstat (limited to 'lib')
692 files changed, 66954 insertions, 31502 deletions
diff --git a/lib/Makefile b/lib/Makefile index cdb3f3f3dc..ea2827bef0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -28,7 +28,7 @@ ERTS_APPLICATIONS = stdlib sasl kernel compiler ERLANG_APPLICATIONS = tools common_test runtime_tools inets parsetools # These are only build if -a is given to otp_build or make is used directly -ALL_ERLANG_APPLICATIONS = xmerl edoc erl_docgen snmp otp_mibs erl_interface \ +ALL_ERLANG_APPLICATIONS = xmerl edoc erl_docgen snmp erl_interface \ asn1 jinterface \ wx debugger reltool \ mnesia crypto os_mon syntax_tools \ @@ -58,10 +58,10 @@ else SUB_DIRECTORIES = hipe parsetools asn1/src else ifdef TERTIARY_BOOTSTRAP - SUB_DIRECTORIES = snmp sasl jinterface syntax_tools wx + SUB_DIRECTORIES = snmp sasl erl_interface jinterface syntax_tools wx else ifdef DOC_BOOTSTRAP - SUB_DIRECTORIES = xmerl edoc erl_docgen + SUB_DIRECTORIES = xmerl edoc erl_docgen public_key else # Not bootstrap build SUB_DIRECTORIES = $(ERTS_APPLICATIONS) \ $(ERLANG_APPLICATIONS) \ diff --git a/lib/asn1/src/asn1ct_check.erl b/lib/asn1/src/asn1ct_check.erl index 321980e5e4..9eec05abd1 100644 --- a/lib/asn1/src/asn1ct_check.erl +++ b/lib/asn1/src/asn1ct_check.erl @@ -1710,7 +1710,7 @@ check_value(S,#valuedef{pos=Pos,name=Name,type=Type, {valueset, check_type(S,#typedef{pos=Pos,name=Name,typespec=NewType},NewType)}; check_value(S, #valuedef{}=V) -> - ?dbg("check_value, V: ~p~n",[V0]), + ?dbg("check_value, V: ~p~n",[V]), case V of #valuedef{checked=true} -> V; @@ -1721,7 +1721,8 @@ check_value(S, #valuedef{}=V) -> check_valuedef(#state{recordtopname=TopName}=S0, V0) -> #valuedef{name=Name,type=Vtype0,value=Value,module=ModName} = V0, V = V0#valuedef{checked=true}, - Vtype = check_type(S0, #typedef{name=Name,typespec=Vtype0},Vtype0), + Vtype1 = expand_valuedef_type(Vtype0), + Vtype = check_type(S0, #typedef{name=Name,typespec=Vtype1},Vtype1), Def = Vtype#type.def, S1 = S0#state{tname=Def}, SVal = update_state(S1, ModName), @@ -1767,6 +1768,27 @@ check_valuedef(#state{recordtopname=TopName}=S0, V0) -> V#valuedef{value=normalize_value(SVal, Vtype, Value, TopName)} end. +expand_valuedef_type(#type{def=Seq}=Type) + when is_record(Seq,'SEQUENCE') -> + NewComponents = case Seq#'SEQUENCE'.components of + {R1,_Ext,R2} -> R1 ++ R2; + {Root,_Ext} -> Root; + Root -> take_only_rootset(Root) + end, + NewSeq = Seq#'SEQUENCE'{components = NewComponents}, + Type#type{def=NewSeq}; +expand_valuedef_type(#type{def=Set}=Type) + when is_record(Set,'SET') -> + NewComponents = case Set#'SET'.components of + {R1,_Ext,R2} -> R1 ++ R2; + {Root,_Ext} -> Root; + Root -> take_only_rootset(Root) + end, + NewSet = Set#'SET'{components = NewComponents}, + Type#type{def=NewSet}; +expand_valuedef_type(Type) -> + Type. + is_contextswitchtype(#typedef{name='EXTERNAL'})-> true; is_contextswitchtype(#typedef{name='EMBEDDED PDV'}) -> @@ -1998,7 +2020,8 @@ normalize_value(S, Type, {'DEFAULT',Value}, NameList) -> {'ENUMERATED',CType,_} -> normalize_enumerated(S,Value,CType); {'CHOICE',CType,NewNameList} -> - normalize_choice(S,Value,CType,NewNameList); + ChoiceComponents = get_choice_components(S, {'CHOICE',CType}), + normalize_choice(S,Value,ChoiceComponents,NewNameList); {'SEQUENCE',CType,NewNameList} -> normalize_sequence(S,Value,CType,NewNameList); {'SEQUENCE OF',CType,NewNameList} -> @@ -2140,6 +2163,9 @@ normalize_octetstring(S, Value) -> _ -> asn1_error(S, illegal_octet_string_value) end; + Val when is_binary(Val) -> + %% constant default value + Val; _ -> asn1_error(S, illegal_octet_string_value) end. @@ -2751,8 +2777,9 @@ check_type(S=#state{recordtopname=TopName},Type,Ts) when is_record(Ts,type) -> TempNewDef#newt{type={'SEQUENCE OF',check_sequenceof(S,Type,Components)}, tag= merge_tags(Tag,?TAG_CONSTRUCTED(?N_SEQUENCE))}; - {'CHOICE',Components} -> + {'CHOICE',_} = Choice-> Ct = maybe_illicit_implicit_tag(S, choice, Tag), + Components = get_choice_components(S, Choice), TempNewDef#newt{type={'CHOICE',check_choice(S,Type,Components)},tag=Ct}; Set when is_record(Set,'SET') -> RecordName= @@ -5770,7 +5797,7 @@ format_error({missing_ocft,Component}) -> format_error(multiple_uniqs) -> "implementation limitation: only one UNIQUE field is allowed in CLASS"; format_error({namelist_redefinition,Name}) -> - io_lib:format("the name '~s' can not be redefined", [Name]); + io_lib:format("the name '~s' cannot be redefined", [Name]); format_error({param_bad_type, Ref}) -> io_lib:format("'~p' is not a parameterized type", [Ref]); format_error(param_wrong_number_of_arguments) -> diff --git a/lib/asn1/test/Makefile b/lib/asn1/test/Makefile index c38d1c6ebd..6ff4aa8d0f 100644 --- a/lib/asn1/test/Makefile +++ b/lib/asn1/test/Makefile @@ -60,6 +60,7 @@ MODULES= \ testSeqOf \ testSeqOfIndefinite \ testSeqOfCho \ + testSeqOfChoExt \ testSeqOfExternal \ testSeqOfTag \ testSetDefault \ @@ -72,6 +73,7 @@ MODULES= \ testSetTypeRefPrim \ testSetTypeRefSeq \ testSetTypeRefSet \ + testDefaultOctetString \ testChoiceIndefinite \ testSetOf \ testSetOfCho \ diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl index ab78678110..a88e464996 100644 --- a/lib/asn1/test/asn1_SUITE.erl +++ b/lib/asn1/test/asn1_SUITE.erl @@ -99,6 +99,7 @@ groups() -> testChoTypeRefPrim, testChoTypeRefSeq, testChoTypeRefSet, + testDefaultOctetString, testMultipleLevels, testOpt, testSeqDefault, @@ -118,6 +119,7 @@ groups() -> {group, [], [testSeqOf, testSeqOfIndefinite]}, % Uses 'Mvrasn*' testSeqOfCho, + testSeqOfChoExt, testSetDefault, testExtensionAdditionGroup, testSetOptional, @@ -430,6 +432,11 @@ testChoTypeRefSet(Config, Rule, Opts) -> asn1_test_lib:compile("ChoTypeRefSet", Config, [Rule|Opts]), testChoTypeRefSet:set(Rule). +testDefaultOctetString(Config) -> test(Config, fun testDefaultOctetString/3). +testDefaultOctetString(Config, Rule, Opts) -> + asn1_test_lib:compile("DefaultOctetString", Config, [Rule|Opts]), + testDefaultOctetString:dos(Rule). + testMultipleLevels(Config) -> test(Config, fun testMultipleLevels/3). testMultipleLevels(Config, Rule, Opts) -> asn1_test_lib:compile("MultipleLevels", Config, [Rule|Opts]), @@ -535,6 +542,11 @@ testSeqOfCho(Config, Rule, Opts) -> asn1_test_lib:compile("SeqOfCho", Config, [Rule|Opts]), testSeqOfCho:main(Rule). +testSeqOfChoExt(Config) -> test(Config, fun testSeqOfChoExt/3). +testSeqOfChoExt(Config, Rule, Opts) -> + asn1_test_lib:compile("SeqOfChoExt", Config, [Rule|Opts]), + testSeqOfChoExt:main(Rule). + testSeqOfIndefinite(Config) -> test(Config, fun testSeqOfIndefinite/3, [ber]). testSeqOfIndefinite(Config, Rule, Opts) -> diff --git a/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 b/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 index 18473bae30..c488704196 100644 --- a/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 +++ b/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 @@ -41,4 +41,6 @@ ChoExt4 ::= CHOICE str OCTET STRING } +choExt1 ChoExt1 ::= int : 1 + END diff --git a/lib/asn1/test/asn1_SUITE_data/DefaultOctetString.asn b/lib/asn1/test/asn1_SUITE_data/DefaultOctetString.asn new file mode 100644 index 0000000000..076e965d58 --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/DefaultOctetString.asn @@ -0,0 +1,15 @@ +DefaultOctetString +DEFINITIONS +AUTOMATIC TAGS + ::= +BEGIN +Dos ::= SEQUENCE { + opt [2] OCTET STRING (SIZE(2..4)) OPTIONAL, + def [10] OCTET STRING (SIZE (1)) DEFAULT '05'H +} + +dos Dos ::= { + opt '1234'H +} + +END diff --git a/lib/asn1/test/asn1_SUITE_data/IN-CS-1-Datatypes.asn b/lib/asn1/test/asn1_SUITE_data/IN-CS-1-Datatypes.asn index ff0361f5c5..fb092f3f9c 100644 --- a/lib/asn1/test/asn1_SUITE_data/IN-CS-1-Datatypes.asn +++ b/lib/asn1/test/asn1_SUITE_data/IN-CS-1-Datatypes.asn @@ -1152,7 +1152,7 @@ FilteringCriteria ::= CHOICE { -- In case calledAddressValue is specified, the numbers to be filtered are from calledAddressValue
-- up to and including calledAddressValue + maximumNumberOfCounters-1.
--- The last two digits of calledAddressvalue can not exceed 100-maximumNumberOfCounters.
+-- The last two digits of calledAddressvalue cannot exceed 100-maximumNumberOfCounters.
FilteringTimeOut ::= CHOICE {
duration [0] Duration,
stopTime [1] DateAndTime
diff --git a/lib/asn1/test/asn1_SUITE_data/SeqOfChoExt.asn1 b/lib/asn1/test/asn1_SUITE_data/SeqOfChoExt.asn1 new file mode 100644 index 0000000000..51077754fd --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/SeqOfChoExt.asn1 @@ -0,0 +1,27 @@ +SeqOfChoExt DEFINITIONS AUTOMATIC TAGS EXTENSIBILITY IMPLIED ::=
+BEGIN
+
+Seq2 ::= SEQUENCE {
+ octstr [PRIVATE 6] OCTET STRING OPTIONAL
+}
+
+SeqOfCho ::= SEQUENCE OF CHOICE {
+ nullable NULL,
+ seq2 Seq2
+}
+
+Seq1 ::= SEQUENCE {
+ int INTEGER,
+ soc SeqOfCho
+}
+
+seq1 Seq1 ::= {
+ int 10,
+ soc {
+ seq2 : {
+ octstr '01020A'H
+ }
+ }
+}
+
+END
diff --git a/lib/asn1/test/testChoExtension.erl b/lib/asn1/test/testChoExtension.erl index 4c632aab81..cfb28be5c7 100644 --- a/lib/asn1/test/testChoExtension.erl +++ b/lib/asn1/test/testChoExtension.erl @@ -28,6 +28,7 @@ extension(_Rules) -> roundtrip('ChoExt1', {bool,true}), roundtrip('ChoExt1', {int,33}), + {int, 1} = 'ChoExtension':choExt1(), %% A trick to encode with another compatible CHOICE type to test reception %% extension alternative diff --git a/lib/asn1/test/testDefaultOctetString.erl b/lib/asn1/test/testDefaultOctetString.erl new file mode 100644 index 0000000000..82cd5810e5 --- /dev/null +++ b/lib/asn1/test/testDefaultOctetString.erl @@ -0,0 +1,34 @@ +-module(testDefaultOctetString). + +-export([dos/1]). + +-include_lib("common_test/include/ct.hrl"). + +-record('Dos', { + opt = asn1_NOVALUE, + def = asn1_DEFAULT +}). + +-define(def_DEFAULT, <<5>>). + +dos(Rules) -> + %% test roundtrip default + E1 = roundtrip(#'Dos'{}, #'Dos'{def = ?def_DEFAULT}), + %% test the value dos defined in the .asn file + E2 = roundtrip('DefaultOctetString':dos()), + %% sanity test a fully specified SEQUENCE + E3 = roundtrip(#'Dos'{opt = <<1,2,3>>, def = <<6>>}), + %% test def is/isn't encoded according to the value + if Rules == ber -> + <<48, 0>> = E1, + <<48, 4, 16#82, 2, 16#12, 16#34>> = E2, + <<48, 8, 16#82, 3, 1, 2, 3, 16#8A, 1, 6>> = E3; + true -> + ignore + end, + ok. + +roundtrip(Value) -> + roundtrip(Value, Value). +roundtrip(Value, Exp) -> + asn1_test_lib:roundtrip('DefaultOctetString', 'Dos', Value, Exp). diff --git a/lib/asn1/test/testSeqOfChoExt.erl b/lib/asn1/test/testSeqOfChoExt.erl new file mode 100644 index 0000000000..1e72c60841 --- /dev/null +++ b/lib/asn1/test/testSeqOfChoExt.erl @@ -0,0 +1,15 @@ +-module(testSeqOfChoExt). + +-export([main/1]). + +%-record('Seq2', { octstr = asn1_NOVALUE }). +%-record('Seq1', { int, soc }). + +main(_Rules) -> + roundtrip('SeqOfChoExt':seq1()). + +roundtrip(Value) -> + roundtrip(Value, Value). +roundtrip(Value, Exp) -> + Type = element(1,Value), + asn1_test_lib:roundtrip('SeqOfChoExt', Type, Value, Exp). diff --git a/lib/common_test/doc/src/basics_chapter.xml b/lib/common_test/doc/src/basics_chapter.xml index 95599ca1f1..899a52fa31 100644 --- a/lib/common_test/doc/src/basics_chapter.xml +++ b/lib/common_test/doc/src/basics_chapter.xml @@ -125,7 +125,7 @@ The test case is the smallest unit that the <c>Common Test</c> test server deals with. </p> <p> - Subsets of test cases, called test case groups, can also be defined. A test case + Sets of test cases, called test case groups, can also be defined. A test case group can have execution properties associated with it. Execution properties specify if the test cases in the group are to be executed in random order, in parallel, or in sequence, and if the execution of the group diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index b9bc54ff63..ff9969ebc3 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -493,7 +493,7 @@ <seealso marker="common_test#Module:end_per_testcase-2"><c>end_per_testcase</c></seealso> instead.</p> - <p>This function can not change the result of the test case by returning skip or fail + <p>This function cannot change the result of the test case by returning skip or fail tuples, but it may insert items in <c>Config</c> that can be read in <c>end_per_testcase/2</c> or in <c>post_end_per_testcase/5</c>.</p> diff --git a/lib/common_test/doc/src/ct_master.xml b/lib/common_test/doc/src/ct_master.xml index 2ab421fe9e..f003b7de11 100644 --- a/lib/common_test/doc/src/ct_master.xml +++ b/lib/common_test/doc/src/ct_master.xml @@ -210,7 +210,7 @@ </type> <desc><marker id="run_test-2"/> <p>Tests are spawned on <c>Node</c> using - <seealso marker="ct:run_test-1"><c>ct:run_test/1</c></seealso></p> + <seealso marker="ct#run_test-1"><c>ct:run_test/1</c></seealso></p> </desc> </func> </funcs> diff --git a/lib/common_test/doc/src/event_handler_chapter.xml b/lib/common_test/doc/src/event_handler_chapter.xml index bd9ed21cb4..182abba7ca 100644 --- a/lib/common_test/doc/src/event_handler_chapter.xml +++ b/lib/common_test/doc/src/event_handler_chapter.xml @@ -378,7 +378,7 @@ <note><p>To ensure that printouts to <c>stdout</c> (or printouts made with <seealso marker="ct#log-2"><c>ct:log/2,3</c></seealso> or - <seealso marker="ct:pal-2"><c>ct:pal,2,3</c></seealso>) get written to the test case log + <seealso marker="ct#pal-2"><c>ct:pal,2,3</c></seealso>) get written to the test case log file, and not to the <c>Common Test</c> framework log, you can synchronize with the <c>Common Test</c> server by matching on evvents <c>tc_start</c> and <c>tc_done</c>. In the period between these events, all I/O is directed to the diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index d39e86deb3..a68cc3cca7 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -2288,7 +2288,7 @@ ct_netconfc:close_session sometimes returned {error,closed} because the ssh connection was closed (from the server side) before the rpc-reply was received - by the client. This is normal and can not be helped. It + by the client. This is normal and cannot be helped. It has been corrected so the return will be 'ok' in this case. Other error situations will still give {error,Reason}.</p> @@ -2302,7 +2302,7 @@ {error,{process_down,Pid,normal}} because the ssh connection was closed (from the server side) before the rpc-reply was received by the client. This is normal and - can not be helped. It has been corrected so the return + cannot be helped. It has been corrected so the return will be 'ok' in this situation.</p> <p> Own Id: OTP-10570</p> @@ -2954,8 +2954,8 @@ The info function for <c>init/end_per_suite(Config)</c> is <c>init/end_per_suite()</c>, and for <c>init/end_per_group(GroupName,Config)</c> it's - <c>init/end_per_group(GroupName)</c>. Info functions can - not be used with <c>init/end_per_testcase(TestCase, + <c>init/end_per_group(GroupName)</c>. Info functions + cannot be used with <c>init/end_per_testcase(TestCase, Config)</c>, since these configuration functions execute on the test case process and will use the same properties as the test case (i.e. properties set by the test case @@ -4039,7 +4039,7 @@ If the Erlang runtime system was started without access to an erlang shell (e.g. -noshell), compilation errors would cause a crash in the Common Test application. - Without access to a shell, Common Test can not prompt the + Without access to a shell, Common Test cannot prompt the user to choose to continue or abort the test session, but must assume that the session should proceed.</p> <p> diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 56f6f7bcc4..2695e597cf 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -1297,7 +1297,7 @@ </taglist> <p>For example, if a test is started with:</p> - <p><c>$ ct_run -suite my_SUITE -logopts no_src</c></p> + <p><c>$ ct_run -suite my_SUITE -logopts no_nl</c></p> <p>then printouts during the test made by successive calls to <c>io:format("x")</c>, appears in the test case log as:</p> <p><c>xxx</c></p> diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 778ea2e9e2..bfa7b25862 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -390,11 +390,7 @@ testcases(TestDir, Suite) -> end. make_and_load(Dir, Suite) -> - EnvInclude = - case os:getenv("CT_INCLUDE_PATH") of - false -> []; - CtInclPath -> string:lexemes(CtInclPath, [$:,$ ,$,]) - end, + EnvInclude = string:lexemes(os:getenv("CT_INCLUDE_PATH", ""), [$:,$ ,$,]), StartInclude = case init:get_argument(include) of {ok,[Dirs]} -> Dirs; diff --git a/lib/common_test/src/ct_cover.erl b/lib/common_test/src/ct_cover.erl index d286f20a4d..bcd98dcc58 100644 --- a/lib/common_test/src/ct_cover.erl +++ b/lib/common_test/src/ct_cover.erl @@ -262,6 +262,11 @@ get_app_info(App=#cover{app=Name}, [{src_files,Name,Src1}|Terms], Dir) -> Src = App#cover.src, get_app_info(App#cover{src=Src++Src1},Terms,Dir); +get_app_info(App=#cover{app=none}, [{local_only,Bool}|Terms], Dir) -> + get_app_info(App, [{local_only,none,Bool}|Terms], Dir); +get_app_info(App=#cover{app=Name}, [{local_only,Name,Bool}|Terms], Dir) -> + get_app_info(App#cover{local_only=Bool},Terms,Dir); + get_app_info(App, [_|Terms], Dir) -> get_app_info(App, Terms, Dir); diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index dcf7c97244..bce6420042 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -314,8 +314,8 @@ add_defaults(Mod,Func, GroupPath) -> ErrStr = io_lib:format("~n*** ERROR *** " "~w:suite/0 failed: ~tp~n", [Suite,Reason]), - io:format(ErrStr, []), - io:format(?def_gl, ErrStr, []), + io:format("~ts", [ErrStr]), + io:format(?def_gl, "~ts", [ErrStr]), {suite0_failed,{exited,Reason}}; SuiteInfo when is_list(SuiteInfo) -> case lists:all(fun(E) when is_tuple(E) -> true; @@ -337,16 +337,16 @@ add_defaults(Mod,Func, GroupPath) -> "Invalid return value from " "~w:suite/0: ~tp~n", [Suite,SuiteInfo]), - io:format(ErrStr, []), - io:format(?def_gl, ErrStr, []), + io:format("~ts", [ErrStr]), + io:format(?def_gl, "~ts", [ErrStr]), {suite0_failed,bad_return_value} end; SuiteInfo -> ErrStr = io_lib:format("~n*** ERROR *** " "Invalid return value from " "~w:suite/0: ~tp~n", [Suite,SuiteInfo]), - io:format(ErrStr, []), - io:format(?def_gl, ErrStr, []), + io:format("~ts", [ErrStr]), + io:format(?def_gl, "~ts", [ErrStr]), {suite0_failed,bad_return_value} end. @@ -373,8 +373,8 @@ add_defaults1(Mod,Func, GroupPath, SuiteInfo) -> "Invalid return value from " "~w:group(~tw): ~tp~n", [Mod,GrName,BadGr0Val]), - io:format(Gr0ErrStr, []), - io:format(?def_gl, Gr0ErrStr, []), + io:format("~ts", [Gr0ErrStr]), + io:format(?def_gl, "~ts", [Gr0ErrStr]), {group0_failed,bad_return_value}; _ -> Args = if Func == init_per_group ; Func == end_per_group -> @@ -395,8 +395,8 @@ add_defaults1(Mod,Func, GroupPath, SuiteInfo) -> "Invalid return value from " "~w:~tw/0: ~tp~n", [Mod,Func,BadTC0Val]), - io:format(TC0ErrStr, []), - io:format(?def_gl, TC0ErrStr, []), + io:format("~ts", [TC0ErrStr]), + io:format(?def_gl, "~ts", [TC0ErrStr]), {testcase0_failed,bad_return_value}; _ -> %% let test case info (also for all config funcs) override @@ -983,11 +983,10 @@ error_notification(Mod,Func,_Args,{Error,Loc}) -> end, PrintError = fun(ErrorFormat, ErrorArgs) -> - Div = "~n- - - - - - - - - - - - - - - - - - - " - "- - - - - - - - - - - - - - - - - - - - -~n", + Div = "\n- - - - - - - - - - - - - - - - - - - " + "- - - - - - - - - - - - - - - - - - - - -\n", ErrorStr2 = io_lib:format(ErrorFormat, ErrorArgs), - io:format(?def_gl, lists:concat([Div,ErrorStr2,Div,"~n"]), - []), + io:format(?def_gl, "~ts~n", [lists:concat([Div,ErrorStr2,Div])]), Link = "\n\n<a href=\"#end\">" "Full error description and stacktrace" @@ -996,7 +995,8 @@ error_notification(Mod,Func,_Args,{Error,Loc}) -> ct_logs:tc_log(ct_error_notify, ?MAX_IMPORTANCE, "CT Error Notification", - ErrorHtml2++Link, [], []) + "~ts", [ErrorHtml2++Link], + []) end, case Loc of [{?MODULE,error_in_suite}] -> @@ -1258,7 +1258,7 @@ get_all(Mod, ConfTests) -> ErrStr = io_lib:format("~n*** ERROR *** " "~w:all/0 failed: ~tp~n", [Mod,ExitReason]), - io:format(?def_gl, ErrStr, []), + io:format(?def_gl, "~ts", [ErrStr]), %% save the error info so it doesn't get printed twice ct_util:set_testdata_async({{error_in_suite,Mod}, ExitReason}); diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index a11613bca2..ca262b350f 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -542,7 +542,7 @@ tc_print(Category,Importance,Format,Args,Opts) -> undefined -> atom_to_list(Category); Hd -> Hd end, - Str = lists:concat([get_header(Heading),Format,"\n\n"]), + Str = lists:flatten([get_header(Heading),Format,"\n\n"]), try io:format(?def_gl, Str, Args) catch @@ -935,7 +935,7 @@ create_io_fun(FromPid, CtLogFd, EscChars) -> {_HdOrFt,S,A} -> {false,S,A}; {S,A} -> {true,S,A} end, - try io_lib:format(Str, Args) of + try io_lib:format(lists:flatten(Str), Args) of IoStr when Escapable, EscChars, IoList == [] -> escape_chars(IoStr); IoStr when Escapable, EscChars -> @@ -1138,10 +1138,10 @@ set_evmgr_gl(GL) -> open_ctlog(MiscIoName) -> {ok,Fd} = file:open(?ct_log_name,[write,{encoding,utf8}]), - io:format(Fd, header("Common Test Framework Log", {[],[1,2],[]}), []), + io:format(Fd, "~ts", [header("Common Test Framework Log", {[],[1,2],[]})]), case file:consult(ct_run:variables_file_name("../")) of {ok,Vars} -> - io:format(Fd, config_table(Vars), []); + io:format(Fd, "~ts", [config_table(Vars)]); {error,Reason} -> {ok,Cwd} = file:get_cwd(), Dir = filename:dirname(Cwd), @@ -1213,7 +1213,7 @@ print_style_error(Fd, IoFormat, StyleSheet, Reason) -> close_ctlog(Fd) -> io:format(Fd, "\n</pre>\n", []), - io:format(Fd, [xhtml("<br><br>\n", "<br /><br />\n") | footer()], []), + io:format(Fd, "~ts", [[xhtml("<br><br>\n", "<br /><br />\n") | footer()]]), ok = file:close(Fd). %%%----------------------------------------------------------------- diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl index fd33ee2280..9fc169789c 100644 --- a/lib/common_test/src/ct_master.erl +++ b/lib/common_test/src/ct_master.erl @@ -350,7 +350,7 @@ master_loop(#state{node_ctrl_pids=[], io_lib:format("~-40.40.*ts~tp\n", [$_,atom_to_list(Node),Result]) end,lists:reverse(Finished)), - log(all,"TEST RESULTS",Str,[]), + log(all,"TEST RESULTS","~ts", [Str]), log(all,"Info","Updating log files",[]), refresh_logs(LogDirs,[]), @@ -574,7 +574,7 @@ refresh_logs([],Refreshed) -> io_lib:format("Refreshing logs in ~tp... ~tp", [D,Result]) end,Refreshed), - log(all,"Info",Str,[]). + log(all,"Info","~ts", [Str]). %%%----------------------------------------------------------------- %%% NODE CONTROLLER, runs and controls tests on a test node. diff --git a/lib/common_test/src/ct_release_test.erl b/lib/common_test/src/ct_release_test.erl index 60d17f43dc..ac3dcab7c9 100644 --- a/lib/common_test/src/ct_release_test.erl +++ b/lib/common_test/src/ct_release_test.erl @@ -152,10 +152,10 @@ %% returned configuration must therefore also be returned from %% the calling `init_per_*'. %% -%% If the initialization fails, e.g. if a required release can -%% not be found, the function returns `{skip,Reason}'. In +%% If the initialization fails, e.g. if a required release +%% cannot be found, the function returns `{skip,Reason}'. In %% this case the other test support functions in this mudule -%% can not be used. +%% cannot be used. %% %% Example: %% @@ -426,7 +426,7 @@ init_upgrade_test(Level) -> case OldRel of false -> ct:log("Release ~tp is not available." - " Upgrade on '~p' level can not be tested.", + " Upgrade on '~p' level cannot be tested.", [FromVsn,Level]), undefined; _ -> @@ -528,7 +528,7 @@ target_system(Apps,CreateDir,InstallDir,{FromVsn,_,AllAppsVsns,Path}) -> {path,Path}]]), %% Unpack the tar to complete the installation - erl_tar:extract(RelName ++ ".tar.gz", [{cwd, InstallDir}, compressed]), + ok = erl_tar:extract(RelName ++ ".tar.gz", [{cwd, InstallDir}, compressed]), %% Add bin and log dirs BinDir = filename:join([InstallDir, "bin"]), @@ -554,11 +554,11 @@ target_system(Apps,CreateDir,InstallDir,{FromVsn,_,AllAppsVsns,Path}) -> %% create start_erl.data, sys.config and start.src StartErlData = filename:join([InstallDir, "releases", "start_erl.data"]), - write_file(StartErlData, io_lib:fwrite("~s ~s~n", [ErtsVsn, FromVsn])), + ok = write_file(StartErlData, io_lib:fwrite("~s ~s~n", [ErtsVsn, FromVsn])), SysConfig = filename:join([InstallDir, "releases", FromVsn, "sys.config"]), - write_file(SysConfig, "[]."), + ok = write_file(SysConfig, "[]."), StartSrc = filename:join(ErtsBinDir,"start.src"), - write_file(StartSrc,start_script()), + ok = write_file(StartSrc,start_script()), ok = file:change_mode(StartSrc,8#0755), %% Make start_erl executable @@ -620,7 +620,7 @@ upgrade_system(Apps, FromRel, CreateDir, InstallDir, {_,ToVsn,_,_}) -> [{path,[FromPath]}, {outdir,CreateDir}]]), SysConfig = filename:join([CreateDir, "sys.config"]), - write_file(SysConfig, "[]."), + ok = write_file(SysConfig, "[]."), ok = systools(make_tar,[RelName,[{erts,code:root_dir()}]]), @@ -858,7 +858,7 @@ subst_file(Src, Dest, Vars, Opts) -> {ok, Bin} = file:read_file(Src), Conts = unicode:characters_to_list(Bin), NConts = subst(Conts, Vars), - write_file(Dest, NConts), + ok = write_file(Dest, NConts), case lists:member(preserve, Opts) of true -> {ok, FileInfo} = file:read_file_info(Src), diff --git a/lib/common_test/src/ct_repeat.erl b/lib/common_test/src/ct_repeat.erl index 8b1c7d47bb..b97c6e59e7 100644 --- a/lib/common_test/src/ct_repeat.erl +++ b/lib/common_test/src/ct_repeat.erl @@ -278,7 +278,7 @@ log_loop_info(Args) -> ForceStop -> io_lib:format("force_stop is set to: ~w",[ForceStop]) end, - ct_logs:log("Test loop info",LogStr1++LogStr2++LogStr3++LogStr4,[]) + ct_logs:log("Test loop info","~ts", [LogStr1++LogStr2++LogStr3++LogStr4]) end. ts(Secs) -> diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index c9d406f1fd..960252a6fe 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -2345,18 +2345,24 @@ start_cover(Opts=#opts{coverspec=CovData,cover_stop=CovStop},LogDir) -> CovImport, _CovExport, #cover{app = CovApp, + local_only = LocalOnly, level = CovLevel, excl_mods = CovExcl, incl_mods = CovIncl, cross = CovCross, src = _CovSrc}} = CovData, + case LocalOnly of + true -> cover:local_only(); + false -> ok + end, ct_logs:log("COVER INFO", "Using cover specification file: ~ts~n" "App: ~w~n" + "Local only: ~w~n" "Cross cover: ~w~n" "Including ~w modules~n" "Excluding ~w modules", - [CovFile,CovApp,CovCross, + [CovFile,CovApp,LocalOnly,CovCross, length(CovIncl),length(CovExcl)]), %% Tell test_server to print a link in its coverlog diff --git a/lib/common_test/src/ct_telnet.erl b/lib/common_test/src/ct_telnet.erl index 174008c790..219f58dcf5 100644 --- a/lib/common_test/src/ct_telnet.erl +++ b/lib/common_test/src/ct_telnet.erl @@ -1127,7 +1127,7 @@ match_line(Name,Pid,Line,[Pattern|Patterns],FoundPrompt,Term,EO,RetTag) -> end; match_line(Name,Pid,Line,[],FoundPrompt,Term,EO,match) -> match_line(Name,Pid,Line,EO#eo.haltpatterns,FoundPrompt,Term,EO,halt); -%% print any terminated line that can not be matched +%% print any terminated line that cannot be matched match_line(Name,Pid,Line,[],_FoundPrompt,true,_EO,halt) -> log(name_or_pid(Name,Pid)," ~ts",[Line]), nomatch; diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index d8fd401a64..9f489e9bfb 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -192,7 +192,10 @@ do_start(Parent, Mode, LogDir, Verbosity) -> ok end, - ct_default_gl:start_link(group_leader()), + case ct_default_gl:start_link(group_leader()) of + {ok, _} -> ok; + ignore -> ok + end, {StartTime,TestLogDir} = ct_logs:init(Mode, Verbosity), diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index 039c8168ec..d5c93d05ba 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -62,6 +62,7 @@ merge_tests=true}). -record(cover, {app=none, + local_only=false, level=details, excl_mods=[], incl_mods=[], diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl index 4980d1ee4b..fe869a4373 100644 --- a/lib/common_test/src/cth_log_redirect.erl +++ b/lib/common_test/src/cth_log_redirect.erl @@ -124,7 +124,8 @@ start_log_handler() -> shutdown=>2000, type=>worker, modules=>[?MODULE]}, - {ok,_} = supervisor:start_child(logger_sup,ChildSpec); + {ok,_} = supervisor:start_child(logger_sup,ChildSpec), + ok; _Pid -> ok end, @@ -193,10 +194,10 @@ handle_call({log, case LogFunc of tc_log -> ct_logs:tc_log(Category, ?STD_IMPORTANCE, - Header, String, [], []); + Header, "~ts", [String], []); tc_log_async -> ct_logs:tc_log_async(sasl, ?STD_IMPORTANCE, - Header, String, []) + Header, "~ts", [String]) end, {reply,ok,State}; @@ -260,34 +261,34 @@ handle_remote_events(Bool) -> format_header(#eh_state{curr_suite = undefined, curr_group = undefined, curr_func = undefined}) -> - io_lib:format("System report", []); + lists:flatten(io_lib:format("System report", [])); format_header(#eh_state{curr_suite = Suite, curr_group = undefined, curr_func = undefined}) -> - io_lib:format("System report during ~w", [Suite]); + lists:flatten(io_lib:format("System report during ~w", [Suite])); format_header(#eh_state{curr_suite = Suite, curr_group = undefined, curr_func = TcOrConf}) -> - io_lib:format("System report during ~w:~tw/1", - [Suite,TcOrConf]); + lists:flatten(io_lib:format("System report during ~w:~tw/1", + [Suite,TcOrConf])); format_header(#eh_state{curr_suite = Suite, curr_group = Group, curr_func = Conf}) when Conf == init_per_group; Conf == end_per_group -> - io_lib:format("System report during ~w:~w/2 for ~tw", - [Suite,Conf,Group]); + lists:flatten(io_lib:format("System report during ~w:~w/2 for ~tw", + [Suite,Conf,Group])); format_header(#eh_state{curr_suite = Suite, curr_group = Group, parallel_tcs = true}) -> - io_lib:format("System report during ~tw in ~w", - [Group,Suite]); + lists:flatten(io_lib:format("System report during ~tw in ~w", + [Group,Suite])); format_header(#eh_state{curr_suite = Suite, curr_group = Group, curr_func = TC}) -> - io_lib:format("System report during ~w:~tw/1 in ~tw", - [Suite,TC,Group]). + lists:flatten(io_lib:format("System report during ~w:~tw/1 in ~tw", + [Suite,TC,Group])). diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl index b0742717ae..c9b4cb10c6 100644 --- a/lib/common_test/src/cth_surefire.erl +++ b/lib/common_test/src/cth_surefire.erl @@ -235,7 +235,7 @@ close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) -> terminate(State = #state{ test_cases = [] }) -> {ok,D} = file:open(State#state.filepath,[write,{encoding,utf8}]), io:format(D, "<?xml version=\"1.0\" encoding= \"UTF-8\" ?>", []), - io:format(D, to_xml(State), []), + io:format(D, "~ts", [to_xml(State)]), catch file:sync(D), catch file:close(D); terminate(State) -> diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index fdcaaab2eb..003d08d70d 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -1393,7 +1393,7 @@ temp_nodename([Chr|Base], Acc) -> %% %% Counts the test cases that are about to run and returns that number. %% If there's a conf group in TestSpec with a repeat property, the total number -%% of cases can not be calculated and NoOfCases = unknown. +%% of cases cannot be calculated and NoOfCases = unknown. count_test_cases(TopCases, SkipCases) when is_list(TopCases) -> case collect_all_cases(TopCases, SkipCases) of {error,_Why} = Error -> @@ -1560,7 +1560,7 @@ do_test_cases(TopCases, SkipCases, Html1} end, - print(html, Header), + print(html, "~ts", [Header]), print(html, xhtml("<p>", "<h4>")), print_timestamp(html, "Test started at "), @@ -1607,10 +1607,10 @@ do_test_cases(TopCases, SkipCases, [?suitelog_name,CoverLog,?unexpected_io_log]), print(html, "<p>~ts</p>\n" ++ - xhtml(["<table bgcolor=\"white\" border=\"3\" cellpadding=\"5\">\n", - "<thead>\n"], - ["<table id=\"",?sortable_table_name,"\">\n", - "<thead>\n"]) ++ + xhtml("<table bgcolor=\"white\" border=\"3\" cellpadding=\"5\">\n" ++ + "<thead>\n", + "<table id=\"" ++ ?sortable_table_name ++ "\">\n" ++ + "<thead>\n") ++ "<tr><th>Num</th><th>Module</th><th>Group</th>" ++ "<th>Case</th><th>Log</th><th>Time</th><th>Result</th>" ++ "<th>Comment</th></tr>\n</thead>\n<tbody>\n", @@ -3354,7 +3354,8 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) -> true -> print(2,"*** Skipping test case #~w ~tw ***", [CaseNum,{Mod,Func}]) end, - TR = xhtml("<tr valign=\"top\">", ["<tr class=\"",odd_or_even(),"\">"]), + TR = xhtml("<tr valign=\"top\">", + "<tr class=\"" ++ odd_or_even() ++ "\">"), GroupName = case get_name(Mode) of undefined -> ""; Name -> cast_to_list(Name) @@ -3854,8 +3855,8 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, end, print(minor, - escape_chars(io_lib:format("Config value:\n\n ~tp\n", [Args2Print])), - []), + "~ts", + [escape_chars(io_lib:format("Config value:\n\n ~tp\n", [Args2Print]))]), print(minor, "Current directory is ~tp\n", [Cwd]), GrNameStr = case GrName of @@ -3864,7 +3865,7 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, end, print(major, "=started ~s", [lists:flatten(timestamp_get(""))]), {{Col0,Col1},Style} = get_font_style((RunInit==run_init), Mode), - TR = xhtml("<tr valign=\"top\">", ["<tr class=\"",odd_or_even(),"\">"]), + TR = xhtml("<tr valign=\"top\">", "<tr class=\"" ++ odd_or_even() ++ "\">"), EncMinorBase = uri_encode(MinorBase), print(html, TR ++ "<td>" ++ Col0 ++ "~ts" ++ Col1 ++ "</td>" "<td>" ++ Col0 ++ "~w" ++ Col1 ++ "</td>" @@ -3889,7 +3890,7 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, print(minor, "<a name=\"end\"></a>", [], internal_raw), print(minor, "\n", [], internal_raw), print_timestamp(minor, "Ended at "), - print(major, "=ended ~s", [lists:flatten(timestamp_get(""))]), + print(major, "=ended ~s", [timestamp_get("")]), do_unless_parallel(Main, fun() -> file:set_cwd(filename:dirname(TSDir)) end), @@ -4137,9 +4138,9 @@ progress(failed, CaseNum, Mod, Func, GrName, Loc, {testcase_aborted,Reason}, _T, FormatLoc = test_server_sup:format_loc(Loc), print(minor, "=== Location: ~ts", [FormatLoc]), print(minor, - escape_chars(io_lib:format("=== Reason: {testcase_aborted,~tp}", - [Reason])), - []), + "~ts", + [escape_chars(io_lib:format("=== Reason: {testcase_aborted,~tp}", + [Reason]))]), failed; progress(failed, CaseNum, Mod, Func, GrName, unknown, Reason, Time, @@ -4177,8 +4178,8 @@ progress(failed, CaseNum, Mod, Func, GrName, unknown, Reason, Time, print(minor, "=== Location: ~w", [unknown]), {FStr,FormattedReason} = format_exception(Reason), print(minor, - escape_chars(io_lib:format("=== Reason: " ++ FStr, [FormattedReason])), - []), + "~ts", + [escape_chars(io_lib:format("=== Reason: " ++ FStr, [FormattedReason]))]), failed; progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time, @@ -4212,8 +4213,9 @@ progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time, FormatLoc = test_server_sup:format_loc(LocMin), print(minor, "=== Location: ~ts", [FormatLoc]), {FStr,FormattedReason} = format_exception(Reason), - print(minor, "=== Reason: " ++ - escape_chars(io_lib:format(FStr, [FormattedReason])), []), + print(minor, "~ts", + ["=== Reason: " ++ + escape_chars(io_lib:format(FStr, [FormattedReason]))]), failed; progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, @@ -4246,8 +4248,8 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, "~ts</tr>\n", [TimeStr,Comment]), print(minor, - escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal])), - []), + "~ts", + [escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal]))]), ok. %%-------------------------------------------------------------------- @@ -4604,7 +4606,7 @@ timestamp_get(Leader) -> timestamp_get_internal(Leader, Format) -> {YY,MM,DD,H,M,S} = time_get(), - io_lib:format(Format, [Leader,YY,MM,DD,H,M,S]). + lists:flatten(io_lib:format(Format, [Leader,YY,MM,DD,H,M,S])). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% time_get() -> {YY,MM,DD,H,M,S} @@ -4976,7 +4978,7 @@ collect_files(Dir, Pattern, St, Mode) -> fullname_to_mod(Path) when is_list(Path) -> %% If this is called with a binary, then we are probably in +fnu %% mode and have found a beam file with name encoded as latin1. We - %% will let this crash since it can not work to load such a module + %% will let this crash since it cannot work to load such a module %% anyway. It should be removed or renamed! list_to_atom(filename:rootname(filename:basename(Path))). diff --git a/lib/common_test/src/test_server_node.erl b/lib/common_test/src/test_server_node.erl index 3ae4a047d8..f77d5a4966 100644 --- a/lib/common_test/src/test_server_node.erl +++ b/lib/common_test/src/test_server_node.erl @@ -18,7 +18,7 @@ %% %CopyrightEnd% %% -module(test_server_node). --compile(r16). +-compile(r20). %%% %%% The same compiled code for this module must be possible to load @@ -598,11 +598,20 @@ pick_erl_program(L) -> {prog, S} -> S; {release, S} -> + clear_erl_aflags(), find_release(S); this -> ct:get_progname() end. +clear_erl_aflags() -> + %% When starting a node with a previous release, options in + %% ERL_AFLAGS could prevent the node from starting. For example, + %% if ERL_AFLAGS is set to "-emu_type lcnt", the node will only + %% start if the previous release happens to also have a lock + %% counter emulator installed (unlikely). + os:unsetenv("ERL_AFLAGS"). + %% This is an attempt to distinguish between spaces in the program %% path and spaces that separate arguments. The program is quoted to %% allow spaces in the path. diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index 5ffc735d6a..ec5278b96d 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -211,18 +211,12 @@ reformat_events(Events, EH) -> %%% Test related to 'localtime' will often fail if the test host is %%% time warping, so let's just skip the 'dynamic' tests then. skip_dynamic() -> - case os:getenv("TS_EXTRA_PLATFORM_LABEL") of - TSExtraPlatformLabel when is_list(TSExtraPlatformLabel) -> - case string:find(TSExtraPlatformLabel,"TimeWarpingOS") of - nomatch -> false; - _ -> true - end; - _ -> - false + case string:find(os:getenv("TS_EXTRA_PLATFORM_LABEL", ""), "TimeWarpingOS") of + nomatch -> false; + _ -> true end. - %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl index 0d17481e95..69a7de1431 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl @@ -205,7 +205,7 @@ hello_required_exists(Config) -> SshDir = ?config(ssh_dir,Config), {ok,_Client1} = open_configured_success(my_named_connection,SshDir), - %% Check that same name can not be used twice + %% Check that same name cannot be used twice {error,{connection_exists,_Client1}} = ct_netconfc:open(my_named_connection,[{user_dir,SshDir}]), @@ -271,7 +271,7 @@ no_client_hello(Config) -> %% Tell server to receive a get request and then die without %% replying since no hello has been received. (is this correct - %% behavoiur??) + %% behaviour??) ?NS:expect_do(get,close), {error,closed} = ct_netconfc:get(Client,whatever), ok. @@ -385,7 +385,7 @@ timeout_get(Config) -> %% received, the timeout message might already be sent when the timer %% is cancelled. This test checks that the timeout message is flushed %% from the message queue. If it isn't, the client crashes and the -%% session can not be closed afterwards. +%% session cannot be closed afterwards. %% Note that we can only hope that the test case triggers the problem %% every now and then, as it is very timing dependent... flush_timeout_get(Config) -> diff --git a/lib/common_test/test/test_server_SUITE.erl b/lib/common_test/test/test_server_SUITE.erl index 05737cfac9..6e52f24025 100644 --- a/lib/common_test/test/test_server_SUITE.erl +++ b/lib/common_test/test/test_server_SUITE.erl @@ -260,7 +260,7 @@ translate_filename(Filename,EncodingOnTestNode) -> end. get_latest_run_dir(Dir) -> - %% For the time being, filelib:wildcard can not take a binary + %% For the time being, filelib:wildcard cannot take a binary %% argument, so we avoid using this here. case file:list_dir(Dir) of {ok,Files} -> @@ -315,7 +315,7 @@ generate_and_run_unicode_test(Config0,Encoding) -> DataDir = ?config(data_dir,Config0), Suite = create_unicode_test_suite(DataDir,Encoding), - %% We can not run this test on default node since it must be + %% We cannot run this test on default node since it must be %% started with correct file name mode (+fnu/+fnl). %% OBS: the node are stopped by end_per_testcase/2 Config1 = lists:keydelete(node,1,Config0), diff --git a/lib/common_test/test/test_server_test_lib.erl b/lib/common_test/test/test_server_test_lib.erl index 9ee946af0b..58b3aaee9b 100644 --- a/lib/common_test/test/test_server_test_lib.erl +++ b/lib/common_test/test/test_server_test_lib.erl @@ -22,7 +22,7 @@ -export([parse_suite/1]). -export([init/2, pre_init_per_testcase/3, post_end_per_testcase/4]). -%% for test_server_SUITE when node can not be started as slave +%% for test_server_SUITE when node cannot be started as slave -export([prepare_tester_node/2]). -include("test_server_test_lib.hrl"). diff --git a/lib/common_test/test_server/configure.in b/lib/common_test/test_server/configure.in index e07bd4c2aa..a32d050081 100644 --- a/lib/common_test/test_server/configure.in +++ b/lib/common_test/test_server/configure.in @@ -171,7 +171,7 @@ case $system in fi SHLIB_EXTRACT_ALL="" ;; - *-openbsd*) + *-openbsd*|*-netbsd*|*-freebsd*|*-dragonfly*) # Not available on all versions: check for include file. AC_CHECK_HEADER(dlfcn.h, [ SHLIB_CFLAGS="-fpic" @@ -194,29 +194,6 @@ case $system in ]) SHLIB_EXTRACT_ALL="" ;; - *-netbsd*|*-freebsd*|*-dragonfly*) - # Not available on all versions: check for include file. - AC_CHECK_HEADER(dlfcn.h, [ - SHLIB_CFLAGS="-fpic" - SHLIB_LD="ld" - SHLIB_LDFLAGS="$LDFLAGS -Bshareable -x" - SHLIB_SUFFIX=".so" - if test X${enable_m64_build} = Xyes; then - AC_MSG_ERROR(don't know how to link 64-bit dynamic drivers) - fi - if test X${enable_m32_build} = Xyes; then - AC_MSG_ERROR(don't know how to link 32-bit dynamic drivers) - fi - ], [ - # No dynamic loading. - SHLIB_CFLAGS="" - SHLIB_LD="ld" - SHLIB_LDFLAGS="" - SHLIB_SUFFIX="" - AC_MSG_ERROR(don't know how to compile and link dynamic drivers) - ]) - SHLIB_EXTRACT_ALL="" - ;; *-solaris2*|*-sysv4*) SHLIB_CFLAGS="-KPIC" SHLIB_LD="/usr/ccs/bin/ld" diff --git a/lib/common_test/test_server/ts_benchmark.erl b/lib/common_test/test_server/ts_benchmark.erl index e4e06b54c2..a9486262b3 100644 --- a/lib/common_test/test_server/ts_benchmark.erl +++ b/lib/common_test/test_server/ts_benchmark.erl @@ -45,7 +45,7 @@ run(Specs, Opts, Vars) -> || Spec <- Specs], file:delete(filename:join(Cwd,"latest_benchmark")), {ok,D} = file:open(filename:join(Cwd,"latest_benchmark"),[write]), - io:format(D,BDir,[]), + io:format(D,"~ts", [BDir]), file:close(D). diff --git a/lib/common_test/test_server/ts_install.erl b/lib/common_test/test_server/ts_install.erl index 048e5493d2..86f16fa072 100644 --- a/lib/common_test/test_server/ts_install.erl +++ b/lib/common_test/test_server/ts_install.erl @@ -112,12 +112,6 @@ get_vars([], name, [], Result) -> get_vars(_, _, _, _) -> {error, fatal_bad_conf_vars}. -config_flags() -> - case os:getenv("CONFIG_FLAGS") of - false -> []; - CF -> string:lexemes(CF, " \t\n") - end. - unix_autoconf(XConf) -> Configure = filename:absname("configure"), Flags = proplists:get_value(crossflags,XConf,[]), @@ -128,7 +122,7 @@ unix_autoconf(XConf) -> erlang:system_info(threads) /= false], Debug = [" --enable-debug-mode" || string:find(erlang:system_info(system_version),"debug") =/= nomatch], - MXX_Build = [Y || Y <- config_flags(), + MXX_Build = [Y || Y <- string:lexemes(os:getenv("CONFIG_FLAGS", ""), " \t\n"), Y == "--enable-m64-build" orelse Y == "--enable-m32-build"], Args = Host ++ Build ++ Threads ++ Debug ++ " " ++ MXX_Build, @@ -164,7 +158,7 @@ assign_vars(FlagsStr) -> assign_all_vars([$$ | Rest], FlagSoFar) -> {VarName,Rest1} = get_var_name(Rest, []), - assign_all_vars(Rest1, FlagSoFar ++ assign_var(VarName)); + assign_all_vars(Rest1, FlagSoFar ++ os:getenv(VarName, "")); assign_all_vars([Char | Rest], FlagSoFar) -> assign_all_vars(Rest, FlagSoFar ++ [Char]); assign_all_vars([], Flag) -> @@ -177,12 +171,6 @@ get_var_name([Ch | Rest] = Str, VarR) -> end; get_var_name([], VarR) -> {lists:reverse(VarR),[]}. - -assign_var(VarName) -> - case os:getenv(VarName) of - false -> ""; - Val -> Val - end. valid_char(Ch) when Ch >= $a, Ch =< $z -> true; valid_char(Ch) when Ch >= $A, Ch =< $Z -> true; @@ -280,7 +268,7 @@ add_vars(Vars0, Opts0) -> {Opts, [{longnames, LongNames}, {platform_id, PlatformId}, {platform_filename, PlatformFilename}, - {rsh_name, get_rsh_name()}, + {rsh_name, os:getenv("ERL_RSH", "ssh")}, {platform_label, PlatformLabel}, {ts_net_dir, Mounted}, {erl_flags, []}, @@ -301,16 +289,10 @@ get_testcase_callback() -> end end. -get_rsh_name() -> - case os:getenv("ERL_RSH") of - false -> "rsh"; - Str -> Str - end. - platform_id(Vars) -> {Id,_,_,_} = platform(Vars), Id. - + platform(Vars) -> Hostname = hostname(), diff --git a/lib/compiler/doc/src/Makefile b/lib/compiler/doc/src/Makefile index 32f150eef8..2fb163b9e7 100644 --- a/lib/compiler/doc/src/Makefile +++ b/lib/compiler/doc/src/Makefile @@ -31,6 +31,7 @@ APPLICATION=compiler # Release directory specification # ---------------------------------------------------- RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) +COMPILER_DIR = $(ERL_TOP)/lib/compiler/src # ---------------------------------------------------- # Target Specs @@ -38,7 +39,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) XML_APPLICATION_FILES = ref_man.xml XML_REF3_FILES = compile.xml -XML_PART_FILES = +XML_PART_FILES = internal.xml XML_CHAPTER_FILES = notes.xml BOOK_FILES = book.xml @@ -49,6 +50,9 @@ XML_FILES = \ $(BOOK_FILES) $(XML_CHAPTER_FILES) \ $(XML_PART_FILES) $(XML_REF3_FILES) $(XML_APPLICATION_FILES) +XML_INTERNAL_FILES = \ + cerl.xml cerl_trees.xml cerl_clauses.xml + # ---------------------------------------------------- HTML_FILES = $(XML_APPLICATION_FILES:%.xml=$(HTMLDIR)/%.html) \ @@ -62,6 +66,8 @@ HTML_REF_MAN_FILE = $(HTMLDIR)/index.html TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf +XML_GEN_FILES = $(XML_INTERNAL_FILES:%=$(XMLDIR)/%) + # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- @@ -85,6 +91,9 @@ man: $(MAN3_FILES) gifs: $(GIF_FILES:%=$(HTMLDIR)/%) +$(XML_INTERNAL_FILES:%=$(XMLDIR)/%): $(COMPILER_DIR)/$(@:$(XMLDIR)/%.xml=%.erl) + $(gen_verbose)escript $(DOCGEN)/priv/bin/xml_from_edoc.escript -def vsn $(COMPILER_VSN) -dir $(XMLDIR) $(COMPILER_DIR)/$(@:$(XMLDIR)/%.xml=%.erl) + debug opt: clean clean_docs: diff --git a/lib/compiler/doc/src/book.xml b/lib/compiler/doc/src/book.xml index af6b4cf47a..d101d40cb4 100644 --- a/lib/compiler/doc/src/book.xml +++ b/lib/compiler/doc/src/book.xml @@ -38,6 +38,9 @@ <applications> <xi:include href="ref_man.xml"/> </applications> + <internals> + <xi:include href="internal.xml"/> + </internals> <releasenotes> <xi:include href="notes.xml"/> </releasenotes> diff --git a/lib/compiler/doc/src/compile.xml b/lib/compiler/doc/src/compile.xml index 7f3d6aa60e..549b1049d8 100644 --- a/lib/compiler/doc/src/compile.xml +++ b/lib/compiler/doc/src/compile.xml @@ -416,6 +416,17 @@ module.beam: module.erl \ is not documented, and can change between releases.</p> </item> + <tag><c>no_spawn_compiler_process</c></tag> + <item> + <p>By default, all code is compiled in a separate process + which is terminated at the end of compilation. However, + some tools, like Dialyzer or compilers for other BEAM languages, + may already manage their own worker processes and spawning + an extra process may slow the compilation down. + In such scenarios, you can pass this option to stop the + compiler from spawning an additional process.</p> + </item> + <tag><c>no_strict_record_tests</c></tag> <item> <p>This option is not recommended.</p> @@ -621,6 +632,22 @@ module.beam: module.erl \ to be deprecated.</p> </item> + <tag><c>nowarn_removed</c></tag> + <item> + <p>Turns off warnings for calls to functions that have + been removed. Default is to emit warnings for every call + to a function known by the compiler to have been recently + removed from Erlang/OTP.</p> + </item> + + <tag><c>{nowarn_removed, ModulesOrMFAs}</c></tag> + <item> + <p>Turns off warnings for calls to modules or functions + that have been removed. Default is to emit warnings for + every call to a function known by the compiler to have + been recently removed from Erlang/OTP.</p> + </item> + <tag><c>nowarn_obsolete_guard</c></tag> <item> <p>Turns off warnings for calls to old type testing BIFs, @@ -684,12 +711,13 @@ module.beam: module.erl \ </note> <note> - <p>The options <c>{nowarn_unused_function, FAs}</c>, - <c>{nowarn_bif_clash, FAs}</c>, and - <c>{nowarn_deprecated_function, MFAs}</c> are only - recognized when given in files. They are not affected by - options <c>warn_unused_function</c>, <c>warn_bif_clash</c>, or - <c>warn_deprecated_function</c>.</p> + <p>Before OTP 22, the option <c>{nowarn_deprecated_function, + MFAs}</c> was only recognized when given in the file with + attribute <c>-compile()</c>. (The option + <c>{nowarn_unused_function,FAs}</c> was incorrectly documented + to only work in a file, but it also worked when given in the + option list.) Starting from OTP 22, all options that can be + given in the file can also be given in the option list.</p> </note> <p>For debugging of the compiler, or for pure curiosity, diff --git a/lib/otp_mibs/doc/src/part.xml b/lib/compiler/doc/src/internal.xml index 0a8ddce268..f24b363c1c 100644 --- a/lib/otp_mibs/doc/src/part.xml +++ b/lib/compiler/doc/src/internal.xml @@ -1,10 +1,10 @@ <?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE part SYSTEM "part.dtd"> -<part xmlns:xi="http://www.w3.org/2001/XInclude"> +<internal xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>2003</year><year>2016</year> + <year>2018</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -22,18 +22,17 @@ </legalnotice> - <title>OTP_Mibs User's Guide</title> - <prepared>Ingela Anderton</prepared> + <title>Compiler Internal Documentation</title> + <prepared>Lukas Larsson</prepared> <docno></docno> - <date>2002-09-17</date> - <rev>A</rev> + <date>2018-07-07</date> + <rev>1.0.0</rev> + <file>internal.xml</file> </header> <description> - <p>The <em>OTP_Mibs</em> application provides an SNMP management - information base for Erlang nodes.</p> </description> - <xi:include href="introduction.xml"/> - <xi:include href="mibs.xml"/> -</part> - + <xi:include href="cerl.xml"/> + <xi:include href="cerl_trees.xml"/> + <xi:include href="cerl_clauses.xml"/> +</internal> diff --git a/lib/compiler/scripts/.gitignore b/lib/compiler/scripts/.gitignore new file mode 100644 index 0000000000..4e4eba766d --- /dev/null +++ b/lib/compiler/scripts/.gitignore @@ -0,0 +1 @@ +/smoke-build diff --git a/lib/compiler/scripts/smoke b/lib/compiler/scripts/smoke new file mode 100755 index 0000000000..ae31c923b8 --- /dev/null +++ b/lib/compiler/scripts/smoke @@ -0,0 +1,123 @@ +#!/usr/bin/env escript +%% -*- erlang -*- +-mode(compile). + +main(_Args) -> + setup(), + clone_elixir(), + build_elixir(), + test_elixir(), + setup_mix(), + smoke(main), + smoke(rabbitmq), + halt(0). + +setup() -> + ScriptsDir = scripts_dir(), + SmokeBuildDir = filename:join(ScriptsDir, "smoke-build"), + _ = file:make_dir(SmokeBuildDir), + ok = file:set_cwd(SmokeBuildDir), + ok. + +clone_elixir() -> + {ok,SmokeDir} = file:get_cwd(), + DotGitDir = filename:join([SmokeDir,"elixir",".git"]), + ElixirRepo = "[email protected]:elixir-lang/elixir.git", + case filelib:is_dir(DotGitDir) of + false -> + cmd("git clone " ++ ElixirRepo); + true -> + GetHeadSHA1 = "cd elixir && git rev-parse --verify HEAD", + Before = os:cmd(GetHeadSHA1), + cmd("cd elixir && git pull --ff-only origin master"), + case os:cmd(GetHeadSHA1) of + Before -> + ok; + _After -> + %% There were some changes. Clean to force a re-build. + cmd("cd elixir && make clean") + end + end. + +build_elixir() -> + cmd("cd elixir && make compile"). + +test_elixir() -> + cmd("cd elixir && make test_stdlib"). + +setup_mix() -> + MixExsFile = filename:join(scripts_dir(), "smoke-mix.exs"), + {ok,MixExs} = file:read_file(MixExsFile), + ok = file:write_file("mix.exs", MixExs), + + {ok,SmokeDir} = file:get_cwd(), + ElixirBin = filename:join([SmokeDir,"elixir","bin"]), + PATH = ElixirBin ++ ":" ++ os:getenv("PATH"), + os:putenv("PATH", PATH), + mix("local.hex --force"), + mix("local.rebar --force"), + ok. + +smoke(Set) -> + os:putenv("SMOKE_DEPS_SET", atom_to_list(Set)), + _ = file:delete("mix.lock"), + cmd("touch mix.exs"), + mix("deps.clean --all"), + mix("deps.get"), + mix("deps.compile"), + ok. + +scripts_dir() -> + Root = code:lib_dir(compiler), + filename:join(Root, "scripts"). + +mix(Cmd) -> + cmd("mix " ++ Cmd). + +cmd(Cmd) -> + run("sh", ["-c",Cmd]). + +run(Program0, Args) -> + Program = case os:find_executable(Program0) of + Path when is_list(Path) -> + Path; + false -> + abort("Unable to find program: ~s\n", [Program0]) + end, + Cmd = case {Program0,Args} of + {"sh",["-c"|ShCmd]} -> + ShCmd; + {_,_} -> + lists:join(" ", [Program0|Args]) + end, + io:format("\n# ~s\n", [Cmd]), + Options = [{args,Args},binary,exit_status,stderr_to_stdout], + try open_port({spawn_executable,Program}, Options) of + Port -> + case run_loop(Port, <<>>) of + 0 -> + ok; + ExitCode -> + abort("*** Failed with exit code: ~p\n", + [ExitCode]) + end + catch + error:_ -> + abort("Failed to execute ~s\n", [Program0]) + end. + +run_loop(Port, Output) -> + receive + {Port,{exit_status,Status}} -> + Status; + {Port,{data,Bin}} -> + io:put_chars(Bin), + run_loop(Port, <<Output/binary,Bin/binary>>); + Msg -> + io:format("L: ~p~n", [Msg]), + run_loop(Port, Output) + end. + +abort(Format, Args) -> + io:format(Format, Args), + halt(1). diff --git a/lib/compiler/scripts/smoke-mix.exs b/lib/compiler/scripts/smoke-mix.exs new file mode 100644 index 0000000000..ba0815e465 --- /dev/null +++ b/lib/compiler/scripts/smoke-mix.exs @@ -0,0 +1,103 @@ +defmodule Smoke.MixProject do + use Mix.Project + + def project do + [ + app: :smoke, + version: "0.1.0", + elixir: "~> 1.8", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications. + def application do + [ + extra_applications: [:logger] + ] + end + + # Run "mix help deps" to learn about dependencies. + defp deps do + case :os.getenv('SMOKE_DEPS_SET') do + 'main' -> + [ + {:bear, "~> 0.8.7"}, + {:cloudi_core, "~> 1.7"}, + {:cloudi_service_monitoring, "~> 1.7"}, + {:cloudi_service_tcp, "~> 1.7"}, + {:cloudi_service_queue, "~> 1.7"}, + {:cloudi_service_udp, "~> 1.7"}, + {:cloudi_service_map_reduce, "~> 1.7"}, + {:cloudi_service_api_requests, "~> 1.7"}, + {:cloudi_service_router, "~> 1.7"}, + {:cloudi_service_request_rate, "~> 1.7"}, + {:concuerror, "~> 0.20.0"}, + {:cowboy, "~> 2.6.1"}, + {:ecto, "~> 3.0.6"}, + {:ex_doc, "~> 0.19.3"}, + {:distillery, "~> 2.0.12"}, + {:erlydtl, "~> 0.12.1"}, + {:gen_smtp, "~> 0.13.0"}, + {:getopt, "~> 1.0.1"}, + {:gettext, "~> 0.16.1"}, + {:gpb, "~> 4.6"}, + {:gproc, "~> 0.8.0"}, + {:graphql, "~> 0.15.0", hex: :graphql_erl}, + {:hackney, "~> 1.15.0"}, + {:ibrowse, "~> 4.4.1"}, + {:jose, "~> 1.9.0"}, + {:lager, "~> 3.6"}, + {:locus, "~> 1.6"}, + {:nimble_parsec, "~> 0.5.0"}, + {:phoenix, "~> 1.4.0"}, + {:riak_pb, "~> 2.3"}, + {:scalaris, git: "https://github.com/scalaris-team/scalaris", + compile: build_scalaris()}, + {:tdiff, "~> 0.1.2"}, + {:webmachine, "~> 1.11"}, + {:wings, git: "https://github.com/dgud/wings.git", + compile: build_wings()}, + {:zotonic_stdlib, "~> 1.0"}, + ] + 'rabbitmq' -> + [{:rabbit_common, "~> 3.7"}] + _ -> + [] + end + end + + defp build_scalaris do + # Only compile the Erlang code. + + """ + echo '-include("rt_simple.hrl").' >include/rt.hrl + (cd src && erlc -W0 -I ../include -I ../contrib/log4erl/include -I ../contrib/yaws/include *.erl) + (cd src/comm_layer && erlc -W0 -I ../../include -I *.erl) + (cd src/cp && erlc -W0 -I ../../include -I *.erl) + (cd src/crdt && erlc -W0 -I ../../include -I *.erl) + (cd src/json && erlc -W0 -I ../../include -I *.erl) + (cd src/paxos && erlc -W0 -I ../../include -I *.erl) + (cd src/rbr && erlc -W0 -I ../../include -I *.erl) + (cd src/rrepair && erlc -W0 -I ../../include -I *.erl) + (cd src/time && erlc -W0 -I ../../include -I *.erl) + (cd src/transactions && erlc -W0 -I ../../include -I *.erl) + (cd src/tx && erlc -W0 -I ../../include -I *.erl) + """ + end + + defp build_wings do + # If the Erlang system is not installed, the build will + # crash in plugins_src/accel when attempting to build + # the accel driver. Since there is very little Erlang code in + # the directory, skip the entire directory. + + """ + echo "all:\n\t" >plugins_src/accel/Makefile + git commit -a -m'Disable for smoke testing' + git tag -a -m'Smoke test' vsmoke_test + make + """ + end +end diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index 2408c76b48..87b0d345f2 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -49,10 +49,7 @@ MODULES = \ beam_a \ beam_asm \ beam_block \ - beam_bs \ - beam_bsm \ beam_clean \ - beam_dead \ beam_dict \ beam_disasm \ beam_except \ @@ -61,12 +58,20 @@ MODULES = \ beam_listing \ beam_opcodes \ beam_peep \ - beam_receive \ - beam_reorder \ - beam_record \ - beam_split \ + beam_ssa \ + beam_ssa_bsm \ + beam_ssa_codegen \ + beam_ssa_dead \ + beam_ssa_funs \ + beam_ssa_lint \ + beam_ssa_opt \ + beam_ssa_pp \ + beam_ssa_pre_codegen \ + beam_ssa_recv \ + beam_ssa_share \ + beam_ssa_type \ + beam_kernel_to_ssa \ beam_trim \ - beam_type \ beam_utils \ beam_validator \ beam_z \ @@ -85,12 +90,10 @@ MODULES = \ rec_env \ sys_core_alias \ sys_core_bsm \ - sys_core_dsetel \ sys_core_fold \ sys_core_fold_lists \ sys_core_inline \ sys_pre_attributes \ - v3_codegen \ v3_core \ v3_kernel \ v3_kernel_pp @@ -99,6 +102,8 @@ BEAM_H = $(wildcard ../priv/beam_h/*.h) HRL_FILES= \ beam_disasm.hrl \ + beam_ssa_opt.hrl \ + beam_ssa.hrl \ core_parse.hrl \ v3_kernel.hrl @@ -124,9 +129,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif ERL_COMPILE_FLAGS += +inline +warn_unused_import \ - -Werror \ -I../../stdlib/include -I$(EGEN) -W +warn_missing_spec # ---------------------------------------------------- @@ -185,7 +191,20 @@ release_docs_spec: # ---------------------------------------------------- $(EBIN)/beam_disasm.beam: $(EGEN)/beam_opcodes.hrl beam_disasm.hrl -$(EBIN)/beam_listing.beam: core_parse.hrl v3_kernel.hrl +$(EBIN)/beam_listing.beam: core_parse.hrl v3_kernel.hrl beam_ssa.hrl +$(EBIN)/beam_kernel_to_ssa.beam: v3_kernel.hrl beam_ssa.hrl +$(EBIN)/beam_ssa.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_bsm.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_codegen.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_dead.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_funs.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_lint.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_opt.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_pp.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_pre_codegen.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_recv.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_share.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_type.beam: beam_ssa.hrl $(EBIN)/cerl.beam: core_parse.hrl $(EBIN)/compile.beam: core_parse.hrl ../../stdlib/include/erl_compile.hrl $(EBIN)/core_lib.beam: core_parse.hrl @@ -193,11 +212,9 @@ $(EBIN)/core_lint.beam: core_parse.hrl $(EBIN)/core_parse.beam: core_parse.hrl $(EGEN)/core_parse.erl $(EBIN)/core_pp.beam: core_parse.hrl $(EBIN)/sys_core_alias.beam: core_parse.hrl -$(EBIN)/sys_core_dsetel.beam: core_parse.hrl $(EBIN)/sys_core_fold.beam: core_parse.hrl $(EBIN)/sys_core_fold_lists.beam: core_parse.hrl $(EBIN)/sys_core_inline.beam: core_parse.hrl -$(EBIN)/v3_codegen.beam: v3_kernel.hrl $(EBIN)/v3_core.beam: core_parse.hrl $(EBIN)/v3_kernel.beam: core_parse.hrl v3_kernel.hrl $(EBIN)/v3_kernel_pp.beam: v3_kernel.hrl diff --git a/lib/compiler/src/beam_a.erl b/lib/compiler/src/beam_a.erl index 6fd4ace540..0bccad1ecd 100644 --- a/lib/compiler/src/beam_a.erl +++ b/lib/compiler/src/beam_a.erl @@ -39,14 +39,29 @@ function({function,Name,Arity,CLabel,Is0}) -> %% Remove unusued labels for cleanliness and to help %% optimization passes and HiPE. - Is = beam_jump:remove_unused_labels(Is1), - {function,Name,Arity,CLabel,Is} + Is2 = beam_jump:remove_unused_labels(Is1), + + %% Some optimization passes can't handle consecutive labels. + %% Coalesce multiple consecutive labels. + Is = coalesce_consecutive_labels(Is2, [], []), + + {function,Name,Arity,CLabel,Is} catch Class:Error:Stack -> io:fwrite("Function: ~w/~w\n", [Name,Arity]), erlang:raise(Class, Error, Stack) end. +rename_instrs([{test,is_eq_exact,_,[Dst,Src]}=Test, + {move,Src,Dst}|Is]) -> + %% The move instruction is not needed. + rename_instrs([Test|Is]); +rename_instrs([{test,is_eq_exact,_,[Same,Same]}|Is]) -> + %% Same literal or same register. Will always succeed. + rename_instrs(Is); +rename_instrs([{loop_rec,{f,Fail},{x,0}},{loop_rec_end,_},{label,Fail}|Is]) -> + %% This instruction sequence does nothing. + rename_instrs(Is); rename_instrs([{apply_last,A,N}|Is]) -> [{apply,A},{deallocate,N},return|rename_instrs(Is)]; rename_instrs([{call_last,A,F,N}|Is]) -> @@ -85,8 +100,12 @@ rename_instr({bs_put_utf16=I,F,Fl,Src}) -> {bs_put,F,{I,Fl},[Src]}; rename_instr({bs_put_utf32=I,F,Fl,Src}) -> {bs_put,F,{I,Fl},[Src]}; -rename_instr({bs_put_string,_,_}=I) -> - {bs_put,{f,0},I,[]}; +rename_instr({bs_put_string,_,{string,String}}) -> + %% Only happens when compiling from .S files. In old + %% .S files, String is a list. In .S in OTP 22 and later, + %% String is a binary. + {bs_put,{f,0},{bs_put_binary,8,{field_flags,[unsigned,big]}}, + [{atom,all},{literal,iolist_to_binary([String])}]}; rename_instr({bs_add=I,F,[Src1,Src2,U],Dst}) when is_integer(U) -> {bif,I,F,[Src1,Src2,{integer,U}],Dst}; rename_instr({bs_utf8_size=I,F,Src,Dst}) -> @@ -103,16 +122,14 @@ rename_instr({bs_private_append=I,F,Sz,U,Src,Flags,Dst}) -> {bs_init,F,{I,U,Flags},none,[Sz,Src],Dst}; rename_instr(bs_init_writable=I) -> {bs_init,{f,0},I,1,[{x,0}],{x,0}}; -rename_instr({test,Op,F,[Ctx,Bits,{string,Str}]}) -> - %% When compiling from a .S file. - <<Bs:Bits/bits,_/bits>> = list_to_binary(Str), - {test,Op,F,[Ctx,Bs]}; rename_instr({put_map_assoc,Fail,S,D,R,L}) -> {put_map,Fail,assoc,S,D,R,L}; rename_instr({put_map_exact,Fail,S,D,R,L}) -> {put_map,Fail,exact,S,D,R,L}; rename_instr({test,has_map_fields,Fail,Src,{list,List}}) -> {test,has_map_fields,Fail,[Src|List]}; +rename_instr({test,is_nil,Fail,[Src]}) -> + {test,is_eq_exact,Fail,[Src,nil]}; rename_instr({select_val=I,Reg,Fail,{list,List}}) -> {select,I,Reg,Fail,List}; rename_instr({select_tuple_arity=I,Reg,Fail,{list,List}}) -> @@ -120,3 +137,11 @@ rename_instr({select_tuple_arity=I,Reg,Fail,{list,List}}) -> rename_instr(send) -> {call_ext,2,send}; rename_instr(I) -> I. + +coalesce_consecutive_labels([{label,L}=Lbl,{label,Alias}|Is], Replace, Acc) -> + coalesce_consecutive_labels([Lbl|Is], [{Alias,L}|Replace], Acc); +coalesce_consecutive_labels([I|Is], Replace, Acc) -> + coalesce_consecutive_labels(Is, Replace, [I|Acc]); +coalesce_consecutive_labels([], Replace, Acc) -> + D = maps:from_list(Replace), + beam_utils:replace_labels(Acc, [], D, fun(L) -> L end). diff --git a/lib/compiler/src/beam_asm.erl b/lib/compiler/src/beam_asm.erl index df0321e85a..df09dcb06c 100644 --- a/lib/compiler/src/beam_asm.erl +++ b/lib/compiler/src/beam_asm.erl @@ -407,14 +407,14 @@ encode_arg({atom, Atom}, Dict0) when is_atom(Atom) -> {Index, Dict} = beam_dict:atom(Atom, Dict0), {encode(?tag_a, Index), Dict}; encode_arg({integer, N}, Dict) -> - %% Conservatily assume that all integers whose absolute + %% Conservatively assume that all integers whose absolute %% value is greater than 1 bsl 128 will be bignums in %% the runtime system. if N >= 1 bsl 128 -> - encode_arg({literal, N}, Dict); + encode_literal(N, Dict); N =< -(1 bsl 128) -> - encode_arg({literal, N}, Dict); + encode_literal(N, Dict); true -> {encode(?tag_i, N), Dict} end; @@ -424,8 +424,8 @@ encode_arg({f, W}, Dict) -> {encode(?tag_f, W), Dict}; %% encode_arg({'char', C}, Dict) -> %% {encode(?tag_h, C), Dict}; -encode_arg({string, String}, Dict0) -> - {Offset, Dict} = beam_dict:string(String, Dict0), +encode_arg({string, BinString}, Dict0) when is_binary(BinString) -> + {Offset, Dict} = beam_dict:string(BinString, Dict0), {encode(?tag_u, Offset), Dict}; encode_arg({extfunc, M, F, A}, Dict0) -> {Index, Dict} = beam_dict:import(M, F, A, Dict0), @@ -434,7 +434,7 @@ encode_arg({list, List}, Dict0) -> {L, Dict} = encode_list(List, Dict0, []), {[encode(?tag_z, 1), encode(?tag_u, length(List))|L], Dict}; encode_arg({float, Float}, Dict) when is_float(Float) -> - encode_arg({literal,Float}, Dict); + encode_literal(Float, Dict); encode_arg({fr,Fr}, Dict) -> {[encode(?tag_z, 2),encode(?tag_u, Fr)], Dict}; encode_arg({field_flags,Flags0}, Dict) -> @@ -442,12 +442,24 @@ encode_arg({field_flags,Flags0}, Dict) -> {encode(?tag_u, Flags), Dict}; encode_arg({alloc,List}, Dict) -> encode_alloc_list(List, Dict); -encode_arg({literal,Lit}, Dict0) -> - {Index,Dict} = beam_dict:literal(Lit, Dict0), - {[encode(?tag_z, 4),encode(?tag_u, Index)],Dict}; +encode_arg({literal,Lit}, Dict) -> + if + Lit =:= [] -> + encode_arg(nil, Dict); + is_atom(Lit) -> + encode_arg({atom,Lit}, Dict); + is_integer(Lit) -> + encode_arg({integer,Lit}, Dict); + true -> + encode_literal(Lit, Dict) + end; encode_arg(Int, Dict) when is_integer(Int) -> {encode(?tag_u, Int),Dict}. +encode_literal(Literal, Dict0) -> + {Index,Dict} = beam_dict:literal(Literal, Dict0), + {[encode(?tag_z, 4),encode(?tag_u, Index)],Dict}. + %%flag_to_bit(aligned) -> 16#01; %% No longer useful. flag_to_bit(little) -> 16#02; flag_to_bit(big) -> 16#00; diff --git a/lib/compiler/src/beam_block.erl b/lib/compiler/src/beam_block.erl index fe43163455..707974b2c1 100644 --- a/lib/compiler/src/beam_block.erl +++ b/lib/compiler/src/beam_block.erl @@ -17,39 +17,24 @@ %% %% %CopyrightEnd% %% -%% Purpose : Partitions assembly instructions into basic blocks and -%% optimizes them. +%% Purpose: Partition BEAM instructions into basic blocks. -module(beam_block). -export([module/2]). --import(lists, [reverse/1,reverse/2,member/2]). +-import(lists, [keysort/2,reverse/1,splitwith/2]). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. -module({Mod,Exp,Attr,Fs0,Lc}, Opts) -> - Blockify = not member(no_blockify, Opts), - Fs = [function(F, Blockify) || F <- Fs0], +module({Mod,Exp,Attr,Fs0,Lc}, _Opts) -> + Fs = [function(F) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. -function({function,Name,Arity,CLabel,Is0}, Blockify) -> +function({function,Name,Arity,CLabel,Is0}) -> try - %% Collect basic blocks and optimize them. - Is1 = case Blockify of - false -> Is0; - true -> blockify(Is0) - end, - Is2 = embed_lines(Is1), - Is3 = local_cse(Is2), - Is4 = beam_utils:anno_defs(Is3), - Is5 = move_allocates(Is4), - Is6 = beam_utils:live_opt(Is5), - Is7 = opt_blocks(Is6), - Is8 = beam_utils:delete_annos(Is7), - Is = opt_allocs(Is8), - - %% Done. + Is1 = blockify(Is0), + Is = embed_lines(Is1), {function,Name,Arity,CLabel,Is} catch Class:Error:Stack -> @@ -64,14 +49,12 @@ function({function,Name,Arity,CLabel,Is0}, Blockify) -> blockify(Is) -> blockify(Is, []). -blockify([{loop_rec,{f,Fail},{x,0}},{loop_rec_end,_Lbl},{label,Fail}|Is], Acc) -> - %% Useless instruction sequence. - blockify(Is, Acc); blockify([I|Is0]=IsAll, Acc) -> case collect(I) of error -> blockify(Is0, [I|Acc]); Instr when is_tuple(Instr) -> - {Block,Is} = collect_block(IsAll), + {Block0,Is} = collect_block(IsAll), + Block = sort_moves(Block0), blockify(Is, [{block,Block}|Acc]) end; blockify([], Acc) -> reverse(Acc). @@ -80,12 +63,10 @@ collect_block(Is) -> collect_block(Is, []). collect_block([{allocate,N,R}|Is0], Acc) -> - {Inits,Is} = lists:splitwith(fun ({init,{y,_}}) -> true; - (_) -> false - end, Is0), + {Inits,Is} = splitwith(fun ({init,{y,_}}) -> true; + (_) -> false + end, Is0), collect_block(Is, [{set,[],[],{alloc,R,{nozero,N,0,Inits}}}|Acc]); -collect_block([{allocate_zero,Ns,R},{test_heap,Nh,R}|Is], Acc) -> - collect_block(Is, [{set,[],[],{alloc,R,{zero,Ns,Nh,[]}}}|Acc]); collect_block([I|Is]=Is0, Acc) -> case collect(I) of error -> {reverse(Acc),Is0}; @@ -100,23 +81,20 @@ collect({allocate_heap,Ns,Nh,R}) -> {set,[],[],{alloc,R,{nozero,Ns,Nh,[]}}}; collect({allocate_heap_zero,Ns,Nh,R}) -> {set,[],[],{alloc,R,{zero,Ns,Nh,[]}}}; collect({init,D}) -> {set,[D],[],init}; collect({test_heap,N,R}) -> {set,[],[],{alloc,R,{nozero,nostack,N,[]}}}; -collect({bif,N,F,As,D}) -> {set,[D],As,{bif,N,F}}; -collect({gc_bif,N,F,R,As,D}) -> {set,[D],As,{alloc,R,{gc_bif,N,F}}}; +collect({bif,N,{f,0},As,D}) -> {set,[D],As,{bif,N,{f,0}}}; +collect({gc_bif,N,{f,0},R,As,D}) -> {set,[D],As,{alloc,R,{gc_bif,N,{f,0}}}}; collect({move,S,D}) -> {set,[D],[S],move}; collect({put_list,S1,S2,D}) -> {set,[D],[S1,S2],put_list}; collect({put_tuple,A,D}) -> {set,[D],[],{put_tuple,A}}; collect({put,S}) -> {set,[],[S],put}; +collect({put_tuple2,D,{list,Els}}) -> {set,[D],Els,put_tuple2}; collect({get_tuple_element,S,I,D}) -> {set,[D],[S],{get_tuple_element,I}}; collect({set_tuple_element,S,D,I}) -> {set,[],[S,D],{set_tuple_element,I}}; collect({get_hd,S,D}) -> {set,[D],[S],get_hd}; collect({get_tl,S,D}) -> {set,[D],[S],get_tl}; collect(remove_message) -> {set,[],[],remove_message}; -collect({put_map,F,Op,S,D,R,{list,Puts}}) -> - {set,[D],[S|Puts],{alloc,R,{put_map,Op,F}}}; -collect({'catch'=Op,R,L}) -> - {set,[R],[],{try_catch,Op,L}}; -collect({'try'=Op,R,L}) -> - {set,[R],[],{try_catch,Op,L}}; +collect({put_map,{f,0},Op,S,D,R,{list,Puts}}) -> + {set,[D],[S|Puts],{alloc,R,{put_map,Op,{f,0}}}}; collect(fclearerror) -> {set,[],[],fclearerror}; collect({fcheckerror,{f,0}}) -> {set,[],[],fcheckerror}; collect({fmove,S,D}) -> {set,[D],[S],fmove}; @@ -137,557 +115,39 @@ embed_lines([{block,B2},{line,_}=Line,{block,B1}|T], Acc) -> embed_lines([{block,B1},{line,_}=Line|T], Acc) -> B = {block,[{set,[],[],Line}|B1]}, embed_lines([B|T], Acc); -embed_lines([{block,B2},{block,B1}|T], Acc) -> - %% This can only happen when beam_block is run for - %% the second time. - B = {block,B1++B2}, - embed_lines([B|T], Acc); embed_lines([I|Is], Acc) -> embed_lines(Is, [I|Acc]); embed_lines([], Acc) -> Acc. -opt_blocks([{block,Bl0}|Is]) -> - %% The live annotation at the beginning is not useful. - [{'%anno',_}|Bl] = Bl0, - [{block,opt_block(Bl)}|opt_blocks(Is)]; -opt_blocks([I|Is]) -> - [I|opt_blocks(Is)]; -opt_blocks([]) -> []. - -opt_block(Is0) -> - find_fixpoint(fun(Is) -> - opt_tuple_element(opt(Is)) - end, Is0). - -find_fixpoint(OptFun, Is0) -> - case OptFun(Is0) of - Is0 -> Is0; - Is1 -> find_fixpoint(OptFun, Is1) - end. - -%% move_allocates(Is0) -> Is -%% Move allocate instructions upwards in the instruction stream -%% (within the same block), in the hope of getting more possibilities -%% for optimizing away moves later. -%% -%% For example, we can transform the following instructions: -%% -%% get_tuple_element x(1) Element => x(2) -%% allocate_zero StackSize 3 %% x(0), x(1), x(2) are live -%% -%% to the following instructions: -%% -%% allocate_zero StackSize 2 %% x(0) and x(1) are live -%% get_tuple_element x(1) Element => x(2) -%% -%% NOTE: Since the beam_reorder pass has been run, it is no longer -%% safe to assume that if x(N) is initialized, then all lower-numbered -%% x registers are also initialized. -%% -%% For example, we must be careful when transforming the following -%% instructions: -%% -%% get_tuple_element x(0) Element => x(1) -%% allocate_zero StackSize 3 %x(0), x(1), x(2) are live -%% -%% to the following instructions: -%% -%% allocate_zero StackSize 3 -%% get_tuple_element x(0) Element => x(1) -%% -%% The transformation is safe if and only if x(1) has been -%% initialized previously. We will use the annotations added by -%% beam_utils:anno_defs/1 to determine whether x(a) has been -%% initialized. - -move_allocates([{block,Bl0}|Is]) -> - Bl = move_allocates_1(reverse(Bl0), []), - [{block,Bl}|move_allocates(Is)]; -move_allocates([I|Is]) -> - [I|move_allocates(Is)]; -move_allocates([]) -> []. - -move_allocates_1([{'%anno',_}|Is], Acc) -> - move_allocates_1(Is, Acc); -move_allocates_1([I|Is], [{set,[],[],{alloc,Live0,Info0}}|Acc]=Acc0) -> - case alloc_may_pass(I) of - false -> - move_allocates_1(Is, [I|Acc0]); - true -> - case alloc_live_regs(I, Is, Live0) of - not_possible -> - move_allocates_1(Is, [I|Acc0]); - Live when is_integer(Live) -> - Info = safe_info(Info0), - A = {set,[],[],{alloc,Live,Info}}, - move_allocates_1(Is, [A,I|Acc]) - end - end; -move_allocates_1([I|Is], Acc) -> - move_allocates_1(Is, [I|Acc]); -move_allocates_1([], Acc) -> Acc. - -alloc_may_pass({set,_,[{fr,_}],fmove}) -> false; -alloc_may_pass({set,_,_,{alloc,_,_}}) -> false; -alloc_may_pass({set,_,_,{set_tuple_element,_}}) -> false; -alloc_may_pass({set,_,_,put_list}) -> false; -alloc_may_pass({set,_,_,put}) -> false; -alloc_may_pass({set,_,_,_}) -> true. - -safe_info({nozero,Stack,Heap,_}) -> - %% nozero is not safe if the allocation instruction is moved - %% upwards past an instruction that may throw an exception - %% (such as element/2). - {zero,Stack,Heap,[]}; -safe_info(Info) -> Info. - -%% opt([Instruction]) -> [Instruction] -%% Optimize the instruction stream inside a basic block. - -opt([{set,[X],[X],move}|Is]) -> opt(Is); -opt([{set,[Dst],_,move},{set,[Dst],[Src],move}=I|Is]) when Dst =/= Src -> - opt([I|Is]); -opt([{set,[{x,0}],[S1],move}=I1,{set,[D2],[{x,0}],move}|Is]) -> - opt([I1,{set,[D2],[S1],move}|Is]); -opt([{set,[{x,0}],[S1],move}=I1,{set,[D2],[S2],move}|Is0]) when S1 =/= D2 -> - %% Place move S x0 at the end of move sequences so that - %% loader can merge with the following instruction - {Ds,Is} = opt_moves([D2], Is0), - [{set,Ds,[S2],move}|opt([I1|Is])]; -opt([{set,_,_,{line,_}}=Line1, - {set,[D1],[{integer,Idx1},Reg],{bif,element,{f,0}}}=I1, - {set,_,_,{line,_}}=Line2, - {set,[D2],[{integer,Idx2},Reg],{bif,element,{f,0}}}=I2|Is]) - when Idx1 < Idx2, D1 =/= D2, D1 =/= Reg, D2 =/= Reg -> - opt([Line2,I2,Line1,I1|Is]); -opt([{set,[D1],[{integer,Idx1},Reg],{bif,element,{f,L}}}=I1, - {set,[D2],[{integer,Idx2},Reg],{bif,element,{f,L}}}=I2|Is]) - when Idx1 < Idx2, D1 =/= D2, D1 =/= Reg, D2 =/= Reg -> - opt([I2,I1|Is]); -opt([{set,Hd0,Cons,get_hd}=GetHd, - {set,Tl0,Cons,get_tl}=GetTl|Is0]) -> - case {opt_moves(Hd0, [GetTl|Is0]),opt_moves(Tl0, [GetHd|Is0])} of - {{Hd0,Is},{Tl0,_}} -> - [GetHd|opt(Is)]; - {{Hd,Is},{Tl0,_}} -> - [{set,Hd,Cons,get_hd}|opt(Is)]; - {{_,_},{Tl,Is}} -> - [{set,Tl,Cons,get_tl}|opt(Is)] - end; -opt([{set,Ds0,Ss,Op}|Is0]) -> - {Ds,Is} = opt_moves(Ds0, Is0), - [{set,Ds,Ss,Op}|opt(Is)]; -opt([{'%anno',_}=I|Is]) -> - [I|opt(Is)]; -opt([]) -> []. - -%% opt_moves([Dest], [Instruction]) -> {[Dest],[Instruction]} -%% For each Dest, does the optimization described in opt_move/2. - -opt_moves([], Is0) -> {[],Is0}; -opt_moves([D0]=Ds, Is0) -> - case opt_move(D0, Is0) of - not_possible -> {Ds,Is0}; - {D1,Is} -> {[D1],Is} - end. - -%% opt_move(Dest, [Instruction]) -> {UpdatedDest,[Instruction]} | not_possible -%% If there is a {move,Dest,FinalDest} instruction -%% in the instruction stream, remove the move instruction -%% and let FinalDest be the destination. - -opt_move(Dest, Is) -> - opt_move_1(Dest, Is, []). - -opt_move_1(R, [{set,[D],[R],move}|Is0], Acc) -> - %% Provided that the source register is killed by instructions - %% that follow, the optimization is safe. - case eliminate_use_of_from_reg(Is0, R, D) of - {yes,Is} -> opt_move_rev(D, Acc, Is); - no -> not_possible - end; -opt_move_1(_R, [{set,_,_,{alloc,_,_}}|_], _) -> - %% The optimization is either not possible or not safe. - %% - %% If R is an X register killed by allocation, the optimization is - %% not safe. On the other hand, if the X register is killed, there - %% will not follow a 'move' instruction with this X register as - %% the source. - %% - %% If R is a Y register, the optimization is still not safe - %% because the new target register is an X register that cannot - %% safely pass the alloc instruction. - not_possible; -opt_move_1(R, [{set,_,_,_}=I|Is], Acc) -> - %% If the source register is either killed or used by this - %% instruction, the optimimization is not possible. - case is_killed_or_used(R, I) of - true -> not_possible; - false -> opt_move_1(R, Is, [I|Acc]) - end; -opt_move_1(_, _, _) -> - not_possible. - -%% opt_tuple_element([Instruction]) -> [Instruction] -%% If possible, move get_tuple_element instructions forward -%% in the instruction stream to a move instruction, eliminating -%% the move instruction. Example: -%% -%% get_tuple_element Tuple Pos Dst1 -%% ... -%% move Dst1 Dst2 -%% -%% This code may be possible to rewrite to: -%% -%% %%(Moved get_tuple_element instruction) -%% ... -%% get_tuple_element Tuple Pos Dst2 -%% - -opt_tuple_element([{set,[D],[S],{get_tuple_element,_}}=I|Is0]) -> - case opt_tuple_element_1(Is0, I, {S,D}, []) of - no -> - [I|opt_tuple_element(Is0)]; - {yes,Is} -> - opt_tuple_element(Is) - end; -opt_tuple_element([I|Is]) -> - [I|opt_tuple_element(Is)]; -opt_tuple_element([]) -> []. - -opt_tuple_element_1([{set,_,_,{alloc,_,_}}|_], _, _, _) -> - no; -opt_tuple_element_1([{set,_,_,{try_catch,_,_}}|_], _, _, _) -> - no; -opt_tuple_element_1([{set,[D],[S],move}|Is0], I0, {_,S}, Acc) -> - case eliminate_use_of_from_reg(Is0, S, D) of - no -> - no; - {yes,Is1} -> - {set,[S],Ss,Op} = I0, - I = {set,[D],Ss,Op}, - case opt_move_rev(S, Acc, [I|Is1]) of - not_possible -> - %% Not safe because the move of the - %% get_tuple_element instruction would cause the - %% result of a previous instruction to be ignored. - no; - {_,Is} -> - {yes,Is} - end - end; -opt_tuple_element_1([{set,Ds,Ss,_}=I|Is], MovedI, {S,D}=Regs, Acc) -> - case member(S, Ds) orelse member(D, Ss) of - true -> - no; - false -> - opt_tuple_element_1(Is, MovedI, Regs, [I|Acc]) - end; -opt_tuple_element_1(_, _, _, _) -> no. - -%% Reverse the instructions, while checking that there are no -%% instructions that would interfere with using the new destination -%% register (D). - -opt_move_rev(D, [I|Is], Acc) -> - case is_killed_or_used(D, I) of - true -> not_possible; - false -> opt_move_rev(D, Is, [I|Acc]) - end; -opt_move_rev(D, [], Acc) -> {D,Acc}. - -%% is_killed_or_used(Register, {set,_,_,_}) -> bool() -%% Test whether the register is used by the instruction. - -is_killed_or_used(R, {set,Ss,Ds,_}) -> - member(R, Ds) orelse member(R, Ss). - -%% eliminate_use_of_from_reg([Instruction], FromRegister, ToRegister, Acc) -> -%% {yes,Is} | no -%% Eliminate any use of FromRegister in the instruction sequence -%% by replacing uses of FromRegister with ToRegister. If FromRegister -%% is referenced by an allocation instruction, return 'no' to indicate -%% that FromRegister is still used and that the optimization is not -%% possible. - -eliminate_use_of_from_reg(Is, From, To) -> - try - eliminate_use_of_from_reg(Is, From, To, []) - catch - throw:not_possible -> - no - end. - -eliminate_use_of_from_reg([{set,_,_,{alloc,Live,_}}|_]=Is0, {x,X}, _, Acc) -> - if - X < Live -> - no; - true -> - {yes,reverse(Acc, Is0)} - end; -eliminate_use_of_from_reg([{set,Ds,Ss0,Op}=I0|Is], From, To, Acc) -> - ensure_safe_tuple(I0, To), - I = case member(From, Ss0) of - true -> - Ss = [case S of - From -> To; - _ -> S - end || S <- Ss0], - {set,Ds,Ss,Op}; - false -> - I0 - end, - case member(From, Ds) of - true -> - {yes,reverse(Acc, [I|Is])}; - false -> - case member(To, Ds) of - true -> - case beam_utils:is_killed_block(From, Is) of - true -> - {yes,reverse(Acc, [I|Is])}; - false -> - no - end; - false -> - eliminate_use_of_from_reg(Is, From, To, [I|Acc]) - end - end; -eliminate_use_of_from_reg([I]=Is, From, _To, Acc) -> - case beam_utils:is_killed_block(From, [I]) of - true -> - {yes,reverse(Acc, Is)}; - false -> - no - end. - -ensure_safe_tuple({set,[To],[],{put_tuple,_}}, To) -> - throw(not_possible); -ensure_safe_tuple(_, _) -> ok. - -%% opt_allocs(Instructions) -> Instructions. Optimize allocate -%% instructions inside blocks. If safe, replace an allocate_zero -%% instruction with the slightly cheaper allocate instruction. - -opt_allocs(Is) -> - D = beam_utils:index_labels(Is), - opt_allocs_1(Is, D). - -opt_allocs_1([{block,Bl0}|Is], D) -> - Bl = opt_alloc(Bl0, {D,Is}), - [{block,Bl}|opt_allocs_1(Is, D)]; -opt_allocs_1([I|Is], D) -> - [I|opt_allocs_1(Is, D)]; -opt_allocs_1([], _) -> []. - -%% opt_alloc(Instructions) -> Instructions' -%% Optimises all allocate instructions. - -opt_alloc([{set,[],[],{alloc,Live0,Info0}}, - {set,[],[],{alloc,Live,Info}}|Is], D) -> - Live = Live0, %Assertion. - Alloc = combine_alloc(Info0, Info), - I = {set,[],[],{alloc,Live,Alloc}}, - opt_alloc([I|Is], D); -opt_alloc([{set,[],[],{alloc,R,{_,Ns,Nh,[]}}}|Is], D) -> - [{set,[],[],opt_alloc(Is, D, Ns, Nh, R)}|Is]; -opt_alloc([I|Is], D) -> [I|opt_alloc(Is, D)]; -opt_alloc([], _) -> []. - -combine_alloc({_,Ns,Nh1,Init}, {_,nostack,Nh2,[]}) -> - {zero,Ns,beam_utils:combine_heap_needs(Nh1, Nh2),Init}. - -%% opt_alloc(Instructions, FrameSize, HeapNeed, LivingRegs) -> [Instr] -%% Generates the optimal sequence of instructions for -%% allocating and initalizing the stack frame and needed heap. - -opt_alloc(_Is, _D, nostack, Nh, LivingRegs) -> - {alloc,LivingRegs,{nozero,nostack,Nh,[]}}; -opt_alloc(Bl, {D,OuterIs}, Ns, Nh, LivingRegs) -> - Is = [{block,Bl}|OuterIs], - InitRegs = init_yregs(Ns, Is, D), - case count_ones(InitRegs) of - N when N*2 > Ns -> - {alloc,LivingRegs,{nozero,Ns,Nh,gen_init(Ns, InitRegs)}}; - _ -> - {alloc,LivingRegs,{zero,Ns,Nh,[]}} - end. - -gen_init(Fs, Regs) -> gen_init(Fs, Regs, 0, []). - -gen_init(SameFs, _Regs, SameFs, Acc) -> reverse(Acc); -gen_init(Fs, Regs, Y, Acc) when Regs band 1 =:= 0 -> - gen_init(Fs, Regs bsr 1, Y+1, [{init,{y,Y}}|Acc]); -gen_init(Fs, Regs, Y, Acc) -> - gen_init(Fs, Regs bsr 1, Y+1, Acc). - -init_yregs(Y, Is, D) when Y >= 0 -> - case beam_utils:is_killed({y,Y}, Is, D) of - true -> - (1 bsl Y) bor init_yregs(Y-1, Is, D); - false -> - init_yregs(Y-1, Is, D) - end; -init_yregs(_, _, _) -> 0. - -count_ones(Bits) -> count_ones(Bits, 0). -count_ones(0, Acc) -> Acc; -count_ones(Bits, Acc) -> - count_ones(Bits bsr 1, Acc + (Bits band 1)). - -%% Calculate the new number of live registers when we move an allocate -%% instruction upwards, passing a 'set' instruction. - -alloc_live_regs({set,Ds,Ss,_}, Is, Regs0) -> - Rset = x_live(Ss, x_dead(Ds, (1 bsl Regs0)-1)), - Live = live_regs(0, Rset), - case ensure_contiguous(Rset, Live) of - not_possible -> - %% Liveness information (looking forward in the - %% instruction stream) can't prove that moving this - %% allocation instruction is safe. Now use the annotation - %% of defined registers at the beginning of the current - %% block to see whether moving would be safe. - Def0 = defined_regs(Is, 0), - Def = Def0 band ((1 bsl Live) - 1), - ensure_contiguous(Rset bor Def, Live); - Live -> - %% Safe based on liveness information. - Live - end. - -live_regs(N, 0) -> - N; -live_regs(N, Regs) -> - live_regs(N+1, Regs bsr 1). - -ensure_contiguous(Regs, Live) -> - case (1 bsl Live) - 1 of - Regs -> Live; - _ -> not_possible - end. - -x_dead([{x,N}|Rs], Regs) -> x_dead(Rs, Regs band (bnot (1 bsl N))); -x_dead([_|Rs], Regs) -> x_dead(Rs, Regs); -x_dead([], Regs) -> Regs. - -x_live([{x,N}|Rs], Regs) -> x_live(Rs, Regs bor (1 bsl N)); -x_live([_|Rs], Regs) -> x_live(Rs, Regs); -x_live([], Regs) -> Regs. - -%% defined_regs(ReversedInstructions) -> RegBitmap. -%% Given a reversed instruction stream, determine the -%% the registers that are defined. - -defined_regs([{'%anno',{def,Def}}|_], Regs) -> - Def bor Regs; -defined_regs([{set,Ds,_,{alloc,Live,_}}|_], Regs) -> - x_live(Ds, Regs bor ((1 bsl Live) - 1)); -defined_regs([{set,Ds,_,_}|Is], Regs) -> - defined_regs(Is, x_live(Ds, Regs)). - -%%% -%%% Do local common sub expression elimination (CSE) in each block. -%%% - -local_cse([{block,Bl0}|Is]) -> - Bl = cse_block(Bl0, orddict:new(), []), - [{block,Bl}|local_cse(Is)]; -local_cse([I|Is]) -> - [I|local_cse(Is)]; -local_cse([]) -> []. - -cse_block([I|Is], Es0, Acc0) -> - Es1 = cse_clear(I, Es0), - case cse_expr(I) of - none -> - %% Instruction is not suitable for CSE. - cse_block(Is, Es1, [I|Acc0]); - {ok,D,Expr} -> - %% Suitable instruction. First update the dictionary of - %% suitable expressions for the next iteration. - Es = cse_add(D, Expr, Es1), - - %% Search for a previous identical expression. - case cse_find(Expr, Es0) of - error -> - %% Nothing found - cse_block(Is, Es, [I|Acc0]); - Src -> - %% Use the previously calculated result. - %% Also eliminate any line instruction. - Move = {set,[D],[Src],move}, - case Acc0 of - [{set,_,_,{line,_}}|Acc] -> - cse_block(Is, Es, [Move|Acc]); - [_|_] -> - cse_block(Is, Es, [Move|Acc0]) - end - end - end; -cse_block([], _, Acc) -> - reverse(Acc). - -%% cse_find(Expr, Expressions) -> error | Register. -%% Find a previously evaluated expression whose result can be reused, -%% or return 'error' if no such expression is found. - -cse_find(Expr, Es) -> - case orddict:find(Expr, Es) of - {ok,{Src,_}} -> Src; - error -> error - end. - -cse_expr({set,[D],Ss,{bif,N,_}}) -> - case D of - {fr,_} -> - %% There are too many things that can go wrong. - none; - _ -> - {ok,D,{{bif,N},Ss}} - end; -cse_expr({set,[D],Ss,{alloc,_,{gc_bif,N,_}}}) -> - {ok,D,{{gc_bif,N},Ss}}; -cse_expr({set,[D],Ss,put_list}) -> - {ok,D,{put_list,Ss}}; -cse_expr(_) -> none. - -%% cse_clear(Instr, Expressions0) -> Expressions. -%% Remove all previous expressions that will become -%% invalid when this instruction is executed. Basically, -%% an expression is no longer safe to reuse when the -%% register it has been stored to has been modified, killed, -%% or if any of the source operands have changed. - -cse_clear({set,Ds,_,{alloc,Live,_}}, Es) -> - cse_clear_1(Es, Live, Ds); -cse_clear({set,Ds,_,_}, Es) -> - cse_clear_1(Es, all, Ds). - -cse_clear_1(Es, Live, Ds0) -> - Ds = ordsets:from_list(Ds0), - [E || E <- Es, cse_is_safe(E, Live, Ds)]. - -cse_is_safe({_,{Dst,Interfering}}, Live, Ds) -> - ordsets:is_disjoint(Interfering, Ds) andalso - case Dst of - {x,X} -> - X < Live; - _ -> - true - end. - -%% cse_add(Dest, Expr, Expressions0) -> Expressions. -%% Provided that it is safe, add a new expression to the dictionary -%% of already evaluated expressions. - -cse_add(D, {_,Ss}=Expr, Es) -> - case member(D, Ss) of - false -> - Interfering = ordsets:from_list([D|Ss]), - orddict:store(Expr, {D,Interfering}, Es); - true -> - %% Unsafe because the instruction overwrites one of - %% source operands. - Es +%% sort_moves([Instruction]) -> [Instruction]. +%% Sort move instructions on the Y register to give the loader +%% more opportunities for combining instructions. + +sort_moves([{set,[{x,_}],[{y,_}],move}=I|Is0]) -> + {Moves,Is} = sort_moves_1(Is0, x, y, [I]), + Moves ++ sort_moves(Is); +sort_moves([{set,[{y,_}],[{x,_}],move}=I|Is0]) -> + {Moves,Is} = sort_moves_1(Is0, y, x, [I]), + Moves ++ sort_moves(Is); +sort_moves([I|Is]) -> + [I|sort_moves(Is)]; +sort_moves([]) -> []. + +sort_moves_1([{set,[{x,0}],[_],move}=I|Is], _DTag, _STag, Acc) -> + %% The loader sometimes combines a move to x0 with the + %% instruction that follows, producing, for example, a move_call + %% instruction. Therefore, we don't want include this move + %% instruction in the sorting. + {sort_on_yreg(Acc)++[I],Is}; +sort_moves_1([{set,[{DTag,_}],[{STag,_}],move}=I|Is], DTag, STag, Acc) -> + sort_moves_1(Is, DTag, STag, [I|Acc]); +sort_moves_1(Is, _DTag, _STag, Acc) -> + {sort_on_yreg(Acc),Is}. + +sort_on_yreg([{set,[Dst],[Src],move}|_]=Moves) -> + case {Dst,Src} of + {{y,_},{x,_}} -> + keysort(2, Moves); + {{x,_},{y,_}} -> + keysort(3, Moves) end. diff --git a/lib/compiler/src/beam_bs.erl b/lib/compiler/src/beam_bs.erl deleted file mode 100644 index 5f1b9ed488..0000000000 --- a/lib/compiler/src/beam_bs.erl +++ /dev/null @@ -1,280 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1999-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% -%% Purpose : Partitions assembly instructions into basic blocks and -%% optimizes them. - --module(beam_bs). - --export([module/2]). --import(lists, [mapfoldl/3,reverse/1]). - --spec module(beam_utils:module_code(), [compile:option()]) -> - {'ok',beam_utils:module_code()}. - -module({Mod,Exp,Attr,Fs0,Lc0}, _Opt) -> - {Fs,Lc} = mapfoldl(fun function/2, Lc0, Fs0), - {ok,{Mod,Exp,Attr,Fs,Lc}}. - -function({function,Name,Arity,CLabel,Is0}, Lc0) -> - try - Is1 = bs_put_opt(Is0), - {Is,Lc} = bsm_opt(Is1, Lc0), - {{function,Name,Arity,CLabel,Is},Lc} - catch - Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [Name,Arity]), - erlang:raise(Class, Error, Stack) - end. - -%%% -%%% Evaluation of constant bit fields. -%%% - -bs_put_opt([{bs_put,_,_,_}=I|Is0]) -> - {BsPuts0,Is} = collect_bs_puts(Is0, [I]), - BsPuts = opt_bs_puts(BsPuts0), - BsPuts ++ bs_put_opt(Is); -bs_put_opt([I|Is]) -> - [I|bs_put_opt(Is)]; -bs_put_opt([]) -> []. - -collect_bs_puts([{bs_put,_,_,_}=I|Is], Acc) -> - collect_bs_puts(Is, [I|Acc]); -collect_bs_puts([_|_]=Is, Acc) -> - {reverse(Acc),Is}. - -opt_bs_puts(Is) -> - opt_bs_1(Is, []). - -opt_bs_1([{bs_put,Fail, - {bs_put_float,1,Flags0},[{integer,Sz},Src]}=I0|Is], Acc) -> - try eval_put_float(Src, Sz, Flags0) of - <<Int:Sz>> -> - Flags = force_big(Flags0), - I = {bs_put,Fail,{bs_put_integer,1,Flags}, - [{integer,Sz},{integer,Int}]}, - opt_bs_1([I|Is], Acc) - catch - error:_ -> - opt_bs_1(Is, [I0|Acc]) - end; -opt_bs_1([{bs_put,_,{bs_put_integer,1,_},[{integer,8},{integer,_}]}|_]=IsAll, - Acc0) -> - {Is,Acc} = bs_collect_string(IsAll, Acc0), - opt_bs_1(Is, Acc); -opt_bs_1([{bs_put,Fail,{bs_put_integer,1,F},[{integer,Sz},{integer,N}]}=I|Is0], - Acc) when Sz > 8 -> - case field_endian(F) of - big -> - %% We can do this optimization for any field size without - %% risk for code explosion. - case bs_split_int(N, Sz, Fail, Is0) of - no_split -> opt_bs_1(Is0, [I|Acc]); - Is -> opt_bs_1(Is, Acc) - end; - little when Sz < 128 -> - %% We only try to optimize relatively small fields, to - %% avoid an explosion in code size. - <<Int:Sz>> = <<N:Sz/little>>, - Flags = force_big(F), - Is = [{bs_put,Fail,{bs_put_integer,1,Flags}, - [{integer,Sz},{integer,Int}]}|Is0], - opt_bs_1(Is, Acc); - _ -> %native or too wide little field - opt_bs_1(Is0, [I|Acc]) - end; -opt_bs_1([{bs_put,Fail,{Op,U,F},[{integer,Sz},Src]}|Is], Acc) when U > 1 -> - opt_bs_1([{bs_put,Fail,{Op,1,F},[{integer,U*Sz},Src]}|Is], Acc); -opt_bs_1([I|Is], Acc) -> - opt_bs_1(Is, [I|Acc]); -opt_bs_1([], Acc) -> reverse(Acc). - -eval_put_float(Src, Sz, Flags) when Sz =< 256 -> - %%Only evaluate if Sz is reasonable. - Val = value(Src), - case field_endian(Flags) of - little -> <<Val:Sz/little-float-unit:1>>; - big -> <<Val:Sz/big-float-unit:1>> - %% native intentionally not handled here - we can't optimize - %% it. - end. - -value({integer,I}) -> I; -value({float,F}) -> F. - -bs_collect_string(Is, [{bs_put,_,{bs_put_string,Len,{string,Str}},[]}|Acc]) -> - bs_coll_str_1(Is, Len, reverse(Str), Acc); -bs_collect_string(Is, Acc) -> - bs_coll_str_1(Is, 0, [], Acc). - -bs_coll_str_1([{bs_put,_,{bs_put_integer,U,_},[{integer,Sz},{integer,V}]}|Is], - Len, StrAcc, IsAcc) when U*Sz =:= 8 -> - Byte = V band 16#FF, - bs_coll_str_1(Is, Len+1, [Byte|StrAcc], IsAcc); -bs_coll_str_1(Is, Len, StrAcc, IsAcc) -> - {Is,[{bs_put,{f,0},{bs_put_string,Len,{string,reverse(StrAcc)}},[]}|IsAcc]}. - -field_endian({field_flags,F}) -> field_endian_1(F). - -field_endian_1([big=E|_]) -> E; -field_endian_1([little=E|_]) -> E; -field_endian_1([native=E|_]) -> E; -field_endian_1([_|Fs]) -> field_endian_1(Fs). - -force_big({field_flags,F}) -> - {field_flags,force_big_1(F)}. - -force_big_1([big|_]=Fs) -> Fs; -force_big_1([little|Fs]) -> [big|Fs]; -force_big_1([F|Fs]) -> [F|force_big_1(Fs)]. - -bs_split_int(0, Sz, _, _) when Sz > 64 -> - %% We don't want to split in this case because the - %% string will consist of only zeroes. - no_split; -bs_split_int(-1, Sz, _, _) when Sz > 64 -> - %% We don't want to split in this case because the - %% string will consist of only 255 bytes. - no_split; -bs_split_int(N, Sz, Fail, Acc) -> - FirstByteSz = case Sz rem 8 of - 0 -> 8; - Rem -> Rem - end, - bs_split_int_1(N, FirstByteSz, Sz, Fail, Acc). - -bs_split_int_1(-1, _, Sz, Fail, Acc) when Sz > 64 -> - I = {bs_put,Fail,{bs_put_integer,1,{field_flags,[big]}}, - [{integer,Sz},{integer,-1}]}, - [I|Acc]; -bs_split_int_1(0, _, Sz, Fail, Acc) when Sz > 64 -> - I = {bs_put,Fail,{bs_put_integer,1,{field_flags,[big]}}, - [{integer,Sz},{integer,0}]}, - [I|Acc]; -bs_split_int_1(N, ByteSz, Sz, Fail, Acc) when Sz > 0 -> - Mask = (1 bsl ByteSz) - 1, - I = {bs_put,Fail,{bs_put_integer,1,{field_flags,[big]}}, - [{integer,ByteSz},{integer,N band Mask}]}, - bs_split_int_1(N bsr ByteSz, 8, Sz-ByteSz, Fail, [I|Acc]); -bs_split_int_1(_, _, _, _, Acc) -> Acc. - -%%% -%%% Optimization of bit syntax matching: get rid -%%% of redundant bs_restore2/2 instructions across select_val -%%% instructions, as well as a few other simple peep-hole -%%% optimizations. -%%% - -bsm_opt(Is0, Lc0) -> - {Is1,D0,Lc} = bsm_scan(Is0, [], Lc0, []), - Is2 = case D0 of - [] -> - %% No bit syntax matching in this function. - Is1; - [_|_] -> - %% Optimize the bit syntax matching. - D = gb_trees:from_orddict(orddict:from_list(D0)), - bsm_reroute(Is1, D, none, []) - end, - Is = beam_clean:bs_clean_saves(Is2), - {bsm_opt_2(Is, []),Lc}. - -bsm_scan([{label,L}=Lbl,{bs_restore2,_,Save}=R|Is], D0, Lc, Acc0) -> - D = [{{L,Save},Lc}|D0], - Acc = [{label,Lc},R,Lbl|Acc0], - bsm_scan(Is, D, Lc+1, Acc); -bsm_scan([I|Is], D, Lc, Acc) -> - bsm_scan(Is, D, Lc, [I|Acc]); -bsm_scan([], D, Lc, Acc) -> - {reverse(Acc),D,Lc}. - -bsm_reroute([{bs_save2,Reg,Save}=I|Is], D, _, Acc) -> - bsm_reroute(Is, D, {Reg,Save}, [I|Acc]); -bsm_reroute([{bs_restore2,Reg,Save}=I|Is], D, _, Acc) -> - bsm_reroute(Is, D, {Reg,Save}, [I|Acc]); -bsm_reroute([{label,_}=I|Is], D, S, Acc) -> - bsm_reroute(Is, D, S, [I|Acc]); -bsm_reroute([{select,select_val,Reg,F0,Lbls0}|Is], D, {_,Save}=S, Acc0) -> - [F|Lbls] = bsm_subst_labels([F0|Lbls0], Save, D), - Acc = [{select,select_val,Reg,F,Lbls}|Acc0], - bsm_reroute(Is, D, S, Acc); -bsm_reroute([{test,TestOp,F0,TestArgs}=I|Is], D, {_,Save}=S, Acc0) -> - F = bsm_subst_label(F0, Save, D), - Acc = [{test,TestOp,F,TestArgs}|Acc0], - case bsm_not_bs_test(I) of - true -> - %% The test instruction will not update the bit offset for - %% the binary being matched. Therefore the save position - %% can be kept. - bsm_reroute(Is, D, S, Acc); - false -> - %% The test instruction might update the bit offset. Kill - %% our remembered Save position. - bsm_reroute(Is, D, none, Acc) - end; -bsm_reroute([{test,TestOp,F0,Live,TestArgs,Dst}|Is], D, {_,Save}, Acc0) -> - F = bsm_subst_label(F0, Save, D), - Acc = [{test,TestOp,F,Live,TestArgs,Dst}|Acc0], - %% The test instruction will update the bit offset. Kill our - %% remembered Save position. - bsm_reroute(Is, D, none, Acc); -bsm_reroute([{block,[{set,[],[],{alloc,_,_}}]}=Bl, - {bs_context_to_binary,_}=I|Is], D, S, Acc) -> - %% To help further bit syntax optimizations. - bsm_reroute([I,Bl|Is], D, S, Acc); -bsm_reroute([I|Is], D, _, Acc) -> - bsm_reroute(Is, D, none, [I|Acc]); -bsm_reroute([], _, _, Acc) -> reverse(Acc). - -bsm_opt_2([{test,bs_test_tail2,F,[Ctx,Bits]}|Is], - [{test,bs_skip_bits2,F,[Ctx,{integer,I},Unit,_Flags]}|Acc]) -> - bsm_opt_2(Is, [{test,bs_test_tail2,F,[Ctx,Bits+I*Unit]}|Acc]); -bsm_opt_2([{test,bs_skip_bits2,F,[Ctx,{integer,I1},Unit1,_]}|Is], - [{test,bs_skip_bits2,F,[Ctx,{integer,I2},Unit2,Flags]}|Acc]) -> - bsm_opt_2(Is, [{test,bs_skip_bits2,F, - [Ctx,{integer,I1*Unit1+I2*Unit2},1,Flags]}|Acc]); -bsm_opt_2([I|Is], Acc) -> - bsm_opt_2(Is, [I|Acc]); -bsm_opt_2([], Acc) -> reverse(Acc). - -%% bsm_not_bs_test({test,Name,_,Operands}) -> true|false. -%% Test whether is the test is a "safe", i.e. does not move the -%% bit offset for a binary. -%% -%% 'true' means that the test is safe, 'false' that we don't know or -%% that the test moves the offset (e.g. bs_get_integer2). - -bsm_not_bs_test({test,bs_test_tail2,_,[_,_]}) -> true; -bsm_not_bs_test(Test) -> beam_utils:is_pure_test(Test). - -bsm_subst_labels(Fs, Save, D) -> - bsm_subst_labels_1(Fs, Save, D, []). - -bsm_subst_labels_1([F|Fs], Save, D, Acc) -> - bsm_subst_labels_1(Fs, Save, D, [bsm_subst_label(F, Save, D)|Acc]); -bsm_subst_labels_1([], _, _, Acc) -> - reverse(Acc). - -bsm_subst_label({f,Lbl0}=F, Save, D) -> - case gb_trees:lookup({Lbl0,Save}, D) of - {value,Lbl} -> {f,Lbl}; - none -> F - end; -bsm_subst_label(Other, _, _) -> Other. diff --git a/lib/compiler/src/beam_bsm.erl b/lib/compiler/src/beam_bsm.erl deleted file mode 100644 index 1c8e0e9854..0000000000 --- a/lib/compiler/src/beam_bsm.erl +++ /dev/null @@ -1,708 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2007-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(beam_bsm). --export([module/2,format_error/1]). - --import(lists, [member/2,foldl/3,reverse/1,sort/1,all/2]). - -%%% -%%% We optimize bit syntax matching where the tail end of a binary is -%%% matched out and immediately passed on to a bs_start_match2 instruction, -%%% such as in this code sequence: -%%% -%%% func_info ... -%%% L1 test bs_start_match2 {f,...} {x,0} Live SavePositions {x,0} -%%% . . . -%%% test bs_get_binary2 {f,...} {x,0} all 1 Flags {x,0} -%%% . . . -%%% call_only 2 L1 -%%% -%%% The sequence can be optimized simply by removing the bs_get_binary2 -%%% instruction. Another example: -%%% -%%% func_info ... -%%% L1 test bs_start_match2 {f,...} {x,0} Live SavePositions {x,0} -%%% . . . -%%% test bs_get_binary2 {f,...} {x,0} all 8 Flags {x,1} -%%% . . . -%%% move {x,1} {x,0} -%%% call_only 2 L1 -%%% -%%% In this case, the bs_get_binary2 instruction must be replaced by -%%% -%%% test bs_unit {x,1} 8 -%%% -%%% to ensure that the match fail if the length of the binary in bits -%%% is not evenly divisible by 8. -%%% -%%% Note that the bs_start_match2 instruction doesn't need to be in the same -%%% function as the caller. It can be in the beginning of any function, or -%%% follow the bs_get_binary2 instruction in the same function. The important -%%% thing is that the match context register is not copied or built into -%%% data structures or passed to BIFs. -%%% - --type label() :: beam_asm:label(). --type func_info() :: {beam_asm:reg(),boolean()}. - --record(btb, - {f :: gb_trees:tree(label(), func_info()), - index :: beam_utils:code_index(), %{Label,Code} index (for liveness). - ok_br=gb_sets:empty() :: gb_sets:set(label()), %Labels that are OK. - must_not_save=false :: boolean(), %Must not save position when - % optimizing (reaches - % bs_context_to_binary). - must_save=false :: boolean() %Must save position when optimizing. - }). - - --spec module(beam_utils:module_code(), [compile:option()]) -> - {'ok',beam_utils:module_code()}. - -module({Mod,Exp,Attr,Fs0,Lc}, Opts) -> - FIndex = btb_index(Fs0), - Fs = [function(F, FIndex) || F <- Fs0], - Code = {Mod,Exp,Attr,Fs,Lc}, - case proplists:get_bool(bin_opt_info, Opts) of - true -> - {ok,Code,collect_warnings(Fs)}; - false -> - {ok,Code} - end. - --spec format_error('bin_opt' | {'no_bin_opt', term()}) -> nonempty_string(). - -format_error(bin_opt) -> - "OPTIMIZED: creation of sub binary delayed"; -format_error({no_bin_opt,Reason}) -> - lists:flatten(["NOT OPTIMIZED: "|format_error_1(Reason)]). - -%%% -%%% Local functions. -%%% - -function({function,Name,Arity,Entry,Is}, FIndex) -> - try - Index = beam_utils:index_labels(Is), - D = #btb{f=FIndex,index=Index}, - {function,Name,Arity,Entry,btb_opt_1(Is, D, [])} - catch - Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [Name,Arity]), - erlang:raise(Class, Error, Stack) - end. - -btb_opt_1([{test,bs_get_binary2,F,_,[Reg,{atom,all},U,Fs],Reg}=I0|Is], D, Acc0) -> - case btb_reaches_match(Is, [Reg], D) of - {error,Reason} -> - Comment = btb_comment_no_opt(Reason, Fs), - btb_opt_1(Is, D, [Comment,I0|Acc0]); - {ok,MustSave} -> - Comment = btb_comment_opt(Fs), - Acc1 = btb_gen_save(MustSave, Reg, [Comment|Acc0]), - Acc = case U of - 1 -> Acc1; - _ -> [{test,bs_test_unit,F,[Reg,U]}|Acc1] - end, - btb_opt_1(Is, D, Acc) - end; -btb_opt_1([{test,bs_get_binary2,F,_,[Ctx,{atom,all},U,Fs],Dst}=I0|Is0], D, Acc0) -> - case btb_reaches_match(Is0, [Ctx,Dst], D) of - {error,Reason} -> - Comment = btb_comment_no_opt(Reason, Fs), - btb_opt_1(Is0, D, [Comment,I0|Acc0]); - {ok,MustSave} when U =:= 1 -> - Comment = btb_comment_opt(Fs), - Acc = btb_gen_save(MustSave, Ctx, [Comment|Acc0]), - Is = prepend_move(Ctx, Dst, Is0), - btb_opt_1(Is, D, Acc); - {ok,MustSave} -> - Comment = btb_comment_opt(Fs), - Acc1 = btb_gen_save(MustSave, Ctx, [Comment|Acc0]), - Acc = [{test,bs_test_unit,F,[Ctx,U]}|Acc1], - Is = prepend_move(Ctx, Dst, Is0), - btb_opt_1(Is, D, Acc) - end; -btb_opt_1([I|Is], D, Acc) -> - %%io:format("~p\n", [I]), - btb_opt_1(Is, D, [I|Acc]); -btb_opt_1([], _, Acc) -> - reverse(Acc). - -btb_gen_save(true, Reg, Acc) -> - [{bs_save2,Reg,{atom,start}}|Acc]; -btb_gen_save(false, _, Acc) -> Acc. - -prepend_move(Ctx, Dst, [{block,Bl0}|Is]) -> - Bl = [{set,[Dst],[Ctx],move}|Bl0], - [{block,Bl}|Is]; -prepend_move(Ctx, Dst, Is) -> - [{move,Ctx,Dst}|Is]. - -%% btb_reaches_match([Instruction], [Register], D) -> -%% {ok,MustSave}|{error,Reason} -%% -%% The list of Registers should be a list of registers referencing a -%% match context. The Register may contain one element if the -%% bs_get_binary2 instruction looks like -%% -%% test bs_get_binary2 {f,...} Ctx all _ _ Ctx -%% -%% or two elements if the instruction looks like -%% -%% test bs_get_binary2 {f,...} Ctx all _ _ Dst -%% -%% This function determines whether the bs_get_binary2 instruction -%% can be omitted (retaining the match context instead of creating -%% a sub binary). -%% -%% The rule is that the match context ultimately must end up at a -%% bs_start_match2 instruction and nowhere else. That it, it must not -%% be passed to BIFs, or copied or put into data structures. There -%% must only be one copy alive when the match context reaches the -%% bs_start_match2 instruction. -%% -%% At a branch, we must follow all branches and make sure that the above -%% rule is followed (or that the branch kills the match context). -%% -%% The MustSave return value will be true if control may end up at -%% bs_context_to_binary instruction. Since that instruction uses the -%% saved start position, we must use "bs_save2 Ctx start" to -%% update the saved start position. An additional complication is that -%% "bs_save2 Ctx start" must not be used if Dst and Ctx are -%% different registers and both registers may be passed to -%% a bs_context_to_binary instruction. -%% - -btb_reaches_match(Is, RegList, D) -> - try - Regs = btb_regs_from_list(RegList), - #btb{must_not_save=MustNotSave,must_save=MustSave} = - btb_reaches_match_1(Is, Regs, D), - case MustNotSave andalso MustSave of - true -> btb_error(must_and_must_not_save); - false -> {ok,MustSave} - end - catch - throw:{error,_}=Error -> Error - end. - -btb_reaches_match_1(Is, Regs, D) -> - case btb_are_registers_empty(Regs) of - false -> - btb_reaches_match_2(Is, Regs, D); - true -> - %% The context was killed, which is OK. - D - end. - -btb_reaches_match_2([{block,Bl}|Is], Regs0, D) -> - Regs = btb_reaches_match_block(Bl, Regs0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{call,Arity,{f,Lbl}}|Is], Regs0, D) -> - case is_tail_call(Is) of - true -> - Regs1 = btb_kill_not_live(Arity, Regs0), - Regs = btb_kill_yregs(Regs1), - btb_tail_call(Lbl, Regs, D); - false -> - btb_call(Arity, Lbl, Regs0, Is, D) - end; -btb_reaches_match_2([{apply,Arity}|Is], Regs, D) -> - btb_call(Arity+2, apply, Regs, Is, D); -btb_reaches_match_2([{call_fun,Live}=I|Is], Regs, D) -> - btb_ensure_not_used([{x,Live}], I, Regs), - btb_call(Live, I, Regs, Is, D); -btb_reaches_match_2([{make_fun2,_,_,_,Live}|Is], Regs, D) -> - btb_call(Live, make_fun2, Regs, Is, D); -btb_reaches_match_2([{call_ext,Arity,Func}=I|Is], Regs0, D) -> - %% Allow us scanning beyond the call in case the match - %% context is saved on the stack. - case beam_jump:is_exit_instruction(I) of - false -> - btb_call(Arity, Func, Regs0, Is, D); - true -> - Regs = btb_kill_not_live(Arity, Regs0), - btb_tail_call(Func, Regs, D) - end; -btb_reaches_match_2([{kill,Y}|Is], Regs, D) -> - btb_reaches_match_1(Is, btb_kill([Y], Regs), D); -btb_reaches_match_2([{deallocate,_}|Is], Regs0, D) -> - Regs = btb_kill_yregs(Regs0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([return=I|_], Regs0, D) -> - btb_ensure_not_used([{x,0}], I, Regs0), - D; -btb_reaches_match_2([{gc_bif,_,{f,F},Live,Ss,Dst}=I|Is], Regs0, D0) -> - btb_ensure_not_used(Ss, I, Regs0), - Regs1 = btb_kill_not_live(Live, Regs0), - Regs = btb_kill([Dst], Regs1), - D = btb_follow_branch(F, Regs, D0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{bif,_,{f,F},Ss,Dst}=I|Is], Regs0, D0) -> - btb_ensure_not_used(Ss, I, Regs0), - Regs = btb_kill([Dst], Regs0), - D = btb_follow_branch(F, Regs, D0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{get_map_elements,{f,F},Src,{list,Ls}}=I|Is], Regs0, D0) -> - {Ss,Ds} = beam_utils:split_even(Ls), - btb_ensure_not_used([Src|Ss], I, Regs0), - Regs = btb_kill(Ds, Regs0), - D = btb_follow_branch(F, Regs, D0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{test,bs_start_match2,{f,F},Live,[Ctx,_],Ctx}=I|Is], - Regs0, D0) -> - CtxRegs = btb_context_regs(Regs0), - case member(Ctx, CtxRegs) of - false -> - %% This bs_start_match2 instruction does not use "our" - %% match state. Therefore we can continue the search - %% for another bs_start_match2 instruction. - D = btb_follow_branch(F, Regs0, D0), - Regs = btb_kill_not_live(Live, Regs0), - btb_reaches_match_2(Is, Regs, D); - true -> - %% OK. This instruction will use "our" match state, - %% but we must make sure that all other copies of the - %% match state are killed in the code that follows - %% the instruction. (We know that the fail branch cannot - %% be taken in this case.) - OtherCtxRegs = CtxRegs -- [Ctx], - case btb_are_all_unused(OtherCtxRegs, Is, D0) of - false -> btb_error({OtherCtxRegs,not_all_unused_after,I}); - true -> D0 - end - end; -btb_reaches_match_2([{test,bs_start_match2,{f,F},Live,[Bin,_],Ctx}|Is], - Regs0, D0) -> - CtxRegs = btb_context_regs(Regs0), - case member(Bin, CtxRegs) orelse member(Ctx, CtxRegs) of - false -> - %% This bs_start_match2 does not reference any copy of the - %% match state. Therefore it can safely be passed on the - %% way to another (perhaps more suitable) bs_start_match2 - %% instruction. - D = btb_follow_branch(F, Regs0, D0), - Regs = btb_kill_not_live(Live, Regs0), - btb_reaches_match_2(Is, Regs, D); - true -> - %% This variant of the bs_start_match2 instruction does - %% not accept a match state as source. - btb_error(unsuitable_bs_start_match) - end; -btb_reaches_match_2([{test,_,{f,F},Ss}=I|Is], Regs, D0) -> - btb_ensure_not_used(Ss, I, Regs), - D = btb_follow_branch(F, Regs, D0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{test,_,{f,F},_,Ss,_}=I|Is], Regs, D0) -> - btb_ensure_not_used(Ss, I, Regs), - D = btb_follow_branch(F, Regs, D0), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{select,_,Src,{f,F},Conds}=I|Is], Regs, D0) -> - btb_ensure_not_used([Src], I, Regs), - D1 = btb_follow_branch(F, Regs, D0), - D = btb_follow_branches(Conds, Regs, D1), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{jump,{f,Lbl}}|_], Regs, #btb{index=Li}=D) -> - Is = fetch_code_at(Lbl, Li), - btb_reaches_match_2(Is, Regs, D); -btb_reaches_match_2([{label,_}|Is], Regs, D) -> - btb_reaches_match_2(Is, Regs, D); -btb_reaches_match_2([{bs_init,{f,0},_,_,Ss,Dst}=I|Is], Regs, D) -> - btb_ensure_not_used(Ss, I, Regs), - btb_reaches_match_1(Is, btb_kill([Dst], Regs), D); -btb_reaches_match_2([{bs_put,{f,0},_,Ss}=I|Is], Regs, D) -> - btb_ensure_not_used(Ss, I, Regs), - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([{bs_restore2,Src,_}=I|Is], Regs0, D) -> - case btb_contains_context(Src, Regs0) of - false -> - btb_reaches_match_1(Is, Regs0, D); - true -> - %% Check that all other copies of the context registers - %% are unused by the following instructions. - Regs = btb_kill([Src], Regs0), - CtxRegs = btb_context_regs(Regs), - case btb_are_all_unused(CtxRegs, Is, D) of - false -> btb_error({CtxRegs,not_all_unused_after,I}); - true -> D#btb{must_not_save=true} - end - end; -btb_reaches_match_2([{bs_context_to_binary,Src}=I|Is], Regs0, D) -> - case btb_contains_context(Src, Regs0) of - false -> - btb_reaches_match_1(Is, Regs0, D); - true -> - %% Check that all other copies of the context registers - %% are unused by the following instructions. - Regs = btb_kill([Src], Regs0), - CtxRegs = btb_context_regs(Regs), - case btb_are_all_unused(CtxRegs, Is, D) of - false -> btb_error({CtxRegs,not_all_unused_after,I}); - true -> D#btb{must_not_save=true} - end - end; -btb_reaches_match_2([{badmatch,Src}=I|_], Regs, D) -> - btb_ensure_not_used([Src], I, Regs), - D; -btb_reaches_match_2([{case_end,Src}=I|_], Regs, D) -> - btb_ensure_not_used([Src], I, Regs), - D; -btb_reaches_match_2([if_end|_], _Regs, D) -> - D; -btb_reaches_match_2([{func_info,_,_,Arity}=I|_], Regs0, D) -> - Regs = btb_kill_yregs(btb_kill_not_live(Arity, Regs0)), - case btb_context_regs(Regs) of - [] -> D; - _ -> {binary_used_in,I} - end; -btb_reaches_match_2([{line,_}|Is], Regs, D) -> - btb_reaches_match_1(Is, Regs, D); -btb_reaches_match_2([I|_], Regs, _) -> - btb_error({btb_context_regs(Regs),I,not_handled}). - -is_tail_call([{deallocate,_}|_]) -> true; -is_tail_call([return|_]) -> true; -is_tail_call(_) -> false. - -btb_call(Arity, Lbl, Regs0, Is, D0) -> - Regs = btb_kill_not_live(Arity, Regs0), - case btb_are_x_registers_empty(Regs) of - false -> - %% There is a match context in one of the x registers. - %% First handle the call as if it were a tail call. - D = btb_tail_call(Lbl, Regs, D0), - - %% No problem so far (the called function can handle a - %% match context). Now we must make sure that we don't - %% have any copies of the match context tucked away in an - %% y register. - RegList = btb_context_regs(Regs), - case [R || {y,_}=R <- RegList] of - [] -> - D; - [_|_] -> - btb_error({multiple_uses,RegList}) - end; - true -> - %% No match context in any x register. It could have been - %% saved to an y register, so continue to scan the code following - %% the call. - btb_reaches_match_1(Is, Regs, D0) - end. - -btb_tail_call(Lbl, Regs, #btb{f=Ftree,must_save=MustSave0}=D) -> - %% Ignore any y registers here. - case [R || {x,_}=R <- btb_context_regs(Regs)] of - [] -> - D; - [{x,_}=Reg] -> - case gb_trees:lookup(Lbl, Ftree) of - {value,{Reg,MustSave}} -> - D#btb{must_save=MustSave0 or MustSave}; - _ when is_integer(Lbl) -> - btb_error({{label,Lbl},no_suitable_bs_start_match}); - _ -> - btb_error({binary_used_in,Lbl}) - end; - [_|_] when not is_integer(Lbl) -> - btb_error({binary_used_in,Lbl}); - [_|_]=RegList -> - btb_error({multiple_uses,RegList}) - end. - -%% btb_follow_branches([Cond], Regs, D) -> D' -%% Recursively follow all the branches. - -btb_follow_branches([{f,Lbl}|T], Regs, D0) -> - D = btb_follow_branch(Lbl, Regs, D0), - btb_follow_branches(T, Regs, D); -btb_follow_branches([_|T], Regs, D) -> - btb_follow_branches(T, Regs, D); -btb_follow_branches([], _, D) -> D. - -%% btb_follow_branch(Lbl, Regs, D) -> D' -%% Recursively follow the branch. - -btb_follow_branch(0, _Regs, D) -> D; -btb_follow_branch(Lbl, Regs, #btb{ok_br=Br0,index=Li}=D) -> - Key = {Lbl,Regs}, - case gb_sets:is_member(Key, Br0) of - true -> - %% We have already followed this branch and it was OK. - D; - false -> - %% New branch. Try it. - Is = fetch_code_at(Lbl, Li), - #btb{ok_br=Br,must_not_save=MustNotSave,must_save=MustSave} = - btb_reaches_match_1(Is, Regs, D), - - %% Since we got back, this branch is OK. - D#btb{ok_br=gb_sets:insert(Key, Br),must_not_save=MustNotSave, - must_save=MustSave} - end. - -btb_reaches_match_block([{set,Ds,Ss,{alloc,Live,_}}=I|Is], Regs0) -> - %% An allocation instruction or a GC bif. We'll kill all registers - %% if any copy of the context is used as the source to the BIF. - btb_ensure_not_used(Ss, I, Regs0), - Regs1 = btb_kill_not_live(Live, Regs0), - Regs = btb_kill(Ds, Regs1), - btb_reaches_match_block(Is, Regs); -btb_reaches_match_block([{set,[Dst]=Ds,[Src],move}|Is], Regs0) -> - Regs1 = btb_kill(Ds, Regs0), - Regs = case btb_contains_context(Src, Regs1) of - false -> Regs1; - true -> btb_set_context(Dst, Regs1) - end, - btb_reaches_match_block(Is, Regs); -btb_reaches_match_block([{set,Ds,Ss,_}=I|Is], Regs0) -> - btb_ensure_not_used(Ss, I, Regs0), - Regs = btb_kill(Ds, Regs0), - btb_reaches_match_block(Is, Regs); -btb_reaches_match_block([], Regs) -> - Regs. - -%% btb_are_all_killed([Register], [Instruction], D) -> true|false -%% Test whether all of the register are unused in the instruction stream. - -btb_are_all_unused(RegList, Is, #btb{index=Li}) -> - all(fun(R) -> - beam_utils:is_not_used(R, Is, Li) - end, RegList). - -%% btp_regs_from_list([Register]) -> RegisterSet. -%% Create a register set from a list of registers. - -btb_regs_from_list(L) -> - foldl(fun(R, Regs) -> - btb_set_context(R, Regs) - end, {0,0}, L). - -%% btb_set_context(Register, RegisterSet) -> RegisterSet' -%% Update RegisterSet to indicate that Register contains the matching context. - -btb_set_context({x,N}, {Xregs,Yregs}) -> - {Xregs bor (1 bsl N),Yregs}; -btb_set_context({y,N}, {Xregs,Yregs}) -> - {Xregs,Yregs bor (1 bsl N)}. - -%% btb_ensure_not_used([Register], Instruction, RegisterSet) -> ok -%% If any register in RegisterSet (the register(s) known to contain -%% the match context) is used in the list of registers, generate an error. - -btb_ensure_not_used(Rs, I, Regs) -> - case lists:any(fun(R) -> btb_contains_context(R, Regs) end, Rs) of - true -> btb_error({binary_used_in,I}); - false -> ok - end. - -%% btb_kill([Register], RegisterSet) -> RegisterSet' -%% Kill all registers mentioned in the list of registers. - -btb_kill([{x,N}|Rs], {Xregs,Yregs}) -> - btb_kill(Rs, {Xregs band (bnot (1 bsl N)),Yregs}); -btb_kill([{y,N}|Rs], {Xregs,Yregs}) -> - btb_kill(Rs, {Xregs,Yregs band (bnot (1 bsl N))}); -btb_kill([{fr,_}|Rs], Regs) -> - btb_kill(Rs, Regs); -btb_kill([], Regs) -> Regs. - -%% btb_kill_not_live(Live, RegisterSet) -> RegisterSet' -%% Kill all registers indicated not live by Live. - -btb_kill_not_live(Live, {Xregs,Yregs}) -> - {Xregs band ((1 bsl Live)-1),Yregs}. - -%% btb_kill(Regs0) -> Regs -%% Kill all y registers. - -btb_kill_yregs({Xregs,_}) -> {Xregs,0}. - -%% btb_are_registers_empty(RegisterSet) -> true|false -%% Test whether the register set is empty. - -btb_are_registers_empty({0,0}) -> true; -btb_are_registers_empty({_,_}) -> false. - -%% btb_are_x_registers_empty(Regs) -> true|false -%% Test whether the x registers are empty. - -btb_are_x_registers_empty({0,_}) -> true; -btb_are_x_registers_empty({_,_}) -> false. - -%% btb_contains_context(Register, RegisterSet) -> true|false -%% Test whether Register contains the context. - -btb_contains_context({x,N}, {Regs,_}) -> Regs band (1 bsl N) =/= 0; -btb_contains_context({y,N}, {_,Regs}) -> Regs band (1 bsl N) =/= 0; -btb_contains_context(_, _) -> false. - -%% btb_context_regs(RegisterSet) -> [Register] -%% Convert the register set to an explicit list of registers. -btb_context_regs({Xregs,Yregs}) -> - btb_context_regs_1(Xregs, 0, x, btb_context_regs_1(Yregs, 0, y, [])). - -btb_context_regs_1(0, _, _, Acc) -> - Acc; -btb_context_regs_1(Regs, N, Tag, Acc) when (Regs band 1) =:= 1 -> - btb_context_regs_1(Regs bsr 1, N+1, Tag, [{Tag,N}|Acc]); -btb_context_regs_1(Regs, N, Tag, Acc) -> - btb_context_regs_1(Regs bsr 1, N+1, Tag, Acc). - -%% btb_index([Function]) -> GbTree({EntryLabel,{Register,MustSave}}) -%% Build an index of functions that accept a match context instead of -%% a binary. MustSave is true if the function may pass the match -%% context to the bs_context_to_binary instruction (in which case -%% the current position in the binary must have saved into the -%% start position using "bs_save_2 Ctx start"). - -btb_index(Fs) -> - btb_index_1(Fs, []). - -btb_index_1([{function,_,_,Entry,Is0}|Fs], Acc0) -> - Is = drop_to_label(Is0, Entry), - Acc = btb_index_2(Is, Entry, false, Acc0), - btb_index_1(Fs, Acc); -btb_index_1([], Acc) -> gb_trees:from_orddict(sort(Acc)). - -btb_index_2([{test,bs_start_match2,{f,_},_,[Reg,_],Reg}|_], - Entry, MustSave, Acc) -> - [{Entry,{Reg,MustSave}}|Acc]; -btb_index_2(Is0, Entry, _, Acc) -> - try btb_index_find_start_match(Is0) of - Is -> btb_index_2(Is, Entry, true, Acc) - catch - throw:none -> Acc - end. - -drop_to_label([{label,L}|Is], L) -> Is; -drop_to_label([_|Is], L) -> drop_to_label(Is, L). - -btb_index_find_start_match([{test,_,{f,F},_},{bs_context_to_binary,_}|Is]) -> - btb_index_find_label(Is, F); -btb_index_find_start_match(_) -> - throw(none). - -btb_index_find_label([{label,L}|Is], L) -> Is; -btb_index_find_label([_|Is], L) -> btb_index_find_label(Is, L). - -btb_error(Error) -> - throw({error,Error}). - -fetch_code_at(Lbl, Li) -> - case beam_utils:code_at(Lbl, Li) of - Is when is_list(Is) -> Is - end. - -%%% -%%% Compilation information warnings. -%%% - -btb_comment_opt({field_flags,[{anno,A}|_]}) -> - {'%',{bin_opt,A}}; -btb_comment_opt(_) -> - {'%',{bin_opt,[]}}. - -btb_comment_no_opt(Reason, {field_flags,[{anno,A}|_]}) -> - {'%',{no_bin_opt,Reason,A}}; -btb_comment_no_opt(Reason, _) -> - {'%',{no_bin_opt,Reason,[]}}. - -collect_warnings(Fs) -> - D = warning_index_functions(Fs), - foldl(fun(F, A) -> collect_warnings_fun(F, D, A) end, [], Fs). - -collect_warnings_fun({function,_,_,_,Is}, D, A) -> - collect_warnings_instr(Is, D, A). - -collect_warnings_instr([{'%',{bin_opt,Where}}|Is], D, Acc0) -> - Acc = add_warning(bin_opt, Where, Acc0), - collect_warnings_instr(Is, D, Acc); -collect_warnings_instr([{'%',{no_bin_opt,Reason0,Where}}|Is], D, Acc0) -> - Reason = warning_translate_label(Reason0, D), - Acc = add_warning({no_bin_opt,Reason}, Where, Acc0), - collect_warnings_instr(Is, D, Acc); -collect_warnings_instr([_|Is], D, Acc) -> - collect_warnings_instr(Is, D, Acc); -collect_warnings_instr([], _, Acc) -> Acc. - -add_warning(Term, Anno, Ws) -> - Line = get_line(Anno), - File = get_file(Anno), - [{File,[{Line,?MODULE,Term}]}|Ws]. - -warning_translate_label(Term, D) when is_tuple(Term) -> - case element(1, Term) of - {label,F} -> - FA = gb_trees:get(F, D), - setelement(1, Term, FA); - _ -> Term - end; -warning_translate_label(Term, _) -> Term. - -get_line([Line|_]) when is_integer(Line) -> Line; -get_line([_|T]) -> get_line(T); -get_line([]) -> none. - -get_file([{file,File}|_]) -> File; -get_file([_|T]) -> get_file(T); -get_file([]) -> "no_file". % should not happen - -warning_index_functions(Fs) -> - D = [{Entry,{F,A}} || {function,F,A,Entry,_} <- Fs], - gb_trees:from_orddict(sort(D)). - -format_error_1({binary_used_in,{extfunc,M,F,A}}) -> - [io_lib:format("sub binary used by ~p:~p/~p", [M,F,A])| - case {M,F,A} of - {erlang,split_binary,2} -> - "; SUGGEST using binary matching instead of split_binary/2"; - _ -> - "" - end]; -format_error_1({binary_used_in,_}) -> - "sub binary is used or returned"; -format_error_1({multiple_uses,_}) -> - "sub binary is matched or used in more than one place"; -format_error_1(unsuitable_bs_start_match) -> - "the binary matching instruction that follows in the same function " - "have problems that prevent delayed sub binary optimization " - "(probably indicated by INFO warnings)"; -format_error_1({{F,A},no_suitable_bs_start_match}) -> - io_lib:format("called function ~p/~p does not begin with a suitable " - "binary matching instruction", [F,A]); -format_error_1(must_and_must_not_save) -> - "different control paths use different positions in the binary"; -format_error_1({_,I,not_handled}) -> - case I of - {'catch',_,_} -> - "the compiler currently does not attempt the delayed sub binary " - "optimization when catch is used"; - {'try',_,_} -> - "the compiler currently does not attempt the delayed sub binary " - "optimization when try/catch is used"; - _ -> - io_lib:format("compiler limitation: instruction ~p prevents " - "delayed sub binary optimization", [I]) - end; -format_error_1(Term) -> - io_lib:format("~w", [Term]). diff --git a/lib/compiler/src/beam_clean.erl b/lib/compiler/src/beam_clean.erl index 207f1c4deb..7299654476 100644 --- a/lib/compiler/src/beam_clean.erl +++ b/lib/compiler/src/beam_clean.erl @@ -22,34 +22,21 @@ -module(beam_clean). -export([module/2]). --export([bs_clean_saves/1]). -export([clean_labels/1]). --import(lists, [foldl/3,reverse/1]). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. module({Mod,Exp,Attr,Fs0,_}, Opts) -> Order = [Lbl || {function,_,_,Lbl,_} <- Fs0], - All = foldl(fun({function,_,_,Lbl,_}=Func,D) -> dict:store(Lbl, Func, D) end, - dict:new(), Fs0), + All = maps:from_list([{Lbl,Func} || {function,_,_,Lbl,_}=Func <- Fs0]), WorkList = rootset(Fs0, Exp, Attr), - Used = find_all_used(WorkList, All, sets:from_list(WorkList)), + Used = find_all_used(WorkList, All, cerl_sets:from_list(WorkList)), Fs1 = remove_unused(Order, Used, All), {Fs2,Lc} = clean_labels(Fs1), - Fs3 = bs_fix(Fs2), - Fs = maybe_remove_lines(Fs3, Opts), + Fs = maybe_remove_lines(Fs2, Opts), {ok,{Mod,Exp,Attr,Fs,Lc}}. -%% Remove all bs_save2/2 instructions not referenced by a bs_restore2/2. - --spec bs_clean_saves([beam_utils:instruction()]) -> - [beam_utils:instruction()]. - -bs_clean_saves(Is) -> - Needed = bs_restores(Is, []), - bs_clean_saves_1(Is, gb_sets:from_list(Needed), []). - %% Determine the rootset, i.e. exported functions and %% the on_load function (if any). @@ -66,16 +53,16 @@ rootset(Fs, Root0, Attr) -> %% Remove the unused functions. remove_unused([F|Fs], Used, All) -> - case sets:is_element(F, Used) of + case cerl_sets:is_element(F, Used) of false -> remove_unused(Fs, Used, All); - true -> [dict:fetch(F, All)|remove_unused(Fs, Used, All)] + true -> [map_get(F, All)|remove_unused(Fs, Used, All)] end; remove_unused([], _, _) -> []. - + %% Find all used functions. find_all_used([F|Fs0], All, Used0) -> - {function,_,_,_,Code} = dict:fetch(F, All), + {function,_,_,_,Code} = map_get(F, All), {Fs,Used} = update_work_list(Code, {Fs0,Used0}), find_all_used(Fs, All, Used); find_all_used([], _All, Used) -> Used. @@ -89,20 +76,16 @@ update_work_list([_|Is], Sets) -> update_work_list([], Sets) -> Sets. add_to_work_list(F, {Fs,Used}=Sets) -> - case sets:is_element(F, Used) of + case cerl_sets:is_element(F, Used) of true -> Sets; - false -> {[F|Fs],sets:add_element(F, Used)} + false -> {[F|Fs],cerl_sets:add_element(F, Used)} end. %%% %%% Coalesce adjacent labels. Renumber all labels to eliminate gaps. -%%% This cleanup will slightly reduce file size and slightly speed up loading. -%%% -%%% We also expand is_record/3 to a sequence of instructions. It is done -%%% here merely because this module will always be called even if optimization -%%% is turned off. We don't want to do the expansion in beam_asm because we -%%% want to see the expanded code in a .S file. +%%% This cleanup will slightly reduce file size and slightly speed up +%%% loading. %%% -type label() :: beam_asm:label(). @@ -127,45 +110,6 @@ function_renumber([{function,Name,Arity,_Entry,Asm0}|Fs], St0, Acc) -> function_renumber(Fs, St, [{function,Name,Arity,St#st.entry,Asm}|Acc]); function_renumber([], St, Acc) -> {Acc,St}. -renumber_labels([{bif,is_record,{f,_}, - [Term,{atom,Tag}=TagAtom,{integer,Arity}],Dst}|Is0], Acc, St) -> - ContLabel = 900000000+2*St#st.lc, - FailLabel = ContLabel+1, - Fail = {f,FailLabel}, - Tmp = Dst, - Is = case is_record_tuple(Term, Tag, Arity) of - yes -> - [{move,{atom,true},Dst}|Is0]; - no -> - [{move,{atom,false},Dst}|Is0]; - maybe -> - [{test,is_tuple,Fail,[Term]}, - {test,test_arity,Fail,[Term,Arity]}, - {get_tuple_element,Term,0,Tmp}, - {test,is_eq_exact,Fail,[Tmp,TagAtom]}, - {move,{atom,true},Dst}, - {jump,{f,ContLabel}}, - {label,FailLabel}, - {move,{atom,false},Dst}, - {jump,{f,ContLabel}}, %Improves optimization by beam_dead. - {label,ContLabel}|Is0] - end, - renumber_labels(Is, Acc, St); -renumber_labels([{test,is_record,{f,_}=Fail, - [Term,{atom,Tag}=TagAtom,{integer,Arity}]}|Is0], Acc, St) -> - Tmp = {x,1022}, - Is = case is_record_tuple(Term, Tag, Arity) of - yes -> - Is0; - no -> - [{jump,Fail}|Is0]; - maybe -> - [{test,is_tuple,Fail,[Term]}, - {test,test_arity,Fail,[Term,Arity]}, - {get_tuple_element,Term,0,Tmp}, - {test,is_eq_exact,Fail,[Tmp,TagAtom]}|Is0] - end, - renumber_labels(Is, Acc, St); renumber_labels([{label,Old}|Is], [{label,New}|_]=Acc, #st{lmap=D0}=St) -> D = [{Old,New}|D0], renumber_labels(Is, Acc, St#st{lmap=D}); @@ -179,12 +123,6 @@ renumber_labels([I|Is], Acc, St0) -> renumber_labels(Is, [I|Acc], St0); renumber_labels([], Acc, St) -> {Acc,St}. -is_record_tuple({x,_}, _, _) -> maybe; -is_record_tuple({y,_}, _, _) -> maybe; -is_record_tuple({literal,Tuple}, Tag, Arity) - when element(1, Tuple) =:= Tag, tuple_size(Tuple) =:= Arity -> yes; -is_record_tuple(_, _, _) -> no. - function_replace([{function,Name,Arity,Entry,Asm0}|Fs], Dict, Acc) -> Asm = try Fb = fun(Old) -> throw({error,{undefined_label,Old}}) end, @@ -199,100 +137,6 @@ function_replace([{function,Name,Arity,Entry,Asm0}|Fs], Dict, Acc) -> function_replace([], _, Acc) -> Acc. %%% -%%% Final fixup of bs_start_match2/5,bs_save2/bs_restore2 instructions for -%%% new bit syntax matching (introduced in R11B). -%%% -%%% Pass 1: Scan the code, looking for bs_restore2/2 instructions. -%%% -%%% Pass 2: Update bs_save2/2 and bs_restore/2 instructions. Remove -%%% any bs_save2/2 instruction whose save position are never referenced -%%% by any bs_restore2/2 instruction. -%%% -%%% Note this module can be invoked several times, so we must be careful -%%% not to touch instructions that have already been fixed up. -%%% - -bs_fix(Fs) -> - bs_fix(Fs, []). - -bs_fix([{function,Name,Arity,Entry,Asm0}|Fs], Acc) -> - Asm = bs_function(Asm0), - bs_fix(Fs, [{function,Name,Arity,Entry,Asm}|Acc]); -bs_fix([], Acc) -> reverse(Acc). - -bs_function(Is) -> - Dict0 = bs_restores(Is, []), - S0 = sofs:relation(Dict0, [{context,save_point}]), - S1 = sofs:relation_to_family(S0), - S = sofs:to_external(S1), - Dict = make_save_point_dict(S, []), - bs_replace(Is, Dict, []). - -make_save_point_dict([{Ctx,Pts}|T], Acc0) -> - Acc = make_save_point_dict_1(Pts, Ctx, 0, Acc0), - make_save_point_dict(T, Acc); -make_save_point_dict([], Acc) -> - gb_trees:from_orddict(ordsets:from_list(Acc)). - -make_save_point_dict_1([H|T], Ctx, I, Acc) -> - make_save_point_dict_1(T, Ctx, I+1, [{{Ctx,H},I}|Acc]); -make_save_point_dict_1([], Ctx, I, Acc) -> - [{Ctx,I}|Acc]. - -%% Pass 1. -bs_restores([{bs_restore2,_,{Same,Same}}|Is], Dict) -> - %% This save point is special. No explicit save is needed. - bs_restores(Is, Dict); -bs_restores([{bs_restore2,_,{atom,start}}|Is], Dict) -> - %% This instruction can occur if "compilation" - %% started from a .S file. - bs_restores(Is, Dict); -bs_restores([{bs_restore2,_,{_,_}=SavePoint}|Is], Dict) -> - bs_restores(Is, [SavePoint|Dict]); -bs_restores([_|Is], Dict) -> - bs_restores(Is, Dict); -bs_restores([], Dict) -> Dict. - -%% Pass 2. -bs_replace([{test,bs_start_match2,F,Live,[Src,{context,Ctx}],CtxR}|T], Dict, Acc) -> - Slots = case gb_trees:lookup(Ctx, Dict) of - {value,Slots0} -> Slots0; - none -> 0 - end, - I = {test,bs_start_match2,F,Live,[Src,Slots],CtxR}, - bs_replace(T, Dict, [I|Acc]); -bs_replace([{bs_save2,CtxR,{_,_}=SavePoint}|T], Dict, Acc) -> - case gb_trees:lookup(SavePoint, Dict) of - {value,N} -> - bs_replace(T, Dict, [{bs_save2,CtxR,N}|Acc]); - none -> - bs_replace(T, Dict, Acc) - end; -bs_replace([{bs_restore2,_,{atom,start}}=I|T], Dict, Acc) -> - %% This instruction can occur if "compilation" - %% started from a .S file. - bs_replace(T, Dict, [I|Acc]); -bs_replace([{bs_restore2,CtxR,{Same,Same}}|T], Dict, Acc) -> - %% This save point refers to the point in the binary where the match - %% started. It has a special name. - bs_replace(T, Dict, [{bs_restore2,CtxR,{atom,start}}|Acc]); -bs_replace([{bs_restore2,CtxR,{_,_}=SavePoint}|T], Dict, Acc) -> - N = gb_trees:get(SavePoint, Dict), - bs_replace(T, Dict, [{bs_restore2,CtxR,N}|Acc]); -bs_replace([I|Is], Dict, Acc) -> - bs_replace(Is, Dict, [I|Acc]); -bs_replace([], _, Acc) -> reverse(Acc). - -bs_clean_saves_1([{bs_save2,_,{_,_}=SavePoint}=I|Is], Needed, Acc) -> - case gb_sets:is_member(SavePoint, Needed) of - false -> bs_clean_saves_1(Is, Needed, Acc); - true -> bs_clean_saves_1(Is, Needed, [I|Acc]) - end; -bs_clean_saves_1([I|Is], Needed, Acc) -> - bs_clean_saves_1(Is, Needed, [I|Acc]); -bs_clean_saves_1([], _, Acc) -> reverse(Acc). - -%%% %%% Remove line instructions if requested. %%% diff --git a/lib/compiler/src/beam_dead.erl b/lib/compiler/src/beam_dead.erl deleted file mode 100644 index efad082152..0000000000 --- a/lib/compiler/src/beam_dead.erl +++ /dev/null @@ -1,971 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2002-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(beam_dead). - --export([module/2]). - -%%% Dead code is code that is executed but has no effect. This -%%% optimization pass either removes dead code or jumps around it, -%%% potentially making it unreachable and a target for the -%%% the beam_jump pass. - --import(lists, [mapfoldl/3,reverse/1]). - - --spec module(beam_utils:module_code(), [compile:option()]) -> - {'ok',beam_utils:module_code()}. - -module({Mod,Exp,Attr,Fs0,_}, _Opts) -> - {Fs1,Lc1} = beam_clean:clean_labels(Fs0), - {Fs,Lc} = mapfoldl(fun function/2, Lc1, Fs1), - %%{Fs,Lc} = {Fs1,Lc1}, - {ok,{Mod,Exp,Attr,Fs,Lc}}. - -function({function,Name,Arity,CLabel,Is0}, Lc0) -> - try - Is1 = beam_jump:remove_unused_labels(Is0), - - %% Initialize label information with the code - %% for the func_info label. Without it, a register - %% may seem to be live when it is not. - [{label,L}|FiIs] = Is1, - D0 = beam_utils:empty_label_index(), - D = beam_utils:index_label(L, FiIs, D0), - - %% Optimize away dead code. - {Is2,Lc} = forward(Is1, Lc0), - Is3 = backward(Is2, D), - Is = move_move_into_block(Is3, []), - {{function,Name,Arity,CLabel,Is},Lc} - catch - Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [Name,Arity]), - erlang:raise(Class, Error, Stack) - end. - -%% 'move' instructions outside of blocks may thwart the jump optimizer. -%% Move them back into the block. - -move_move_into_block([{block,Bl0},{move,S,D}|Is], Acc) -> - Bl = Bl0 ++ [{set,[D],[S],move}], - move_move_into_block([{block,Bl}|Is], Acc); -move_move_into_block([{move,S,D}|Is], Acc) -> - Bl = [{set,[D],[S],move}], - move_move_into_block([{block,Bl}|Is], Acc); -move_move_into_block([I|Is], Acc) -> - move_move_into_block(Is, [I|Acc]); -move_move_into_block([], Acc) -> reverse(Acc). - -%%% -%%% Scan instructions in execution order and remove redundant 'move' -%%% instructions. 'move' instructions are redundant if we know that -%%% the register already contains the value being assigned, as in the -%%% following code: -%%% -%%% test is_eq_exact SomeLabel Src Dst -%%% move Src Dst -%%% -%%% or in: -%%% -%%% test is_nil SomeLabel Dst -%%% move nil Dst -%%% -%%% or in: -%%% -%%% select_val Register FailLabel [... Literal => L1...] -%%% . -%%% . -%%% . -%%% L1: move Literal Register -%%% -%%% Also add extra labels to help the second backward pass. -%%% - -forward(Is, Lc) -> - forward(Is, #{}, Lc, []). - -forward([{move,_,_}=Move|[{label,L}|_]=Is], D, Lc, Acc) -> - %% move/2 followed by jump/1 is optimized by backward/3. - forward([Move,{jump,{f,L}}|Is], D, Lc, Acc); -forward([{bif,_,_,_,_}=Bif|[{label,L}|_]=Is], D, Lc, Acc) -> - %% bif/4 followed by jump/1 is optimized by backward/3. - forward([Bif,{jump,{f,L}}|Is], D, Lc, Acc); -forward([{block,[]}|Is], D, Lc, Acc) -> - %% Empty blocks can prevent optimizations. - forward(Is, D, Lc, Acc); -forward([{select,select_val,Reg,_,List}=I|Is], D0, Lc, Acc) -> - D = update_value_dict(List, Reg, D0), - forward(Is, D, Lc, [I|Acc]); -forward([{label,Lbl}=LblI,{block,[{set,[Dst],[Lit],move}|BlkIs]}=Blk|Is], D, Lc, Acc) -> - %% Assumption: The target labels in a select_val/3 instruction - %% cannot be reached in any other way than through the select_val/3 - %% instruction (i.e. there can be no fallthrough to such label and - %% it cannot be referenced by, for example, a jump/1 instruction). - Key = {Lbl,Dst}, - Block = case D of - #{Key := Lit} -> {block,BlkIs}; %Safe to remove move instruction. - _ -> Blk %Must keep move instruction. - end, - forward([Block|Is], D, Lc, [LblI|Acc]); -forward([{label,Lbl}=LblI|[{move,Lit,Dst}|Is1]=Is0], D, Lc, Acc) -> - %% Assumption: The target labels in a select_val/3 instruction - %% cannot be reached in any other way than through the select_val/3 - %% instruction (i.e. there can be no fallthrough to such label and - %% it cannot be referenced by, for example, a jump/1 instruction). - Is = case maps:find({Lbl,Dst}, D) of - {ok,Lit} -> Is1; %Safe to remove move instruction. - _ -> Is0 %Keep move instruction. - end, - forward(Is, D, Lc, [LblI|Acc]); -forward([{test,is_eq_exact,_,[Same,Same]}|Is], D, Lc, Acc) -> - forward(Is, D, Lc, Acc); -forward([{test,is_eq_exact,_,[Dst,Src]}=I, - {block,[{set,[Dst],[Src],move}|Bl]}|Is], D, Lc, Acc) -> - forward([I,{block,Bl}|Is], D, Lc, Acc); -forward([{test,is_nil,_,[Dst]}=I, - {block,[{set,[Dst],[nil],move}|Bl]}|Is], D, Lc, Acc) -> - forward([I,{block,Bl}|Is], D, Lc, Acc); -forward([{test,is_eq_exact,_,[Dst,Src]}=I,{move,Src,Dst}|Is], D, Lc, Acc) -> - forward([I|Is], D, Lc, Acc); -forward([{test,is_nil,_,[Dst]}=I,{move,nil,Dst}|Is], D, Lc, Acc) -> - forward([I|Is], D, Lc, Acc); -forward([{test,_,_,_}=I|Is]=Is0, D, Lc, Acc) -> - %% Help the second, backward pass to by inserting labels after - %% relational operators so that they can be skipped if they are - %% known to be true. - case useful_to_insert_label(Is0) of - false -> forward(Is, D, Lc, [I|Acc]); - true -> forward(Is, D, Lc+1, [{label,Lc},I|Acc]) - end; -forward([I|Is], D, Lc, Acc) -> - forward(Is, D, Lc, [I|Acc]); -forward([], _, Lc, Acc) -> {Acc,Lc}. - -update_value_dict([Lit,{f,Lbl}|T], Reg, D0) -> - Key = {Lbl,Reg}, - D = case D0 of - #{Key := inconsistent} -> D0; - #{Key := _} -> D0#{Key := inconsistent}; - _ -> D0#{Key => Lit} - end, - update_value_dict(T, Reg, D); -update_value_dict([], _, D) -> D. - -useful_to_insert_label([_,{label,_}|_]) -> - false; -useful_to_insert_label([{test,Op,_,_}|_]) -> - case Op of - is_lt -> true; - is_ge -> true; - is_eq_exact -> true; - is_ne_exact -> true; - _ -> false - end. - -%%% -%%% Scan instructions in reverse execution order and try to -%%% shortcut branch instructions. -%%% -%%% For example, in this code: -%%% -%%% move Literal Register -%%% jump L1 -%%% . -%%% . -%%% . -%%% L1: test is_{integer,atom} FailLabel Register -%%% select_val {x,0} FailLabel [... Literal => L2...] -%%% . -%%% . -%%% . -%%% L2: ... -%%% -%%% the 'selectval' instruction will always transfer control to L2, -%%% so we can just as well jump to L2 directly by rewriting the -%%% first part of the sequence like this: -%%% -%%% move Literal Register -%%% jump L2 -%%% -%%% If register Register is killed at label L2, we can remove the -%%% 'move' instruction, leaving just the 'jump' instruction: -%%% -%%% jump L2 -%%% -%%% These transformations may leave parts of the code unreachable. -%%% The beam_jump pass will remove the unreachable code. - -backward(Is, D) -> - backward(Is, D, []). - -backward([{test,is_eq_exact,Fail,[Dst,{integer,Arity}]}=I| - [{bif,tuple_size,Fail,[Reg],Dst}|Is]=Is0], D, Acc) -> - %% Provided that Dst is killed following this sequence, - %% we can rewrite the instructions like this: - %% - %% bif tuple_size Fail Reg Dst ==> is_tuple Fail Reg - %% is_eq_exact Fail Dst Integer test_arity Fail Reg Integer - %% - %% (still two instructions, but they they will be combined to - %% one by the loader). - case beam_utils:is_killed(Dst, Acc, D) andalso (Arity bsr 32) =:= 0 of - false -> - %% Not safe because the register Dst is not killed - %% (probably cannot not happen in practice) or the arity - %% does not fit in 32 bits (the loader will fail to load - %% the module). We must move the first instruction to the - %% accumulator to avoid an infinite loop. - backward(Is0, D, [I|Acc]); - true -> - %% Safe. - backward([{test,test_arity,Fail,[Reg,Arity]}, - {test,is_tuple,Fail,[Reg]}|Is], D, Acc) - end; -backward([{label,Lbl}=L|Is], D, Acc) -> - backward(Is, beam_utils:index_label(Lbl, Acc, D), [L|Acc]); -backward([{select,select_val,Reg,{f,Fail0},List0}|Is], D, Acc) -> - List1 = shortcut_select_list(List0, Reg, D, []), - Fail1 = shortcut_label(Fail0, D), - Fail = shortcut_bs_test(Fail1, Is, D), - List = prune_redundant(List1, Fail), - case List of - [] -> - Jump = {jump,{f,Fail}}, - backward([Jump|Is], D, Acc); - [V,F] -> - Test = {test,is_eq_exact,{f,Fail},[Reg,V]}, - Jump = {jump,F}, - backward([Jump,Test|Is], D, Acc); - [{atom,B1},F,{atom,B2},F] when B1 =:= not B2 -> - Test = {test,is_boolean,{f,Fail},[Reg]}, - Jump = {jump,F}, - backward([Jump,Test|Is], D, Acc); - [_|_] -> - Sel = {select,select_val,Reg,{f,Fail},List}, - backward(Is, D, [Sel|Acc]) - end; -backward([{jump,{f,To0}},{move,Src,Reg}=Move|Is], D, Acc) -> - To = shortcut_select_label(To0, Reg, Src, D), - Jump = {jump,{f,To}}, - case is_killed_at(Reg, To, D) of - false -> backward([Move|Is], D, [Jump|Acc]); - true -> backward([Jump|Is], D, Acc) - end; -backward([{jump,{f,To}}=J|[{bif,Op,{f,BifFail},Ops,Reg}|Is]=Is0], D, Acc) -> - try replace_comp_op(To, Reg, Op, Ops, D) of - {Test,Jump} -> - backward([Jump,Test|Is], D, Acc) - catch - throw:not_possible -> - case To =:= BifFail of - true -> - %% The bif instruction is redundant. See the comment - %% in the next clause for why there is no need to - %% test for liveness of Reg at label To. - backward([J|Is], D, Acc); - false -> - backward(Is0, D, [J|Acc]) - end - end; -backward([{jump,{f,To}}=J|[{gc_bif,_,{f,To},_,_,_Dst}|Is]], D, Acc) -> - %% The gc_bif instruction is redundant, since either the gc_bif - %% instruction itself or the jump instruction will transfer control - %% to label To. Note that a gc_bif instruction does not assign its - %% destination register if the failure branch is taken; therefore, - %% the code at label To is not allowed to assume that the destination - %% register is initialized, and it is therefore no need to test - %% for liveness of the destination register at label To. - backward([J|Is], D, Acc); -backward([{test,bs_start_match2,F,Live,[Src,_]=Args,Ctxt}|Is], D, Acc0) -> - {f,To0} = F, - case test_bs_literal(F, Ctxt, D, Acc0) of - {none,Acc} -> - %% Ctxt killed immediately after bs_start_match2. - To = shortcut_bs_context_to_binary(To0, Src, D), - I = {test,is_bitstr,{f,To},[Src]}, - backward(Is, D, [I|Acc]); - {Literal,Acc} -> - %% Ctxt killed after matching a literal. - To = shortcut_bs_context_to_binary(To0, Src, D), - Eq = {test,is_eq_exact,{f,To},[Src,{literal,Literal}]}, - backward(Is, D, [Eq|Acc]); - not_killed -> - %% Ctxt not killed. Not much to do. - To = shortcut_bs_start_match(To0, Src, D), - I = {test,bs_start_match2,{f,To},Live,Args,Ctxt}, - backward(Is, D, [I|Acc0]) - end; -backward([{test,Op,{f,To0},Ops0}|Is], D, Acc) -> - To1 = shortcut_bs_test(To0, Is, D), - To2 = shortcut_label(To1, D), - To3 = shortcut_rel_op(To2, Op, Ops0, D), - - %% Try to shortcut a repeated test: - %% - %% test Op {f,Fail1} Operands test Op {f,Fail2} Operands - %% . . . ==> ... - %% Fail1: test Op {f,Fail2} Operands Fail1: test Op {f,Fail2} Operands - %% - To = case beam_utils:code_at(To3, D) of - [{test,Op,{f,To4},Ops}|_] -> - case equal_ops(Ops0, Ops) of - true -> To4; - false -> To3 - end; - _Code -> - To3 - end, - I = case Op of - is_eq_exact -> combine_eqs(To, Ops0, D, Acc); - _ -> {test,Op,{f,To},Ops0} - end, - case {I,Acc} of - {{test,is_atom,Fail,Ops0},[{test,is_boolean,Fail,Ops0}|_]} -> - %% An is_atom test before an is_boolean test (with the - %% same failure label) is redundant. - backward(Is, D, Acc); - {{test,is_atom,Fail,[R]}, - [{test,is_eq_exact,Fail,[R,{atom,_}]}|_]} -> - %% An is_atom test before a comparison with an atom (with - %% the same failure label) is redundant. - backward(Is, D, Acc); - {{test,is_integer,Fail,[R]}, - [{test,is_eq_exact,Fail,[R,{integer,_}]}|_]} -> - %% An is_integer test before a comparison with an integer - %% (with the same failure label) is redundant. - backward(Is, D, Acc); - {{test,_,_,_},_} -> - %% Still a test instruction. Done. - backward(Is, D, [I|Acc]); - {_,_} -> - %% Rewritten to a select_val. Rescan. - backward([I|Is], D, Acc) - end; -backward([{test,Op,{f,To0},Live,Ops0,Dst}|Is], D, Acc) -> - To1 = shortcut_bs_test(To0, Is, D), - To2 = shortcut_label(To1, D), - %% Try to shortcut a repeated test: - %% - %% test Op {f,Fail1} _ Ops _ test Op {f,Fail2} _ Ops _ - %% . . . ==> ... - %% Fail1: test Op {f,Fail2} _ Ops _ Fail1: test Op {f,Fail2} _ Ops _ - %% - To = case beam_utils:code_at(To2, D) of - [{test,Op,{f,To3},_,Ops,_}|_] -> - case equal_ops(Ops0, Ops) of - true -> To3; - false -> To2 - end; - _Code -> - To2 - end, - I = {test,Op,{f,To},Live,Ops0,Dst}, - backward(Is, D, [I|Acc]); -backward([{kill,_}=I|Is], D, [{line,_},Exit|_]=Acc) -> - case beam_jump:is_exit_instruction(Exit) of - false -> backward(Is, D, [I|Acc]); - true -> backward(Is, D, Acc) - end; -backward([{bif,'or',{f,To0},[Dst,{atom,false}],Dst}=I|Is], D, - [{test,is_eq_exact,{f,To},[Dst,{atom,true}]}|_]=Acc) -> - case shortcut_label(To0, D) of - To -> - backward(Is, D, Acc); - _ -> - backward(Is, D, [I|Acc]) - end; -backward([{bif,map_get,{f,FF},[Key,Map],_}=I0, - {test,has_map_fields,{f,FT}=F,[Map|Keys0]}=I1|Is], D, Acc) when FF =/= 0 -> - case shortcut_label(FF, D) of - FT -> - case lists:delete(Key, Keys0) of - [] -> - backward([I0|Is], D, Acc); - Keys -> - Test = {test,has_map_fields,F,[Map|Keys]}, - backward([Test|Is], D, [I0|Acc]) - end; - _ -> - backward([I1|Is], D, [I0|Acc]) - end; -backward([{bif,map_get,{f,FF},[_,Map],_}=I0, - {test,is_map,{f,FT},[Map]}=I1|Is], D, Acc) when FF =/= 0 -> - case shortcut_label(FF, D) of - FT -> backward([I0|Is], D, Acc); - _ -> backward([I1|Is], D, [I0|Acc]) - end; -backward([I|Is], D, Acc) -> - backward(Is, D, [I|Acc]); -backward([], _D, Acc) -> Acc. - -equal_ops([{field_flags,FlA0}|T0], [{field_flags,FlB0}|T1]) -> - FlA = lists:keydelete(anno, 1, FlA0), - FlB = lists:keydelete(anno, 1, FlB0), - FlA =:= FlB andalso equal_ops(T0, T1); -equal_ops([Op|T0], [Op|T1]) -> - equal_ops(T0, T1); -equal_ops([], []) -> true; -equal_ops(_, _) -> false. - -shortcut_select_list([Lit,{f,To0}|T], Reg, D, Acc) -> - To = shortcut_select_label(To0, Reg, Lit, D), - shortcut_select_list(T, Reg, D, [{f,To},Lit|Acc]); -shortcut_select_list([], _, _, Acc) -> reverse(Acc). - -shortcut_label(0, _) -> - 0; -shortcut_label(To0, D) -> - case beam_utils:code_at(To0, D) of - [{jump,{f,To}}|_] -> shortcut_label(To, D); - _ -> To0 - end. - -shortcut_select_label(To, Reg, Lit, D) -> - shortcut_rel_op(To, is_ne_exact, [Reg,Lit], D). - -prune_redundant([_,{f,Fail}|T], Fail) -> - prune_redundant(T, Fail); -prune_redundant([V,F|T], Fail) -> - [V,F|prune_redundant(T, Fail)]; -prune_redundant([], _) -> []. - -%% Replace a comparison operator with a test instruction and a jump. -%% For example, if we have this code: -%% -%% bif '=:=' Fail Src1 Src2 {x,0} -%% jump L1 -%% . -%% . -%% . -%% L1: select_val {x,0} FailLabel [... true => L2..., ...false => L3...] -%% -%% the first two instructions can be replaced with -%% -%% test is_eq_exact L3 Src1 Src2 -%% jump L2 -%% -%% provided that {x,0} is killed at both L2 and L3. - -replace_comp_op(To, Reg, Op, Ops, D) -> - False = comp_op_find_shortcut(To, Reg, {atom,false}, D), - True = comp_op_find_shortcut(To, Reg, {atom,true}, D), - {bif_to_test(Op, Ops, False),{jump,{f,True}}}. - -comp_op_find_shortcut(To0, Reg, Val, D) -> - case shortcut_select_label(To0, Reg, Val, D) of - To0 -> - not_possible(); - To -> - case is_killed_at(Reg, To, D) of - false -> not_possible(); - true -> To - end - end. - -bif_to_test(Name, Args, Fail) -> - try - beam_utils:bif_to_test(Name, Args, {f,Fail}) - catch - error:_ -> not_possible() - end. - -not_possible() -> throw(not_possible). - -%% combine_eqs(To, Operands, Acc) -> Instruction. -%% Combine two is_eq_exact instructions or (an is_eq_exact -%% instruction and a select_val instruction) to a select_val -%% instruction if possible. -%% -%% Example: -%% -%% is_eq_exact F1 Reg Lit1 select_val Reg F2 [ Lit1 L1 -%% L1: . Lit2 L2 ] -%% . -%% . ==> -%% . -%% F1: is_eq_exact F2 Reg Lit2 F1: is_eq_exact F2 Reg Lit2 -%% L2: .... L2: -%% -combine_eqs(To, [Reg,{Type,_}=Lit1]=Ops, D, Acc) - when Type =:= atom; Type =:= integer -> - Next = case Acc of - [{label,Lbl}|_] -> Lbl; - [{jump,{f,Lbl}}|_] -> Lbl - end, - case beam_utils:code_at(To, D) of - [{test,is_eq_exact,{f,F2},[Reg,{Type,_}=Lit2]}, - {label,L2}|_] when Lit1 =/= Lit2 -> - {select,select_val,Reg,{f,F2},[Lit1,{f,Next},Lit2,{f,L2}]}; - [{test,is_eq_exact,{f,F2},[Reg,{Type,_}=Lit2]}, - {jump,{f,L2}}|_] when Lit1 =/= Lit2 -> - {select,select_val,Reg,{f,F2},[Lit1,{f,Next},Lit2,{f,L2}]}; - [{select,select_val,Reg,{f,F2},[{Type,_}|_]=List0}|_] -> - List = remove_from_list(Lit1, List0), - {select,select_val,Reg,{f,F2},[Lit1,{f,Next}|List]}; - _Is -> - {test,is_eq_exact,{f,To},Ops} - end; -combine_eqs(To, Ops, _D, _Acc) -> - {test,is_eq_exact,{f,To},Ops}. - -remove_from_list(Lit, [Lit,{f,_}|T]) -> - T; -remove_from_list(Lit, [Val,{f,_}=Fail|T]) -> - [Val,Fail|remove_from_list(Lit, T)]; -remove_from_list(_, []) -> []. - - -test_bs_literal(F, Ctxt, D, - [{test,bs_match_string,F,[Ctxt,Bs]}, - {test,bs_test_tail2,F,[Ctxt,0]}|Acc]) -> - test_bs_literal_1(Ctxt, Acc, D, Bs); -test_bs_literal(F, Ctxt, D, [{test,bs_test_tail2,F,[Ctxt,0]}|Acc]) -> - test_bs_literal_1(Ctxt, Acc, D, <<>>); -test_bs_literal(_, Ctxt, D, Acc) -> - test_bs_literal_1(Ctxt, Acc, D, none). - -test_bs_literal_1(Ctxt, Is, D, Literal) -> - case beam_utils:is_killed(Ctxt, Is, D) of - true -> {Literal,Is}; - false -> not_killed - end. - -%% shortcut_bs_test(TargetLabel, ReversedInstructions, D) -> TargetLabel' -%% Try to shortcut the failure label for bit syntax matching. - -shortcut_bs_test(To, Is, D) -> - shortcut_bs_test_1(beam_utils:code_at(To, D), Is, To, D). - -shortcut_bs_test_1([{bs_restore2,Reg,SavePoint}, - {label,_}, - {test,bs_test_tail2,{f,To},[_,TailBits]}|_], - PrevIs, To0, D) -> - case count_bits_matched(PrevIs, {Reg,SavePoint}, 0) of - Bits when Bits > TailBits -> - %% This instruction will fail. We know because a restore has been - %% done from the previous point SavePoint in the binary, and we - %% also know that the binary contains at least Bits bits from - %% SavePoint. - %% - %% Since we will skip a bs_restore2 if we shortcut to label To, - %% we must now make sure that code at To does not depend on - %% the position in the context in any way. - case shortcut_bs_pos_used(To, Reg, D) of - false -> To; - true -> To0 - end; - _Bits -> - To0 - end; -shortcut_bs_test_1([_|_], _, To, _) -> To. - -%% counts_bits_matched(ReversedInstructions, SavePoint, Bits) -> Bits' -%% Given a reversed instruction stream, determine the minimum number -%% of bits that will be matched by bit syntax instructions up to the -%% given save point. - -count_bits_matched([{test,bs_get_utf8,{f,_},_,_,_}|Is], SavePoint, Bits) -> - count_bits_matched(Is, SavePoint, Bits+8); -count_bits_matched([{test,bs_get_utf16,{f,_},_,_,_}|Is], SavePoint, Bits) -> - count_bits_matched(Is, SavePoint, Bits+16); -count_bits_matched([{test,bs_get_utf32,{f,_},_,_,_}|Is], SavePoint, Bits) -> - count_bits_matched(Is, SavePoint, Bits+32); -count_bits_matched([{test,_,_,_,[_,Sz,U,{field_flags,_}],_}|Is], SavePoint, Bits) -> - case Sz of - {integer,N} -> count_bits_matched(Is, SavePoint, Bits+N*U); - _ -> count_bits_matched(Is, SavePoint, Bits) - end; -count_bits_matched([{test,bs_match_string,_,[_,Bs]}|Is], SavePoint, Bits) -> - count_bits_matched(Is, SavePoint, Bits+bit_size(Bs)); -count_bits_matched([{test,_,_,_}|Is], SavePoint, Bits) -> - count_bits_matched(Is, SavePoint, Bits); -count_bits_matched([{bs_save2,Reg,SavePoint}|_], {Reg,SavePoint}, Bits) -> - %% The save point we are looking for - we are done. - Bits; -count_bits_matched([_|_], _, Bits) -> Bits. - -shortcut_bs_pos_used(To, Reg, D) -> - shortcut_bs_pos_used_1(beam_utils:code_at(To, D), Reg, D). - -shortcut_bs_pos_used_1([{bs_context_to_binary,Reg}|_], Reg, _) -> - false; -shortcut_bs_pos_used_1(Is, Reg, D) -> - not beam_utils:is_killed(Reg, Is, D). - -%% shortcut_bs_start_match(TargetLabel, Reg) -> TargetLabel -%% A failing bs_start_match2 instruction means that the source (Reg) -%% cannot be a binary. That means that it is safe to skip -%% bs_context_to_binary instructions operating on Reg, and -%% bs_start_match2 instructions operating on Reg. - -shortcut_bs_start_match(To, Reg, D) -> - shortcut_bs_start_match_1(beam_utils:code_at(To, D), Reg, To, D). - -shortcut_bs_start_match_1([{bs_context_to_binary,Reg}|Is], Reg, To, D) -> - shortcut_bs_start_match_1(Is, Reg, To, D); -shortcut_bs_start_match_1([{jump,{f,To}}|_], Reg, _, D) -> - Code = beam_utils:code_at(To, D), - shortcut_bs_start_match_1(Code, Reg, To, D); -shortcut_bs_start_match_1([{test,bs_start_match2,{f,To},_,[Reg|_],_}|_], - Reg, _, D) -> - Code = beam_utils:code_at(To, D), - shortcut_bs_start_match_1(Code, Reg, To, D); -shortcut_bs_start_match_1(_, _, To, _) -> - To. - -%% shortcut_bs_context_to_binary(TargetLabel, Reg) -> TargetLabel -%% If a bs_start_match2 instruction has been eliminated, the -%% bs_context_to_binary instruction can be eliminated too. - -shortcut_bs_context_to_binary(To, Reg, D) -> - shortcut_bs_ctb_1(beam_utils:code_at(To, D), Reg, To, D). - -shortcut_bs_ctb_1([{bs_context_to_binary,Reg}|Is], Reg, To, D) -> - shortcut_bs_ctb_1(Is, Reg, To, D); -shortcut_bs_ctb_1([{jump,{f,To}}|_], Reg, _, D) -> - Code = beam_utils:code_at(To, D), - shortcut_bs_ctb_1(Code, Reg, To, D); -shortcut_bs_ctb_1(_, _, To, _) -> - To. - -%% shortcut_rel_op(FailLabel, Operator, [Operand], D) -> FailLabel' -%% Try to shortcut the given test instruction. Example: -%% -%% is_ge L1 {x,0} 48 -%% . -%% . -%% . -%% L1: is_ge L2 {x,0} 65 -%% -%% The first test instruction can be rewritten to "is_ge L2 {x,0} 48" -%% since the instruction at L1 will also fail. -%% -%% If there are instructions between L1 and the other test instruction -%% it may still be possible to do the shortcut. For example: -%% -%% L1: is_eq_exact L3 {x,0} 92 -%% is_ge L2 {x,0} 65 -%% -%% Since the first test instruction failed, we know that {x,0} must -%% be less than 48; therefore, we know that {x,0} cannot be equal to -%% 92 and the jump to L3 cannot happen. - -shortcut_rel_op(To, Op, Ops, D) -> - case normalize_op({test,Op,{f,To},Ops}) of - {{NormOp,A,B},_} -> - Normalized = {negate_op(NormOp),A,B}, - shortcut_rel_op_fp(To, Normalized, D); - {_,_} -> - To; - error -> - To - end. - -shortcut_rel_op_fp(To0, Normalized, D) -> - Code = beam_utils:code_at(To0, D), - case shortcut_any_label(Code, Normalized) of - error -> - To0; - To -> - shortcut_rel_op_fp(To, Normalized, D) - end. - -%% shortcut_any_label([Instruction], PrevCondition) -> FailLabel | error -%% Using PrevCondition (a previous condition known to be true), -%% try to shortcut to another failure label. - -shortcut_any_label([{jump,{f,Lbl}}|_], _Prev) -> - Lbl; -shortcut_any_label([{label,Lbl}|_], _Prev) -> - Lbl; -shortcut_any_label([{select,select_val,R,{f,Fail},L}|_], Prev) -> - shortcut_selectval(L, R, Fail, Prev); -shortcut_any_label([I|Is], Prev) -> - case normalize_op(I) of - error -> - error; - {Normalized,Fail} -> - %% We have a relational operator. - case will_succeed(Prev, Normalized) of - no -> - %% This test instruction will always branch - %% to Fail. - Fail; - yes -> - %% This test instruction will never branch, - %% so we will look at the next instruction. - shortcut_any_label(Is, Prev); - maybe -> - %% May or may not branch. From now on, we can only - %% shortcut to the this specific failure label - %% Fail. - shortcut_specific_label(Is, Fail, Prev) - end - end. - -%% shortcut_specific_label([Instruction], FailLabel, PrevCondition) -> -%% FailLabel | error -%% We have previously encountered a test instruction that may or -%% may not branch to FailLabel. Therefore we are only allowed -%% to do the shortcut to the same fail label (FailLabel). - -shortcut_specific_label([{label,_}|Is], Fail, Prev) -> - shortcut_specific_label(Is, Fail, Prev); -shortcut_specific_label([{select,select_val,R,{f,F},L}|_], Fail, Prev) -> - case shortcut_selectval(L, R, F, Prev) of - Fail -> Fail; - _ -> error - end; -shortcut_specific_label([I|Is], Fail, Prev) -> - case normalize_op(I) of - error -> - error; - {Normalized,Fail} -> - case will_succeed(Prev, Normalized) of - no -> - %% Will branch to FailLabel. - Fail; - yes -> - %% Will definitely never branch. - shortcut_specific_label(Is, Fail, Prev); - maybe -> - %% May branch, but still OK since it will branch - %% to FailLabel. - shortcut_specific_label(Is, Fail, Prev) - end; - {Normalized,_} -> - %% This test instruction will branch to a different - %% fail label, if it branches at all. - case will_succeed(Prev, Normalized) of - yes -> - %% Still OK, since the branch will never be - %% taken. - shortcut_specific_label(Is, Fail, Prev); - no -> - %% Give up. The branch will definitely be taken - %% to a different fail label. - error; - maybe -> - %% Give up. If the branch is taken, it will be - %% to a different fail label. - error - end - end. - - -%% shortcut_selectval(List, Reg, Fail, PrevCond) -> FailLabel | error -%% Try to shortcut a selectval instruction. A selectval instruction -%% is equivalent to the following instruction sequence: -%% -%% is_ne_exact L1 Reg Value1 -%% . -%% . -%% . -%% is_ne_exact LN Reg ValueN -%% jump DefaultFailLabel -%% -shortcut_selectval([Val,{f,Lbl}|T], R, Fail, Prev) -> - case will_succeed(Prev, {'=/=',R,get_literal(Val)}) of - yes -> shortcut_selectval(T, R, Fail, Prev); - no -> Lbl; - maybe -> error - end; -shortcut_selectval([], _, Fail, _) -> Fail. - -%% will_succeed(PrevCondition, Condition) -> yes | no | maybe -%% PrevCondition is a condition known to be true. This function -%% will tell whether Condition will succeed. - -will_succeed({Op1,Reg,A}, {Op2,Reg,B}) -> - will_succeed_1(Op1, A, Op2, B); -will_succeed({'=:=',Reg,{literal,A}}, {TypeTest,Reg}) -> - case erlang:TypeTest(A) of - false -> no; - true -> yes - end; -will_succeed({_,_,_}, maybe) -> - maybe; -will_succeed({_,_,_}, Test) when is_tuple(Test) -> - maybe. - -will_succeed_1('=:=', A, '<', B) -> - if - B =< A -> no; - true -> yes - end; -will_succeed_1('=:=', A, '=<', B) -> - if - B < A -> no; - true -> yes - end; -will_succeed_1('=:=', A, '=:=', B) -> - if - A =:= B -> yes; - true -> no - end; -will_succeed_1('=:=', A, '=/=', B) -> - if - A =:= B -> no; - true -> yes - end; -will_succeed_1('=:=', A, '>=', B) -> - if - B > A -> no; - true -> yes - end; -will_succeed_1('=:=', A, '>', B) -> - if - B >= A -> no; - true -> yes - end; - -will_succeed_1('=/=', A, '=/=', B) when A =:= B -> yes; -will_succeed_1('=/=', A, '=:=', B) when A =:= B -> no; - -will_succeed_1('<', A, '=:=', B) when B >= A -> no; -will_succeed_1('<', A, '=/=', B) when B >= A -> yes; -will_succeed_1('<', A, '<', B) when B >= A -> yes; -will_succeed_1('<', A, '=<', B) when B > A -> yes; -will_succeed_1('<', A, '>=', B) when B > A -> no; -will_succeed_1('<', A, '>', B) when B >= A -> no; - -will_succeed_1('=<', A, '=:=', B) when B > A -> no; -will_succeed_1('=<', A, '=/=', B) when B > A -> yes; -will_succeed_1('=<', A, '<', B) when B > A -> yes; -will_succeed_1('=<', A, '=<', B) when B >= A -> yes; -will_succeed_1('=<', A, '>=', B) when B > A -> no; -will_succeed_1('=<', A, '>', B) when B >= A -> no; - -will_succeed_1('>=', A, '=:=', B) when B < A -> no; -will_succeed_1('>=', A, '=/=', B) when B < A -> yes; -will_succeed_1('>=', A, '<', B) when B =< A -> no; -will_succeed_1('>=', A, '=<', B) when B < A -> no; -will_succeed_1('>=', A, '>=', B) when B =< A -> yes; -will_succeed_1('>=', A, '>', B) when B < A -> yes; - -will_succeed_1('>', A, '=:=', B) when B =< A -> no; -will_succeed_1('>', A, '=/=', B) when B =< A -> yes; -will_succeed_1('>', A, '<', B) when B =< A -> no; -will_succeed_1('>', A, '=<', B) when B < A -> no; -will_succeed_1('>', A, '>=', B) when B =< A -> yes; -will_succeed_1('>', A, '>', B) when B < A -> yes; - -will_succeed_1(_, _, _, _) -> maybe. - -%% normalize_op(Instruction) -> {Normalized,FailLabel} | error -%% Normalized = {Operator,Register,Literal} | -%% {TypeTest,Register} | -%% maybe -%% Operation = '<' | '=<' | '=:=' | '=/=' | '>=' | '>' -%% TypeTest = is_atom | is_integer ... -%% Literal = {literal,Term} -%% -%% Normalize a relational operator to facilitate further -%% comparisons between operators. Always make the register -%% operand the first operand. Thus the following instruction: -%% -%% {test,is_ge,{f,99},{integer,13},{x,0}} -%% -%% will be normalized to: -%% -%% {'=<',{x,0},{literal,13}} -%% -%% NOTE: Bit syntax test instructions are scary. They may change the -%% state of match contexts and update registers, so we don't dare -%% mess with them. - -normalize_op({test,is_ge,{f,Fail},Ops}) -> - normalize_op_1('>=', Ops, Fail); -normalize_op({test,is_lt,{f,Fail},Ops}) -> - normalize_op_1('<', Ops, Fail); -normalize_op({test,is_eq_exact,{f,Fail},Ops}) -> - normalize_op_1('=:=', Ops, Fail); -normalize_op({test,is_ne_exact,{f,Fail},Ops}) -> - normalize_op_1('=/=', Ops, Fail); -normalize_op({test,is_nil,{f,Fail},[R]}) -> - normalize_op_1('=:=', [R,nil], Fail); -normalize_op({test,Op,{f,Fail},[R]}) -> - case erl_internal:new_type_test(Op, 1) of - true -> {{Op,R},Fail}; - false -> {maybe,Fail} - end; -normalize_op({test,_,{f,Fail},_}=I) -> - case beam_utils:is_pure_test(I) of - true -> {maybe,Fail}; - false -> error - end; -normalize_op(_) -> - error. - -normalize_op_1(Op, [Op1,Op2], Fail) -> - case {get_literal(Op1),get_literal(Op2)} of - {error,error} -> - %% Both operands are registers. - {maybe,Fail}; - {error,Lit} -> - {{Op,Op1,Lit},Fail}; - {Lit,error} -> - {{turn_op(Op),Op2,Lit},Fail}; - {_,_} -> - %% Both operands are literals. Can probably only - %% happen if the Core Erlang optimizations passes were - %% turned off, so don't bother trying to do something - %% smart here. - {maybe,Fail} - end. - -turn_op('<') -> '>'; -turn_op('>=') -> '=<'; -turn_op('=:='=Op) -> Op; -turn_op('=/='=Op) -> Op. - -negate_op('>=') -> '<'; -negate_op('<') -> '>='; -negate_op('=<') -> '>'; -negate_op('>') -> '=<'; -negate_op('=:=') -> '=/='; -negate_op('=/=') -> '=:='. - -get_literal({atom,Val}) -> - {literal,Val}; -get_literal({integer,Val}) -> - {literal,Val}; -get_literal({float,Val}) -> - {literal,Val}; -get_literal(nil) -> - {literal,[]}; -get_literal({literal,_}=Lit) -> - Lit; -get_literal({_,_}) -> error. - - -%%% -%%% Removing stores to Y registers is not always safe -%%% if there is an instruction that causes an exception -%%% within a catch. In practice, there are few or no -%%% opportunities for removing stores to Y registers anyway -%%% if sys_core_fold has been run. -%%% - -is_killed_at({x,_}=Reg, Lbl, D) -> - beam_utils:is_killed_at(Reg, Lbl, D); -is_killed_at({y,_}, _, _) -> - false. diff --git a/lib/compiler/src/beam_dict.erl b/lib/compiler/src/beam_dict.erl index 990e86062a..b2056332e6 100644 --- a/lib/compiler/src/beam_dict.erl +++ b/lib/compiler/src/beam_dict.erl @@ -126,18 +126,17 @@ import(Mod0, Name0, Arity, #asm{imports=Imp0,next_import=NextIndex}=D0) {NextIndex,D2#asm{imports=Imp,next_import=NextIndex+1}} end. -%% Returns the index for a string in the string table (adding the string to the -%% table if necessary). +%% Returns the index for a binary string in the string table (adding +%% the string to the table if necessary). %% string(String, Dict) -> {Offset, Dict'} --spec string(string(), bdict()) -> {non_neg_integer(), bdict()}. +-spec string(binary(), bdict()) -> {non_neg_integer(), bdict()}. -string(Str, Dict) when is_list(Str) -> +string(BinString, Dict) when is_binary(BinString) -> #asm{strings=Strings,string_offset=NextOffset} = Dict, - StrBin = list_to_binary(Str), - case old_string(StrBin, Strings) of + case old_string(BinString, Strings) of none -> - NewDict = Dict#asm{strings = <<Strings/binary,StrBin/binary>>, - string_offset=NextOffset+byte_size(StrBin)}, + NewDict = Dict#asm{strings = <<Strings/binary,BinString/binary>>, + string_offset=NextOffset+byte_size(BinString)}, {NextOffset,NewDict}; Offset when is_integer(Offset) -> {NextOffset-Offset,Dict} diff --git a/lib/compiler/src/beam_disasm.erl b/lib/compiler/src/beam_disasm.erl index 6cee9acae4..7d048716e4 100644 --- a/lib/compiler/src/beam_disasm.erl +++ b/lib/compiler/src/beam_disasm.erl @@ -373,6 +373,8 @@ disasm_instr(B, Bs, Atoms, Literals) -> disasm_map_inst(get_map_elements, Arity, Bs, Atoms, Literals); has_map_fields -> disasm_map_inst(has_map_fields, Arity, Bs, Atoms, Literals); + put_tuple2 -> + disasm_put_tuple2(Bs, Atoms, Literals); _ -> try decode_n_args(Arity, Bs, Atoms, Literals) of {Args, RestBs} -> @@ -413,6 +415,14 @@ disasm_map_inst(Inst, Arity, Bs0, Atoms, Literals) -> {List, RestBs} = decode_n_args(Len, Bs2, Atoms, Literals), {{Inst, Args ++ [{Z,U,List}]}, RestBs}. +disasm_put_tuple2(Bs, Atoms, Literals) -> + {X, Bs1} = decode_arg(Bs, Atoms, Literals), + {Z, Bs2} = decode_arg(Bs1, Atoms, Literals), + {U, Bs3} = decode_arg(Bs2, Atoms, Literals), + {u, Len} = U, + {List, RestBs} = decode_n_args(Len, Bs3, Atoms, Literals), + {{put_tuple2, [X,{Z,U,List}]}, RestBs}. + %%----------------------------------------------------------------------- %% decode_arg([Byte]) -> {Arg, [Byte]} %% @@ -1095,6 +1105,23 @@ resolve_inst({get_hd,[Src,Dst]},_,_,_) -> resolve_inst({get_tl,[Src,Dst]},_,_,_) -> {get_tl,Src,Dst}; +%% OTP 22 +resolve_inst({bs_start_match3,[Fail,Bin,Live,Dst]},_,_,_) -> + {bs_start_match3,Fail,Bin,Live,Dst}; +resolve_inst({bs_get_tail,[Src,Dst,Live]},_,_,_) -> + {bs_get_tail,Src,Dst,Live}; +resolve_inst({bs_get_position,[Src,Dst,Live]},_,_,_) -> + {bs_get_position,Src,Dst,Live}; +resolve_inst({bs_set_position,[Src,Dst]},_,_,_) -> + {bs_set_position,Src,Dst}; + +%% +%% OTP 22. +%% +resolve_inst({put_tuple2,[Dst,{{z,1},{u,_},List0}]},_,_,_) -> + List = resolve_args(List0), + {put_tuple2,Dst,{list,List}}; + %% %% Catches instructions that are not yet handled. %% diff --git a/lib/compiler/src/beam_except.erl b/lib/compiler/src/beam_except.erl index 05c0f4fbc7..28c89782c9 100644 --- a/lib/compiler/src/beam_except.erl +++ b/lib/compiler/src/beam_except.erl @@ -31,7 +31,7 @@ %%% erlang:error(function_clause, Args) => jump FuncInfoLabel %%% --import(lists, [reverse/1]). +-import(lists, [reverse/1,reverse/2,seq/2,splitwith/2]). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. @@ -53,7 +53,7 @@ function({function,Name,Arity,CLabel,Is0}) -> -record(st, {lbl :: beam_asm:label(), %func_info label loc :: [_], %location for func_info - arity :: arity() %arity for function + arity :: arity() %arity for function }). function_1(Is0) -> @@ -74,27 +74,33 @@ translate([I|Is], St, Acc) -> translate([], _, Acc) -> reverse(Acc). -translate_1(Ar, I, Is, St, [{line,_}=Line|Acc1]=Acc0) -> - case dig_out(Ar, Acc1) of +translate_1(Ar, I, Is, #st{arity=Arity}=St, [{line,_}=Line|Acc1]=Acc0) -> + case dig_out(Ar, Arity, Acc1) of no -> translate(Is, St, [I|Acc0]); - {yes,{function_clause,Arity},Acc2} -> - case {Line,St} of - {{line,Loc},#st{lbl=Fi,loc=Loc,arity=Arity}} -> + {yes,function_clause,Acc2} -> + case {Is,Line,St} of + {[return|_],{line,Loc},#st{lbl=Fi,loc=Loc}} -> Instr = {jump,{f,Fi}}, translate(Is, St, [Instr|Acc2]); - {_,_} -> - %% This must be "error(function_clause, Args)" in - %% the Erlang source code or a fun. Don't translate. + {_,_,_} -> + %% Not a call_only instruction, or not the same + %% location information as in in the line instruction + %% before the func_info instruction. Not safe + %% to translate to a jump. translate(Is, St, [I|Acc0]) end; {yes,Instr,Acc2} -> translate(Is, St, [Instr,Line|Acc2]) end. -dig_out(Ar, [{kill,_}|Is]) -> - dig_out(Ar, Is); -dig_out(1, [{block,Bl0}|Is]) -> +dig_out(1, _Arity, Is) -> + dig_out(Is); +dig_out(2, Arity, Is) -> + dig_out_fc(Arity, Is); +dig_out(_, _, _) -> no. + +dig_out([{block,Bl0}|Is]) -> case dig_out_block(reverse(Bl0)) of no -> no; {yes,What,[]} -> @@ -102,25 +108,13 @@ dig_out(1, [{block,Bl0}|Is]) -> {yes,What,Bl} -> {yes,What,[{block,Bl}|Is]} end; -dig_out(2, [{block,Bl}|Is]) -> - case dig_out_block_fc(Bl) of - no -> no; - {yes,What} -> {yes,What,Is} - end; -dig_out(_, _) -> no. +dig_out(_) -> no. dig_out_block([{set,[{x,0}],[{atom,if_clause}],move}]) -> {yes,if_end,[]}; dig_out_block([{set,[{x,0}],[{literal,{Exc,Value}}],move}|Is]) -> translate_exception(Exc, {literal,Value}, Is, 0); -dig_out_block([{set,[{x,0}],[Tuple],move}, - {set,[],[Value],put}, - {set,[],[{atom,Exc}],put}, - {set,[Tuple],[],{put_tuple,2}}|Is]) -> - translate_exception(Exc, Value, Is, 3); -dig_out_block([{set,[],[Value],put}, - {set,[],[{atom,Exc}],put}, - {set,[{x,0}],[],{put_tuple,2}}|Is]) -> +dig_out_block([{set,[{x,0}],[{atom,Exc},Value],put_tuple2}|Is]) -> translate_exception(Exc, Value, Is, 3); dig_out_block(_) -> no. @@ -138,23 +132,113 @@ fix_block(Is, Words) -> reverse(fix_block_1(Is, Words)). fix_block_1([{set,[],[],{alloc,Live,{F1,F2,Needed0,F3}}}|Is], Words) -> - Needed = Needed0 - Words, - true = Needed >= 0, %Assertion. - [{set,[],[],{alloc,Live,{F1,F2,Needed,F3}}}|Is]; + case Needed0 - Words of + 0 -> + Is; + Needed -> + true = Needed >= 0, %Assertion. + [{set,[],[],{alloc,Live,{F1,F2,Needed,F3}}}|Is] + end; fix_block_1([I|Is], Words) -> [I|fix_block_1(Is, Words)]. -dig_out_block_fc([{set,[],[],{alloc,Live,_}}|Bl]) -> - case dig_out_fc(Bl, Live-1, nil) of - no -> - no; - yes -> - {yes,{function_clause,Live}} + +dig_out_fc(Arity, Is0) -> + Regs0 = maps:from_list([{{x,X},{arg,X}} || X <- seq(0, Arity-1)]), + {Is,Acc0} = splitwith(fun({label,_}) -> false; + ({test,_,_,_}) -> false; + (_) -> true + end, Is0), + {Regs,Acc} = dig_out_fc_1(reverse(Is), Regs0, Acc0), + case Regs of + #{{x,0}:={atom,function_clause},{x,1}:=Args} -> + case moves_from_stack(Args, 0, []) of + {Moves,Arity} -> + {yes,function_clause,reverse(Moves, Acc)}; + {_,_} -> + no + end; + #{} -> + no + end. + +dig_out_fc_1([{block,Bl}|Is], Regs0, Acc) -> + Regs = dig_out_fc_block(Bl, Regs0), + dig_out_fc_1(Is, Regs, Acc); +dig_out_fc_1([{bs_set_position,_,_}=I|Is], Regs, Acc) -> + dig_out_fc_1(Is, Regs, [I|Acc]); +dig_out_fc_1([{bs_get_tail,Src,Dst,Live0}|Is], Regs0, Acc) -> + Regs = prune_xregs(Live0, Regs0), + Live = dig_out_stack_live(Regs, Live0), + I = {bs_get_tail,Src,Dst,Live}, + dig_out_fc_1(Is, Regs, [I|Acc]); +dig_out_fc_1([_|_], _Regs, _Acc) -> + {#{},[]}; +dig_out_fc_1([], Regs, Acc) -> + {Regs,Acc}. + +dig_out_fc_block([{set,[],[],{alloc,Live,_}}|Is], Regs0) -> + Regs = prune_xregs(Live, Regs0), + dig_out_fc_block(Is, Regs); +dig_out_fc_block([{set,[Dst],[Hd,Tl],put_list}|Is], Regs0) -> + Regs = Regs0#{Dst=>{cons,get_reg(Hd, Regs0),get_reg(Tl, Regs0)}}, + dig_out_fc_block(Is, Regs); +dig_out_fc_block([{set,[Dst],[Src],move}|Is], Regs0) -> + Regs = Regs0#{Dst=>get_reg(Src, Regs0)}, + dig_out_fc_block(Is, Regs); +dig_out_fc_block([{set,_,_,_}|_], _Regs) -> + %% Unknown instruction. Fail. + #{}; +dig_out_fc_block([], Regs) -> Regs. + +dig_out_stack_live(Regs, Default) -> + Reg = {x,2}, + case Regs of + #{Reg:=List} -> + dig_out_stack_live_1(List, Default); + #{} -> + Default + end. + +dig_out_stack_live_1({cons,{arg,N},T}, Live) -> + dig_out_stack_live_1(T, max(N + 1, Live)); +dig_out_stack_live_1({cons,_,T}, Live) -> + dig_out_stack_live_1(T, Live); +dig_out_stack_live_1(nil, Live) -> + Live; +dig_out_stack_live_1(_, Live) -> Live. + +prune_xregs(Live, Regs) -> + maps:filter(fun({x,X}, _) -> X < Live end, Regs). + +moves_from_stack({cons,{arg,N},_}, I, _Acc) when N =/= I -> + %% Wrong argument. Give up. + {[],-1}; +moves_from_stack({cons,H,T}, I, Acc) -> + case H of + {arg,I} -> + moves_from_stack(T, I+1, Acc); + _ -> + moves_from_stack(T, I+1, [{move,H,{x,I}}|Acc]) end; -dig_out_block_fc(_) -> no. +moves_from_stack(nil, I, Acc) -> + {reverse(Acc),I}; +moves_from_stack({literal,[H|T]}, I, Acc) -> + Cons = {cons,tag_literal(H),tag_literal(T)}, + moves_from_stack(Cons, I, Acc); +moves_from_stack(_, _, _) -> + %% Not understood. Give up. + {[],-1}. + + +get_reg(R, Regs) -> + case Regs of + #{R:=Val} -> Val; + #{} -> R + end. -dig_out_fc([{set,[Dst],[{x,Reg},Dst0],put_list}|Is], Reg, Dst0) -> - dig_out_fc(Is, Reg-1, Dst); -dig_out_fc([{set,[{x,0}],[{atom,function_clause}],move}], -1, {x,1}) -> - yes; -dig_out_fc(_, _, _) -> no. +tag_literal([]) -> nil; +tag_literal(T) when is_atom(T) -> {atom,T}; +tag_literal(T) when is_float(T) -> {float,T}; +tag_literal(T) when is_integer(T) -> {integer,T}; +tag_literal(T) -> {literal,T}. diff --git a/lib/compiler/src/beam_flatten.erl b/lib/compiler/src/beam_flatten.erl index 20bd23a912..3e6bc1b1ed 100644 --- a/lib/compiler/src/beam_flatten.erl +++ b/lib/compiler/src/beam_flatten.erl @@ -32,8 +32,7 @@ module({Mod,Exp,Attr,Fs,Lc}, _Opt) -> {ok,{Mod,Exp,Attr,[function(F) || F <- Fs],Lc}}. function({function,Name,Arity,CLabel,Is0}) -> - Is1 = block(Is0), - Is = opt(Is1), + Is = block(Is0), {function,Name,Arity,CLabel,Is}. block(Is) -> @@ -44,18 +43,11 @@ block([I|Is], Acc) -> block(Is, [I|Acc]); block([], Acc) -> reverse(Acc). norm_block([{set,[],[],{alloc,R,Alloc}}|Is], Acc0) -> - case insert_alloc_in_bs_init(Acc0, Alloc) of - impossible -> - norm_block(Is, reverse(norm_allocate(Alloc, R), Acc0)); - Acc -> - norm_block(Is, Acc) - end; -norm_block([{set,[D1],[S],get_hd},{set,[D2],[S],get_tl}|Is], Acc) -> - I = {get_list,S,D1,D2}, - norm_block(Is, [I|Acc]); -norm_block([I|Is], Acc) -> norm_block(Is, [norm(I)|Acc]); + norm_block(Is, reverse(norm_allocate(Alloc, R), Acc0)); +norm_block([I|Is], Acc) -> + norm_block(Is, [norm(I)|Acc]); norm_block([], Acc) -> Acc. - + norm({set,[D],As,{bif,N,F}}) -> {bif,N,F,As,D}; norm({set,[D],As,{alloc,R,{gc_bif,N,F}}}) -> {gc_bif,N,F,R,As,D}; norm({set,[D],[],init}) -> {init,D}; @@ -63,6 +55,7 @@ norm({set,[D],[S],move}) -> {move,S,D}; norm({set,[D],[S],fmove}) -> {fmove,S,D}; norm({set,[D],[S],fconv}) -> {fconv,S,D}; norm({set,[D],[S1,S2],put_list}) -> {put_list,S1,S2,D}; +norm({set,[D],Els,put_tuple2}) -> {put_tuple2,D,{list,Els}}; norm({set,[D],[],{put_tuple,A}}) -> {put_tuple,A,D}; norm({set,[],[S],put}) -> {put,S}; norm({set,[D],[S],{get_tuple_element,I}}) -> {get_tuple_element,S,I,D}; @@ -88,57 +81,3 @@ norm_allocate({nozero,Ns,0,Inits}, Regs) -> [{allocate,Ns,Regs}|Inits]; norm_allocate({nozero,Ns,Nh,Inits}, Regs) -> [{allocate_heap,Ns,Nh,Regs}|Inits]. - -%% insert_alloc_in_bs_init(ReverseInstructionStream, AllocationInfo) -> -%% impossible | ReverseInstructionStream' -%% A bs_init/6 instruction should not be followed by a test heap instruction. -%% Given the AllocationInfo from a test heap instruction, merge the -%% allocation amounts into the previous bs_init/6 instruction (if any). -%% -insert_alloc_in_bs_init([{bs_put,_,_,_}=I|Is], Alloc) -> - %% The instruction sequence ends with an bs_put/4 instruction. - %% We'll need to search backwards for the bs_init/6 instruction. - insert_alloc_1(Is, Alloc, [I]); -insert_alloc_in_bs_init(_, _) -> impossible. - -insert_alloc_1([{bs_init=Op,Fail,Info0,Live,Ss,Dst}|Is], - {_,nostack,Ws2,[]}, Acc) when is_integer(Live) -> - %% The number of extra heap words is always in the second position - %% in the Info tuple. - Ws1 = element(2, Info0), - Al = beam_utils:combine_heap_needs(Ws1, Ws2), - Info = setelement(2, Info0, Al), - I = {Op,Fail,Info,Live,Ss,Dst}, - reverse(Acc, [I|Is]); -insert_alloc_1([{bs_put,_,_,_}=I|Is], Alloc, Acc) -> - insert_alloc_1(Is, Alloc, [I|Acc]). - -%% opt(Is0) -> Is -%% Simple peep-hole optimization to move a {move,Any,{x,0}} past -%% any kill up to the next call instruction. (To give the loader -%% an opportunity to combine the 'move' and the 'call' instructions.) -%% -opt(Is) -> - opt_1(Is, []). - -opt_1([{move,_,{x,0}}=I|Is0], Acc0) -> - case move_past_kill(Is0, I, Acc0) of - impossible -> opt_1(Is0, [I|Acc0]); - {Is,Acc} -> opt_1(Is, Acc) - end; -opt_1([I|Is], Acc) -> - opt_1(Is, [I|Acc]); -opt_1([], Acc) -> reverse(Acc). - -move_past_kill([{kill,Src}|_], {move,Src,_}, _) -> - impossible; -move_past_kill([{kill,_}=I|Is], Move, Acc) -> - move_past_kill(Is, Move, [I|Acc]); -move_past_kill([{trim,N,_}=I|Is], {move,Src,Dst}=Move, Acc) -> - case Src of - {y,Y} when Y < N-> impossible; - {y,Y} -> {Is,[{move,{y,Y-N},Dst},I|Acc]}; - _ -> {Is,[Move,I|Acc]} - end; -move_past_kill(Is, Move, Acc) -> - {Is,[Move|Acc]}. diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 22974da398..74f80ca70e 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -22,7 +22,7 @@ -module(beam_jump). -export([module/2, - is_unreachable_after/1,is_exit_instruction/1, + is_exit_instruction/1, remove_unused_labels/1]). %%% The following optimisations are done: @@ -101,6 +101,10 @@ %%% always keep the label. (beam_clean will remove any unused %%% labels.) %%% +%%% (7) Replace a jump to a return instruction with a return instruction. +%%% Similarly, replace a jump to deallocate + return with those +%%% instructions. +%%% %%% Note: This modules depends on (almost) all branches and jumps only %%% going forward, so that we can remove instructions (including definition %%% of labels) after any label that has not been referenced by the code @@ -128,27 +132,136 @@ %%% on the program state. %%% --import(lists, [dropwhile/2,reverse/1,reverse/2,foldl/3]). +-import(lists, [foldl/3,mapfoldl/3,reverse/1,reverse/2]). -type instruction() :: beam_utils:instruction(). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. -module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> - Fs = [function(F) || F <- Fs0], +module({Mod,Exp,Attr,Fs0,Lc0}, _Opt) -> + {Fs,Lc} = mapfoldl(fun function/2, Lc0, Fs0), {ok,{Mod,Exp,Attr,Fs,Lc}}. %% function(Function) -> Function' %% Optimize jumps and branches. %% %% NOTE: This function assumes that there are no labels inside blocks. -function({function,Name,Arity,CLabel,Asm0}) -> - Asm1 = share(Asm0), - Asm2 = move(Asm1), - Asm3 = opt(Asm2, CLabel), - Asm = remove_unused_labels(Asm3), - {function,Name,Arity,CLabel,Asm}. +function({function,Name,Arity,CLabel,Asm0}, Lc0) -> + try + Asm1 = eliminate_moves(Asm0), + {Asm2,Lc} = insert_labels(Asm1, Lc0, []), + Asm3 = share(Asm2), + Asm4 = move(Asm3), + Asm5 = opt(Asm4, CLabel), + Asm6 = unshare(Asm5), + Asm = remove_unused_labels(Asm6), + {{function,Name,Arity,CLabel,Asm},Lc} + catch + Class:Error:Stack -> + io:fwrite("Function: ~w/~w\n", [Name,Arity]), + erlang:raise(Class, Error, Stack) + end. + +%%% +%%% Scan instructions in execution order and remove redundant 'move' +%%% instructions. 'move' instructions are redundant if we know that +%%% the register already contains the value being assigned, as in the +%%% following code: +%%% +%%% select_val Register FailLabel [... Literal => L1...] +%%% . +%%% . +%%% . +%%% L1: move Literal Register +%%% + +eliminate_moves(Is) -> + eliminate_moves(Is, #{}, []). + +eliminate_moves([{select,select_val,Reg,{f,Fail},List}=I|Is], D0, Acc) -> + D1 = add_unsafe_label(Fail, D0), + D = update_value_dict(List, Reg, D1), + eliminate_moves(Is, D, [I|Acc]); +eliminate_moves([{test,is_eq_exact,_,[Reg,Val]}=I, + {block,BlkIs0}|Is], D0, Acc) -> + D = update_unsafe_labels(I, D0), + RegVal = {Reg,Val}, + BlkIs = eliminate_moves_blk(BlkIs0, RegVal), + eliminate_moves([{block,BlkIs}|Is], D, [I|Acc]); +eliminate_moves([{label,Lbl},{block,BlkIs0}=Blk|Is], D, Acc0) -> + Acc = [{label,Lbl}|Acc0], + case {no_fallthrough(Acc0),D} of + {true,#{Lbl:={_,_}=RegVal}} -> + BlkIs = eliminate_moves_blk(BlkIs0, RegVal), + eliminate_moves([{block,BlkIs}|Is], D, Acc); + {_,_} -> + eliminate_moves([Blk|Is], D, Acc) + end; +eliminate_moves([{block,[]}|Is], D, Acc) -> + %% Empty blocks can prevent further jump optimizations. + eliminate_moves(Is, D, Acc); +eliminate_moves([I|Is], D0, Acc) -> + D = update_unsafe_labels(I, D0), + eliminate_moves(Is, D, [I|Acc]); +eliminate_moves([], _, Acc) -> reverse(Acc). + +eliminate_moves_blk([{set,[Dst],[_],move}|_]=Is, {_,Dst}) -> + Is; +eliminate_moves_blk([{set,[Dst],[Lit],move}|Is], {Dst,Lit}) -> + %% Remove redundant 'move' instruction. + Is; +eliminate_moves_blk([{set,[Dst],[_],move}|_]=Is, {Dst,_}) -> + Is; +eliminate_moves_blk([{set,[_],[_],move}=I|Is], {_,_}=RegVal) -> + [I|eliminate_moves_blk(Is, RegVal)]; +eliminate_moves_blk(Is, _) -> Is. + +no_fallthrough([I|_]) -> + is_unreachable_after(I). + +update_value_dict([Lit,{f,Lbl}|T], Reg, D0) -> + D = case D0 of + #{Lbl:=unsafe} -> D0; + #{Lbl:={Reg,Lit}} -> D0; + #{Lbl:=_} -> D0#{Lbl:=unsafe}; + #{} -> D0#{Lbl=>{Reg,Lit}} + end, + update_value_dict(T, Reg, D); +update_value_dict([], _, D) -> D. + +add_unsafe_label(L, D) -> + D#{L=>unsafe}. + +update_unsafe_labels(I, D) -> + Ls = instr_labels(I), + update_unsafe_labels_1(Ls, D). + +update_unsafe_labels_1([L|Ls], D) -> + update_unsafe_labels_1(Ls, D#{L=>unsafe}); +update_unsafe_labels_1([], D) -> D. + +%%% +%%% It seems to be useful to insert extra labels after certain +%%% test instructions. This used to be done by beam_dead. +%%% + +insert_labels([{test,Op,_,_}=I|Is], Lc, Acc) -> + Useful = case Op of + is_lt -> true; + is_ge -> true; + is_eq_exact -> true; + is_ne_exact -> true; + _ -> false + end, + case Useful of + false -> insert_labels(Is, Lc, [I|Acc]); + true -> insert_labels(Is, Lc+1, [{label,Lc},I|Acc]) + end; +insert_labels([I|Is], Lc, Acc) -> + insert_labels(Is, Lc, [I|Acc]); +insert_labels([], Lc, Acc) -> + {reverse(Acc),Lc}. %%% %%% (1) We try to share the code for identical code segments by replacing all @@ -156,41 +269,51 @@ function({function,Name,Arity,CLabel,Asm0}) -> %%% share(Is0) -> - %% We will get more sharing if we never fall through to a label. - Is = eliminate_fallthroughs(Is0, []), - share_1(Is, #{}, [], []). + Is1 = eliminate_fallthroughs(Is0, []), + Is2 = find_fixpoint(fun(Is) -> + share_1(Is, #{}, #{}, [], []) + end, Is1), + reverse(Is2). -share_1([{label,L}=Lbl|Is], Dict0, [_|_]=Seq, Acc) -> +share_1([{label,L}=Lbl|Is], Dict0, Lbls0, [_|_]=Seq, Acc) -> case maps:find(Seq, Dict0) of - error -> - Dict = maps:put(Seq, L, Dict0), - share_1(Is, Dict, [], [Lbl|Seq ++ Acc]); - {ok,Label} -> - share_1(Is, Dict0, [], [Lbl,{jump,{f,Label}}|Acc]) + error -> + Dict = maps:put(Seq, L, Dict0), + share_1(Is, Dict, Lbls0, [], [[Lbl|Seq]|Acc]); + {ok,Label} -> + Lbls = maps:put(L, Label, Lbls0), + share_1(Is, Dict0, Lbls, [], [[Lbl,{jump,{f,Label}}]|Acc]) end; -share_1([{func_info,_,_,_}=I|Is], _, [], Acc) -> - reverse(Is, [I|Acc]); -share_1([{'catch',_,_}=I|Is], Dict0, Seq, Acc) -> - Dict = clean_non_sharable(Dict0), - share_1(Is, Dict, [I|Seq], Acc); -share_1([{'try',_,_}=I|Is], Dict0, Seq, Acc) -> - Dict = clean_non_sharable(Dict0), - share_1(Is, Dict, [I|Seq], Acc); -share_1([{try_case,_}=I|Is], Dict0, Seq, Acc) -> - Dict = clean_non_sharable(Dict0), - share_1(Is, Dict, [I|Seq], Acc); -share_1([{catch_end,_}=I|Is], Dict0, Seq, Acc) -> - Dict = clean_non_sharable(Dict0), - share_1(Is, Dict, [I|Seq], Acc); -share_1([I|Is], Dict, Seq, Acc) -> +share_1([{func_info,_,_,_}|_]=Is0, _, Lbls, [], Acc0) when Lbls =/= #{} -> + lists:foldl(fun(Is, Acc) -> + beam_utils:replace_labels(Is, Acc, Lbls, fun(Old) -> Old end) + end, Is0, Acc0); +share_1([{func_info,_,_,_}|_]=Is, _, Lbls, [], Acc) when Lbls =:= #{} -> + lists:foldl(fun lists:reverse/2, Is, Acc); +share_1([{'catch',_,_}=I|Is], Dict0, Lbls0, Seq, Acc) -> + {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0), + share_1(Is, Dict, Lbls, [I|Seq], Acc); +share_1([{'try',_,_}=I|Is], Dict0, Lbls0, Seq, Acc) -> + {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0), + share_1(Is, Dict, Lbls, [I|Seq], Acc); +share_1([{try_case,_}=I|Is], Dict0, Lbls0, Seq, Acc) -> + {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0), + share_1(Is, Dict, Lbls, [I|Seq], Acc); +share_1([{catch_end,_}=I|Is], Dict0, Lbls0, Seq, Acc) -> + {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0), + share_1(Is, Dict, Lbls, [I|Seq], Acc); +share_1([{jump,{f,To}}=I,{label,L}=Lbl|Is], Dict0, Lbls0, _Seq, Acc) -> + Lbls = maps:put(L, To, Lbls0), + share_1(Is, Dict0, Lbls, [], [[Lbl,I]|Acc]); +share_1([I|Is], Dict, Lbls, Seq, Acc) -> case is_unreachable_after(I) of false -> - share_1(Is, Dict, [I|Seq], Acc); + share_1(Is, Dict, Lbls, [I|Seq], Acc); true -> - share_1(Is, Dict, [I], Acc) + share_1(Is, Dict, Lbls, [I], Acc) end. -clean_non_sharable(Dict) -> +clean_non_sharable(Dict0, Lbls0) -> %% We are passing in or out of a 'catch' or 'try' block. Remove %% sequences that should not be shared over the boundaries of the %% block. Since the end of the sequence must match, the only @@ -198,7 +321,17 @@ clean_non_sharable(Dict) -> %% the 'catch'/'try' block is a sequence that ends with an %% instruction that causes an exception. Any sequence that causes %% an exception must contain a line/1 instruction. - maps:filter(fun(K, _V) -> sharable_with_try(K) end, Dict). + Dict1 = maps:to_list(Dict0), + Lbls1 = maps:to_list(Lbls0), + {Dict2,Lbls2} = foldl(fun({K, V}, {Dict,Lbls}) -> + case sharable_with_try(K) of + true -> + {[{K,V}|Dict],lists:keydelete(V, 2, Lbls)}; + false -> + {Dict,Lbls} + end + end, {[],Lbls1}, Dict1), + {maps:from_list(Dict2),maps:from_list(Lbls2)}. sharable_with_try([{line,_}|_]) -> %% This sequence may cause an exception and may potentially @@ -251,8 +384,6 @@ extract_seq([{line,_}=Line|Is], Acc) -> extract_seq(Is, [Line|Acc]); extract_seq([{block,_}=Bl|Is], Acc) -> extract_seq_1(Is, [Bl|Acc]); -extract_seq([{bs_context_to_binary,_}=I|Is], Acc) -> - extract_seq_1(Is, [I|Acc]); extract_seq([{label,_}|_]=Is, Acc) -> extract_seq_1(Is, Acc); extract_seq(_, _) -> no. @@ -276,14 +407,13 @@ extract_seq_1(_, _) -> no. { entry :: beam_asm:label(), %Entry label (must not be moved). replace :: #{beam_asm:label() := beam_asm:label()}, %Labels to replace. - labels :: cerl_sets:set(), %Set of referenced labels. - index :: beam_utils:code_index() | {lazy,[beam_utils:instruction()]} %Index built lazily only if needed + labels :: cerl_sets:set() %Set of referenced labels. }). opt(Is0, CLabel) -> find_fixpoint(fun(Is) -> Lbls = initial_labels(Is), - St = #st{entry=CLabel,replace=#{},labels=Lbls,index={lazy,Is}}, + St = #st{entry=CLabel,replace=#{},labels=Lbls}, opt(Is, [], St) end, Is0). @@ -293,7 +423,7 @@ find_fixpoint(OptFun, Is0) -> Is -> find_fixpoint(OptFun, Is) end. -opt([{test,_,{f,L}=Lbl,_}=I|[{jump,{f,L}}|_]=Is], Acc0, St0) -> +opt([{test,_,{f,L}=Lbl,_}=I|[{jump,{f,L}}|_]=Is], Acc, St) -> %% We have %% Test Label Ops %% jump Label @@ -302,23 +432,10 @@ opt([{test,_,{f,L}=Lbl,_}=I|[{jump,{f,L}}|_]=Is], Acc0, St0) -> case beam_utils:is_pure_test(I) of false -> %% Test is not pure; we must keep it. - opt(Is, [I|Acc0], label_used(Lbl, St0)); + opt(Is, [I|Acc], label_used(Lbl, St)); true -> %% The test is pure and its failure label is the same %% as in the jump that follows -- thus it is not needed. - %% Check if any of the previous instructions could also be eliminated. - {Acc,St} = opt_useless_loads(Acc0, L, St0), - opt(Is, Acc, St) - end; -opt([{test,_,{f,L}=Lbl,_}=I|[{label,L}|_]=Is], Acc0, St0) -> - %% Similar to the above, except we have a fall-through rather than jump - %% Test Label Ops - %% label Label - case beam_utils:is_pure_test(I) of - false -> - opt(Is, [I|Acc0], label_used(Lbl, St0)); - true -> - {Acc,St} = opt_useless_loads(Acc0, L, St0), opt(Is, Acc, St) end; opt([{test,Test0,{f,L}=Lbl,Ops}=I|[{jump,To}|Is]=Is0], Acc, St) -> @@ -385,51 +502,6 @@ normalize_replace([{From,To0}|Rest], Replace, Acc) -> normalize_replace([], _Replace, Acc) -> maps:from_list(Acc). -%% After eliminating a test, it might happen, that a register was only used -%% in this test. Let's check if that was the case and if it was so, we can -%% eliminate the load into the register completely. -opt_useless_loads([{block,_}|_]=Is, L, #st{index={lazy,FIs}}=St) -> - opt_useless_loads(Is, L, St#st{index=beam_utils:index_labels(FIs)}); -opt_useless_loads([{block,Block0}|Is], L, #st{index=Index}=St) -> - case opt_useless_block_loads(Block0, L, Index) of - [] -> - opt_useless_loads(Is, L, St); - [_|_]=Block -> - {[{block,Block}|Is],St} - end; -%% After eliminating the test and useless blocks, it might happen, -%% that the previous test could also be eliminated. -%% It might be that the label was already marked as used, even if ultimately, -%% it never will be - we can't do much about it at that point, though -opt_useless_loads([{test,_,{f,L},_}=I|Is], L, St) -> - case beam_utils:is_pure_test(I) of - false -> - {[I|Is],St}; - true -> - opt_useless_loads(Is, L, St) - end; -opt_useless_loads(Is, _L, St) -> - {Is,St}. - -opt_useless_block_loads([{set,[Dst],_,_}=I|Is0], L, Index) -> - BlockJump = [{block,Is0},{jump,{f,L}}], - case beam_utils:is_killed(Dst, BlockJump, Index) of - true -> - %% The register is killed and not used, we can remove the load. - %% Remove any `put` instructions in case we just - %% removed a `put_tuple` instruction. - Is = dropwhile(fun({set,_,_,put}) -> true; - (_) -> false - end, Is0), - opt_useless_block_loads(Is, L, Index); - false -> - [I|opt_useless_block_loads(Is0, L, Index)] - end; -opt_useless_block_loads([I|Is], L, Index) -> - [I|opt_useless_block_loads(Is, L, Index)]; -opt_useless_block_loads([], _L, _Index) -> - []. - collect_labels(Is, Label, #st{entry=Entry,replace=Replace} = St) -> collect_labels_1(Is, Label, Entry, Replace, St). @@ -556,52 +628,109 @@ drop_upto_label([{label,_}|_]=Is) -> Is; drop_upto_label([_|Is]) -> drop_upto_label(Is); drop_upto_label([]) -> []. -%% ulbl(Instruction, UsedGbSet) -> UsedGbSet' -%% Update the gb_set UsedGbSet with any function-local labels +%% unshare([Instruction]) -> [Instruction]. +%% Replace a jump to a return sequence (a `return` instruction +%% optionally preced by a `deallocate` instruction) with the return +%% sequence. This always saves execution time and may also save code +%% space (depending on the architecture). Eliminating `jump` +%% instructions also gives beam_trim more opportunities to trim the +%% stack. + +unshare(Is) -> + Short = unshare_collect_short(Is, #{}), + unshare_short(Is, Short). + +unshare_collect_short([{label,L},return|Is], Map) -> + unshare_collect_short(Is, Map#{L=>[return]}); +unshare_collect_short([{label,L},{deallocate,_}=D,return|Is], Map) -> + %% `deallocate` and `return` are combined into one instruction by + %% the loader. + unshare_collect_short(Is, Map#{L=>[D,return]}); +unshare_collect_short([_|Is], Map) -> + unshare_collect_short(Is, Map); +unshare_collect_short([], Map) -> Map. + +unshare_short([{jump,{f,F}}=I|Is], Map) -> + case Map of + #{F:=Seq} -> + Seq ++ unshare_short(Is, Map); + #{} -> + [I|unshare_short(Is, Map)] + end; +unshare_short([I|Is], Map) -> + [I|unshare_short(Is, Map)]; +unshare_short([], _Map) -> []. + +%% ulbl(Instruction, UsedCerlSet) -> UsedCerlSet' +%% Update the cerl_set UsedCerlSet with any function-local labels %% (i.e. not with labels in call instructions) referenced by %% the instruction Instruction. %% %% NOTE: This function does NOT look for labels inside blocks. -ulbl({test,_,Fail,_}, Used) -> - mark_used(Fail, Used); -ulbl({test,_,Fail,_,_,_}, Used) -> - mark_used(Fail, Used); -ulbl({select,_,_,Fail,Vls}, Used) -> - mark_used_list(Vls, mark_used(Fail, Used)); -ulbl({'try',_,Lbl}, Used) -> - mark_used(Lbl, Used); -ulbl({'catch',_,Lbl}, Used) -> - mark_used(Lbl, Used); -ulbl({jump,Lbl}, Used) -> - mark_used(Lbl, Used); -ulbl({loop_rec,Lbl,_}, Used) -> - mark_used(Lbl, Used); -ulbl({loop_rec_end,Lbl}, Used) -> - mark_used(Lbl, Used); -ulbl({wait,Lbl}, Used) -> - mark_used(Lbl, Used); -ulbl({wait_timeout,Lbl,_To}, Used) -> - mark_used(Lbl, Used); -ulbl({bif,_Name,Lbl,_As,_R}, Used) -> - mark_used(Lbl, Used); -ulbl({gc_bif,_Name,Lbl,_Live,_As,_R}, Used) -> - mark_used(Lbl, Used); -ulbl({bs_init,Lbl,_,_,_,_}, Used) -> - mark_used(Lbl, Used); -ulbl({bs_put,Lbl,_,_}, Used) -> - mark_used(Lbl, Used); -ulbl({put_map,Lbl,_Op,_Src,_Dst,_Live,_List}, Used) -> - mark_used(Lbl, Used); -ulbl({get_map_elements,Lbl,_Src,_List}, Used) -> - mark_used(Lbl, Used); -ulbl(_, Used) -> Used. - -mark_used({f,0}, Used) -> Used; -mark_used({f,L}, Used) -> cerl_sets:add_element(L, Used). - -mark_used_list([{f,L}|T], Used) -> - mark_used_list(T, cerl_sets:add_element(L, Used)); -mark_used_list([_|T], Used) -> - mark_used_list(T, Used); -mark_used_list([], Used) -> Used. +ulbl(I, Used) -> + case instr_labels(I) of + [] -> + Used; + [Lbl] -> + cerl_sets:add_element(Lbl, Used); + [_|_]=L -> + ulbl_list(L, Used) + end. + +ulbl_list([L|Ls], Used) -> + ulbl_list(Ls, cerl_sets:add_element(L, Used)); +ulbl_list([], Used) -> Used. + +-spec instr_labels(Instruction) -> Labels when + Instruction :: instruction(), + Labels :: [beam_asm:label()]. + +instr_labels({test,_,Fail,_}) -> + do_instr_labels(Fail); +instr_labels({test,_,Fail,_,_,_}) -> + do_instr_labels(Fail); +instr_labels({select,_,_,Fail,Vls}) -> + do_instr_labels_list(Vls, do_instr_labels(Fail)); +instr_labels({'try',_,Lbl}) -> + do_instr_labels(Lbl); +instr_labels({'catch',_,Lbl}) -> + do_instr_labels(Lbl); +instr_labels({jump,Lbl}) -> + do_instr_labels(Lbl); +instr_labels({loop_rec,Lbl,_}) -> + do_instr_labels(Lbl); +instr_labels({loop_rec_end,Lbl}) -> + do_instr_labels(Lbl); +instr_labels({wait,Lbl}) -> + do_instr_labels(Lbl); +instr_labels({wait_timeout,Lbl,_To}) -> + do_instr_labels(Lbl); +instr_labels({bif,_Name,Lbl,_As,_R}) -> + do_instr_labels(Lbl); +instr_labels({gc_bif,_Name,Lbl,_Live,_As,_R}) -> + do_instr_labels(Lbl); +instr_labels({bs_init,Lbl,_,_,_,_}) -> + do_instr_labels(Lbl); +instr_labels({bs_put,Lbl,_,_}) -> + do_instr_labels(Lbl); +instr_labels({put_map,Lbl,_Op,_Src,_Dst,_Live,_List}) -> + do_instr_labels(Lbl); +instr_labels({get_map_elements,Lbl,_Src,_List}) -> + do_instr_labels(Lbl); +instr_labels({recv_mark,Lbl}) -> + do_instr_labels(Lbl); +instr_labels({recv_set,Lbl}) -> + do_instr_labels(Lbl); +instr_labels({fcheckerror,Lbl}) -> + do_instr_labels(Lbl); +instr_labels(_) -> []. + +do_instr_labels({f,0}) -> []; +do_instr_labels({f,F}) -> [F]. + +do_instr_labels_list([{f,F}|T], Acc) -> + do_instr_labels_list(T, [F|Acc]); +do_instr_labels_list([_|T], Acc) -> + do_instr_labels_list(T, Acc); +do_instr_labels_list([], Acc) -> Acc. diff --git a/lib/compiler/src/beam_kernel_to_ssa.erl b/lib/compiler/src/beam_kernel_to_ssa.erl new file mode 100644 index 0000000000..df95749fb3 --- /dev/null +++ b/lib/compiler/src/beam_kernel_to_ssa.erl @@ -0,0 +1,1312 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% Purpose: Convert the Kernel Erlang format to the SSA format. + +-module(beam_kernel_to_ssa). + +%% The main interface. +-export([module/2]). + +-import(lists, [append/1,duplicate/2,flatmap/2,foldl/3, + keysort/2,mapfoldl/3,map/2,member/2, + reverse/1,reverse/2,sort/1]). + +-include("v3_kernel.hrl"). +-include("beam_ssa.hrl"). + +-type label() :: beam_ssa:label(). + +%% Main codegen structure. +-record(cg, {lcount=1 :: label(), %Label counter + bfail=1 :: label(), + catch_label=none :: 'none' | label(), + vars=#{} :: map(), %Defined variables. + break=0 :: label(), %Break label + recv=0 :: label(), %Receive label + ultimate_failure=0 :: label() %Label for ultimate match failure. + }). + +%% Internal records. +-record(cg_break, {args :: [beam_ssa:value()], + phi :: label() + }). +-record(cg_phi, {vars :: [beam_ssa:b_var()] + }). +-record(cg_unreachable, {}). + +-spec module(#k_mdef{}, [compile:option()]) -> {'ok',#b_module{}}. + +module(#k_mdef{name=Mod,exports=Es,attributes=Attr,body=Forms}, _Opts) -> + Body = functions(Forms, Mod), + Module = #b_module{name=Mod,exports=Es,attributes=Attr,body=Body}, + {ok,Module}. + +functions(Forms, Mod) -> + [function(F, Mod) || F <- Forms]. + +function(#k_fdef{anno=Anno0,func=Name,arity=Arity, + vars=As0,body=Kb}, Mod) -> + try + #k_match{} = Kb, %Assertion. + + %% Generate the SSA form immediate format. + St0 = #cg{}, + {As,St1} = new_ssa_vars(As0, St0), + {Asm,St} = cg_fun(Kb, St1), + Anno1 = line_anno(Anno0), + Anno = Anno1#{func_info=>{Mod,Name,Arity}}, + #b_function{anno=Anno,args=As,bs=Asm,cnt=St#cg.lcount} + catch + Class:Error:Stack -> + io:fwrite("Function: ~w/~w\n", [Name,Arity]), + erlang:raise(Class, Error, Stack) + end. + +%% cg_fun([Lkexpr], [HeadVar], State) -> {[Ainstr],State} + +cg_fun(Ke, St0) -> + {UltimateFail,FailIs,St1} = make_failure(badarg, St0), + St2 = St1#cg{bfail=UltimateFail,ultimate_failure=UltimateFail}, + {B,St} = cg(Ke, St2), + Asm = [{label,0}|B++FailIs], + finalize(Asm, St). + +make_failure(Reason, St0) -> + {Lbl,St1} = new_label(St0), + {Dst,St} = new_ssa_var('@ssa_ret', St1), + Is = [{label,Lbl}, + #b_set{op=call,dst=Dst, + args=[#b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=error}, + arity=1}, + #b_literal{val=Reason}]}, + #b_ret{arg=Dst}], + {Lbl,Is,St}. + +%% cg(Lkexpr, State) -> {[Ainstr],State}. +%% Generate code for a kexpr. + +cg(#k_match{body=M,ret=Rs}, St) -> + do_match_cg(M, Rs, St); +cg(#k_guard_match{body=M,ret=Rs}, St) -> + do_match_cg(M, Rs, St); +cg(#k_seq{arg=Arg,body=Body}, St0) -> + {ArgIs,St1} = cg(Arg, St0), + {BodyIs,St} = cg(Body, St1), + {ArgIs++BodyIs,St}; +cg(#k_call{anno=Le,op=Func,args=As,ret=Rs}, St) -> + call_cg(Func, As, Rs, Le, St); +cg(#k_enter{anno=Le,op=Func,args=As}, St) -> + enter_cg(Func, As, Le, St); +cg(#k_bif{anno=Le}=Bif, St) -> + bif_cg(Bif, Le, St); +cg(#k_try{arg=Ta,vars=Vs,body=Tb,evars=Evs,handler=Th,ret=Rs}, St) -> + try_cg(Ta, Vs, Tb, Evs, Th, Rs, St); +cg(#k_try_enter{arg=Ta,vars=Vs,body=Tb,evars=Evs,handler=Th}, St) -> + try_enter_cg(Ta, Vs, Tb, Evs, Th, St); +cg(#k_catch{body=Cb,ret=[R]}, St) -> + do_catch_cg(Cb, R, St); +cg(#k_receive{anno=Le,timeout=Te,var=Rvar,body=Rm,action=Tes,ret=Rs}, St) -> + recv_loop_cg(Te, Rvar, Rm, Tes, Rs, Le, St); +cg(#k_receive_next{}, #cg{recv=Recv}=St) -> + Is = [#b_set{op=recv_next},make_uncond_branch(Recv)], + {Is,St}; +cg(#k_receive_accept{}, St) -> + Remove = #b_set{op=remove_message}, + {[Remove],St}; +cg(#k_put{anno=Le,arg=Con,ret=Var}, St) -> + put_cg(Var, Con, Le, St); +cg(#k_return{args=[Ret0]}, St) -> + Ret = ssa_arg(Ret0, St), + {[#b_ret{arg=Ret}],St}; +cg(#k_break{args=Bs}, #cg{break=Br}=St) -> + Args = ssa_args(Bs, St), + {[#cg_break{args=Args,phi=Br}],St}; +cg(#k_guard_break{args=Bs}, St) -> + cg(#k_break{args=Bs}, St). + +%% match_cg(Matc, [Ret], State) -> {[Ainstr],State}. +%% Generate code for a match. + +do_match_cg(M, Rs, St0) -> + {B,St1} = new_label(St0), + {Mis,St2} = match_cg(M, St1#cg.bfail, St1#cg{break=B}), + {BreakVars,St} = new_ssa_vars(Rs, St2), + {Mis ++ [{label,B},#cg_phi{vars=BreakVars}], + St#cg{bfail=St0#cg.bfail,break=St1#cg.break}}. + +%% match_cg(Match, Fail, State) -> {[Ainstr],State}. +%% Generate code for a match tree. + +match_cg(#k_alt{first=F,then=S}, Fail, St0) -> + {Tf,St1} = new_label(St0), + {Fis,St2} = match_cg(F, Tf, St1), + {Sis,St3} = match_cg(S, Fail, St2), + {Fis ++ [{label,Tf}] ++ Sis,St3}; +match_cg(#k_select{var=#k_var{}=V,types=Scs}, Fail, St) -> + match_fmf(fun (S, F, Sta) -> + select_cg(S, V, F, Fail, Sta) + end, Fail, St, Scs); +match_cg(#k_guard{clauses=Gcs}, Fail, St) -> + match_fmf(fun (G, F, Sta) -> + guard_clause_cg(G, F, Sta) + end, Fail, St, Gcs); +match_cg(Ke, _Fail, St0) -> + cg(Ke, St0). + +%% select_cg(Sclause, V, TypeFail, ValueFail, State) -> {Is,State}. +%% Selecting type and value needs two failure labels, TypeFail is the +%% label to jump to of the next type test when this type fails, and +%% ValueFail is the label when this type is correct but the value is +%% wrong. These are different as in the second case there is no need +%% to try the next type, it will always fail. + +select_cg(#k_type_clause{type=k_binary,values=[S]}, Var, Tf, Vf, St) -> + select_binary(S, Var, Tf, Vf, St); +select_cg(#k_type_clause{type=k_bin_seg,values=Vs}, Var, Tf, _Vf, St) -> + select_bin_segs(Vs, Var, Tf, St); +select_cg(#k_type_clause{type=k_bin_int,values=Vs}, Var, Tf, _Vf, St) -> + select_bin_segs(Vs, Var, Tf, St); +select_cg(#k_type_clause{type=k_bin_end,values=[S]}, Var, Tf, _Vf, St) -> + select_bin_end(S, Var, Tf, St); +select_cg(#k_type_clause{type=k_map,values=Vs}, Var, Tf, Vf, St) -> + select_map(Vs, Var, Tf, Vf, St); +select_cg(#k_type_clause{type=k_cons,values=[S]}, Var, Tf, Vf, St) -> + select_cons(S, Var, Tf, Vf, St); +select_cg(#k_type_clause{type=k_nil,values=[S]}, Var, Tf, Vf, St) -> + select_nil(S, Var, Tf, Vf, St); +select_cg(#k_type_clause{type=k_literal,values=Vs}, Var, Tf, Vf, St) -> + select_literal(Vs, Var, Tf, Vf, St); +select_cg(#k_type_clause{type=Type,values=Scs}, Var, Tf, Vf, St0) -> + {Vis,St1} = + mapfoldl(fun (S, Sta) -> + {Val,Is,Stb} = select_val(S, Var, Vf, Sta), + {{Is,[Val]},Stb} + end, St0, Scs), + OptVls = combine(lists:sort(combine(Vis))), + {Vls,Sis,St2} = select_labels(OptVls, St1, [], []), + Arg = ssa_arg(Var, St2), + {Is,St} = select_val_cg(Type, Arg, Vls, Tf, Vf, Sis, St2), + {Is,St}. + +select_val_cg(k_tuple, Tuple, Vls, Tf, Vf, Sis, St0) -> + {Is0,St1} = make_cond_branch({bif,is_tuple}, [Tuple], Tf, St0), + {Arity,St2} = new_ssa_var('@ssa_arity', St1), + GetArity = #b_set{op={bif,tuple_size},dst=Arity,args=[Tuple]}, + {Is,St} = select_val_cg(k_int, Arity, Vls, Vf, Vf, Sis, St2), + {Is0++[GetArity]++Is,St}; +select_val_cg(Type, R, Vls, Tf, Vf, Sis, St0) -> + {TypeIs,St1} = if + Tf =:= Vf -> + %% The type and value failure labels are the same; + %% we don't need a type test. + {[],St0}; + true -> + %% Different labels for type failure and + %% label failure; we need a type test. + Test = select_type_test(Type), + make_cond_branch(Test, [R], Tf, St0) + end, + case Vls of + [{Val,Succ}] -> + {Is,St} = make_cond({bif,'=:='}, [R,Val], Vf, Succ, St1), + {TypeIs++Is++Sis,St}; + [_|_] -> + {TypeIs++[#b_switch{arg=R,fail=Vf,list=Vls}|Sis],St1} + end. + +select_type_test(k_int) -> {bif,is_integer}; +select_type_test(k_atom) -> {bif,is_atom}; +select_type_test(k_float) -> {bif,is_float}. + +combine([{Is,Vs1},{Is,Vs2}|Vis]) -> combine([{Is,Vs1 ++ Vs2}|Vis]); +combine([V|Vis]) -> [V|combine(Vis)]; +combine([]) -> []. + +select_labels([{Is,Vs}|Vis], St0, Vls, Sis) -> + {Lbl,St1} = new_label(St0), + select_labels(Vis, St1, add_vls(Vs, Lbl, Vls), [[{label,Lbl}|Is]|Sis]); +select_labels([], St, Vls, Sis) -> + {Vls,append(Sis),St}. + +add_vls([V|Vs], Lbl, Acc) -> + add_vls(Vs, Lbl, [{#b_literal{val=V},Lbl}|Acc]); +add_vls([], _, Acc) -> Acc. + +select_literal(S, V, Tf, Vf, St) -> + Src = ssa_arg(V, St), + F = fun(ValClause, Fail, St0) -> + {Val,ValIs,St1} = select_val(ValClause, V, Vf, St0), + Args = [Src,#b_literal{val=Val}], + {Is,St2} = make_cond_branch({bif,'=:='}, Args, Fail, St1), + {Is++ValIs,St2} + end, + match_fmf(F, Tf, St, S). + +select_cons(#k_val_clause{val=#k_cons{hd=Hd,tl=Tl},body=B}, + V, Tf, Vf, St0) -> + Es = [Hd,Tl], + {Eis,St1} = select_extract_cons(V, Es, St0), + {Bis,St2} = match_cg(B, Vf, St1), + Src = ssa_arg(V, St2), + {Is,St} = make_cond_branch(is_nonempty_list, [Src], Tf, St2), + {Is ++ Eis ++ Bis,St}. + +select_nil(#k_val_clause{val=#k_nil{},body=B}, V, Tf, Vf, St0) -> + {Bis,St1} = match_cg(B, Vf, St0), + Src = ssa_arg(V, St1), + {Is,St} = make_cond_branch({bif,'=:='}, [Src,#b_literal{val=[]}], Tf, St1), + {Is ++ Bis,St}. + +select_binary(#k_val_clause{val=#k_binary{segs=#k_var{name=Ctx0}},body=B}, + #k_var{}=Src, Tf, Vf, St0) -> + {Ctx,St1} = new_ssa_var(Ctx0, St0), + {Bis0,St2} = match_cg(B, Vf, St1), + {TestIs,St} = make_cond_branch(succeeded, [Ctx], Tf, St2), + Bis1 = [#b_set{op=bs_start_match,dst=Ctx, + args=[ssa_arg(Src, St)]}] ++ TestIs ++ Bis0, + Bis = finish_bs_matching(Bis1), + {Bis,St}. + +finish_bs_matching([#b_set{op=bs_match, + args=[#b_literal{val=string},Ctx,#b_literal{val=BinList}]}=Set|Is]) + when is_list(BinList) -> + I = Set#b_set{args=[#b_literal{val=string},Ctx, + #b_literal{val=list_to_bitstring(BinList)}]}, + finish_bs_matching([I|Is]); +finish_bs_matching([I|Is]) -> + [I|finish_bs_matching(Is)]; +finish_bs_matching([]) -> []. + +make_cond(Cond, Args, Fail, Succ, St0) -> + {Bool,St} = new_ssa_var('@ssa_bool', St0), + Bif = #b_set{op=Cond,dst=Bool,args=Args}, + Br = #b_br{bool=Bool,succ=Succ,fail=Fail}, + {[Bif,Br],St}. + +make_cond_branch(Cond, Args, Fail, St0) -> + {Bool,St1} = new_ssa_var('@ssa_bool', St0), + {Succ,St} = new_label(St1), + Bif = #b_set{op=Cond,dst=Bool,args=Args}, + Br = #b_br{bool=Bool,succ=Succ,fail=Fail}, + {[Bif,Br,{label,Succ}],St}. + +make_uncond_branch(Fail) -> + #b_br{bool=#b_literal{val=true},succ=Fail,fail=Fail}. + +%% Instructions for selection of binary segments. + +select_bin_segs(Scs, Ivar, Tf, St) -> + match_fmf(fun(S, Fail, Sta) -> + select_bin_seg(S, Ivar, Fail, Sta) + end, Tf, St, Scs). + +select_bin_seg(#k_val_clause{val=#k_bin_seg{size=Size,unit=U,type=T, + seg=Seg,flags=Fs,next=Next}, + body=B,anno=Anno}, + #k_var{}=Src, Fail, St0) -> + LineAnno = line_anno(Anno), + Ctx = get_context(Src, St0), + {Mis,St1} = select_extract_bin(Next, Size, U, T, Fs, Fail, + Ctx, LineAnno, St0), + {Extracted,St2} = new_ssa_var(Seg#k_var.name, St1), + {Bis,St} = match_cg(B, Fail, St2), + BsGet = #b_set{op=bs_extract,dst=Extracted,args=[ssa_arg(Next, St)]}, + Is = Mis ++ [BsGet] ++ Bis, + {Is,St}; +select_bin_seg(#k_val_clause{val=#k_bin_int{size=Sz,unit=U,flags=Fs, + val=Val,next=Next}, + body=B}, + #k_var{}=Src, Fail, St0) -> + Ctx = get_context(Src, St0), + {Mis,St1} = select_extract_int(Next, Val, Sz, U, Fs, Fail, + Ctx, St0), + {Bis,St} = match_cg(B, Fail, St1), + Is = case Mis ++ Bis of + [#b_set{op=bs_match,args=[#b_literal{val=string},OtherCtx1,Bin1]}, + #b_set{op=succeeded,dst=Bool1}, + #b_br{bool=Bool1,succ=Succ,fail=Fail}, + {label,Succ}, + #b_set{op=bs_match,dst=Dst,args=[#b_literal{val=string},_OtherCtx2,Bin2]}| + [#b_set{op=succeeded,dst=Bool2}, + #b_br{bool=Bool2,fail=Fail}|_]=Is0] -> + %% We used to do this optimization later, but it + %% turns out that in huge functions with many + %% string matching instructions, it's a huge win + %% to do the combination now. To avoid copying the + %% binary data again and again, we'll combine bitstrings + %% in a list and convert all of it to a bitstring later. + {#b_literal{val=B1},#b_literal{val=B2}} = {Bin1,Bin2}, + Bin = #b_literal{val=[B1,B2]}, + Set = #b_set{op=bs_match,dst=Dst,args=[#b_literal{val=string},OtherCtx1,Bin]}, + [Set|Is0]; + Is0 -> + Is0 + end, + {Is,St}. + +get_context(#k_var{}=Var, St) -> + ssa_arg(Var, St). + +select_bin_end(#k_val_clause{val=#k_bin_end{},body=B}, Src, Tf, St0) -> + Ctx = get_context(Src, St0), + {Bis,St1} = match_cg(B, Tf, St0), + {TestIs,St} = make_cond_branch(bs_test_tail, [Ctx,#b_literal{val=0}], Tf, St1), + Is = TestIs++Bis, + {Is,St}. + +select_extract_bin(#k_var{name=Hd}, Size0, Unit, Type, Flags, Vf, + Ctx, Anno, St0) -> + {Dst,St1} = new_ssa_var(Hd, St0), + Size = ssa_arg(Size0, St0), + build_bs_instr(Anno, Type, Vf, Ctx, Size, Unit, Flags, Dst, St1). + +select_extract_int(#k_var{name=Tl}, 0, #k_int{val=0}, _U, _Fs, _Vf, + Ctx, St0) -> + St = set_ssa_var(Tl, Ctx, St0), + {[],St}; +select_extract_int(#k_var{name=Tl}, Val, #k_int{val=Sz}, U, Fs, Vf, + Ctx, St0) -> + {Dst,St1} = new_ssa_var(Tl, St0), + Bits = U*Sz, + Bin = case member(big, Fs) of + true -> + <<Val:Bits>>; + false -> + true = member(little, Fs), %Assertion. + <<Val:Bits/little>> + end, + Bits = bit_size(Bin), %Assertion. + {TestIs,St} = make_cond_branch(succeeded, [Dst], Vf, St1), + Set = #b_set{op=bs_match,dst=Dst, + args=[#b_literal{val=string},Ctx,#b_literal{val=Bin}]}, + {[Set|TestIs],St}. + +build_bs_instr(Anno, Type, Fail, Ctx, Size, Unit0, Flags0, Dst, St0) -> + Unit = #b_literal{val=Unit0}, + Flags = #b_literal{val=Flags0}, + NeedSize = bs_need_size(Type), + TypeArg = #b_literal{val=Type}, + Get = case NeedSize of + true -> + #b_set{anno=Anno,op=bs_match,dst=Dst, + args=[TypeArg,Ctx,Flags,Size,Unit]}; + false -> + #b_set{anno=Anno,op=bs_match,dst=Dst, + args=[TypeArg,Ctx,Flags]} + end, + {Is,St} = make_cond_branch(succeeded, [Dst], Fail, St0), + {[Get|Is],St}. + +select_val(#k_val_clause{val=#k_tuple{es=Es},body=B}, V, Vf, St0) -> + #k{us=Used} = k_get_anno(B), + {Eis,St1} = select_extract_tuple(V, Es, Used, St0), + {Bis,St2} = match_cg(B, Vf, St1), + {length(Es),Eis ++ Bis,St2}; +select_val(#k_val_clause{val=Val0,body=B}, _V, Vf, St0) -> + Val = case Val0 of + #k_atom{val=Lit} -> Lit; + #k_float{val=Lit} -> Lit; + #k_int{val=Lit} -> Lit; + #k_literal{val=Lit} -> Lit + end, + {Bis,St1} = match_cg(B, Vf, St0), + {Val,Bis,St1}. + +%% select_extract_tuple(Src, [V], State) -> {[E],State}. +%% Extract tuple elements, but only if they are actually used. +%% +%% Not extracting tuple elements that are not used is an +%% optimization for compile time and memory use during compilation. +%% It is probably worthwhile because it is common to extract only a +%% few elements from a huge record. + +select_extract_tuple(Src, Vs, Used, St0) -> + Tuple = ssa_arg(Src, St0), + F = fun (#k_var{name=V}, {Elem,S0}) -> + case member(V, Used) of + true -> + Args = [Tuple,#b_literal{val=Elem}], + {Dst,S} = new_ssa_var(V, S0), + Get = #b_set{op=get_tuple_element,dst=Dst,args=Args}, + {[Get],{Elem+1,S}}; + false -> + {[],{Elem+1,S0}} + end + end, + {Es,{_,St}} = flatmapfoldl(F, {0,St0}, Vs), + {Es,St}. + +select_map(Scs, V, Tf, Vf, St0) -> + MapSrc = ssa_arg(V, St0), + {Is,St1} = + match_fmf(fun(#k_val_clause{val=#k_map{op=exact,es=Es}, + body=B}, Fail, St1) -> + select_map_val(V, Es, B, Fail, St1) + end, Vf, St0, Scs), + {TestIs,St} = make_cond_branch({bif,is_map}, [MapSrc], Tf, St1), + {TestIs++Is,St}. + +select_map_val(V, Es, B, Fail, St0) -> + {Eis,St1} = select_extract_map(Es, V, Fail, St0), + {Bis,St2} = match_cg(B, Fail, St1), + {Eis++Bis,St2}. + +select_extract_map([P|Ps], Src, Fail, St0) -> + MapSrc = ssa_arg(Src, St0), + #k_map_pair{key=Key0,val=#k_var{name=Dst0}} = P, + Key = ssa_arg(Key0, St0), + {Dst,St1} = new_ssa_var(Dst0, St0), + Set = #b_set{op=get_map_element,dst=Dst,args=[MapSrc,Key]}, + {TestIs,St2} = make_cond_branch(succeeded, [Dst], Fail, St1), + {Is,St} = select_extract_map(Ps, Src, Fail, St2), + {[Set|TestIs]++Is,St}; +select_extract_map([], _, _, St) -> + {[],St}. + +select_extract_cons(Src0, [#k_var{name=Hd},#k_var{name=Tl}], St0) -> + Src = ssa_arg(Src0, St0), + {HdDst,St1} = new_ssa_var(Hd, St0), + {TlDst,St2} = new_ssa_var(Tl, St1), + GetHd = #b_set{op=get_hd,dst=HdDst,args=[Src]}, + GetTl = #b_set{op=get_tl,dst=TlDst,args=[Src]}, + {[GetHd,GetTl],St2}. + +guard_clause_cg(#k_guard_clause{guard=G,body=B}, Fail, St0) -> + {Gis,St1} = guard_cg(G, Fail, St0), + {Bis,St} = match_cg(B, Fail, St1), + {Gis ++ Bis,St}. + +%% guard_cg(Guard, Fail, State) -> {[Ainstr],State}. +%% A guard is a boolean expression of tests. Tests return true or +%% false. A fault in a test causes the test to return false. Tests +%% never return the boolean, instead we generate jump code to go to +%% the correct exit point. Primops and tests all go to the next +%% instruction on success or jump to a failure label. + +guard_cg(#k_protected{arg=Ts,ret=Rs,inner=Inner}, Fail, St) -> + protected_cg(Ts, Rs, Inner, Fail, St); +guard_cg(#k_test{op=Test0,args=As,inverted=Inverted}, Fail, St0) -> + #k_remote{mod=#k_atom{val=erlang},name=#k_atom{val=Test}} = Test0, + test_cg(Test, Inverted, As, Fail, St0); +guard_cg(#k_seq{arg=Arg,body=Body}, Fail, St0) -> + {ArgIs,St1} = guard_cg(Arg, Fail, St0), + {BodyIs,St} = guard_cg(Body, Fail, St1), + {ArgIs++BodyIs,St}; +guard_cg(G, _Fail, St) -> + cg(G, St). + +test_cg('=/=', Inverted, As, Fail, St) -> + test_cg('=:=', not Inverted, As, Fail, St); +test_cg('/=', Inverted, As, Fail, St) -> + test_cg('==', not Inverted, As, Fail, St); +test_cg(Test, Inverted, As0, Fail, St0) -> + As = ssa_args(As0, St0), + case {Test,ssa_args(As0, St0)} of + {is_record,[Tuple,#b_literal{val=Atom}=Tag,#b_literal{val=Int}=Arity]} + when is_atom(Atom), is_integer(Int) -> + test_is_record_cg(Inverted, Fail, Tuple, Tag, Arity, St0); + {_,As} -> + {Bool,St1} = new_ssa_var('@ssa_bool', St0), + {Succ,St} = new_label(St1), + Bif = #b_set{op={bif,Test},dst=Bool,args=As}, + Br = case Inverted of + false -> #b_br{bool=Bool,succ=Succ,fail=Fail}; + true -> #b_br{bool=Bool,succ=Fail,fail=Succ} + end, + {[Bif,Br,{label,Succ}],St} + end. + +test_is_record_cg(false, Fail, Tuple, TagVal, ArityVal, St0) -> + {Arity,St1} = new_ssa_var('@ssa_arity', St0), + {Tag,St2} = new_ssa_var('@ssa_tag', St1), + {Is0,St3} = make_cond_branch({bif,is_tuple}, [Tuple], Fail, St2), + GetArity = #b_set{op={bif,tuple_size},dst=Arity,args=[Tuple]}, + {Is1,St4} = make_cond_branch({bif,'=:='}, [Arity,ArityVal], Fail, St3), + GetTag = #b_set{op=get_tuple_element,dst=Tag, + args=[Tuple,#b_literal{val=0}]}, + {Is2,St} = make_cond_branch({bif,'=:='}, [Tag,TagVal], Fail, St4), + Is = Is0 ++ [GetArity] ++ Is1 ++ [GetTag] ++ Is2, + {Is,St}; +test_is_record_cg(true, Fail, Tuple, TagVal, ArityVal, St0) -> + {Succ,St1} = new_label(St0), + {Arity,St2} = new_ssa_var('@ssa_arity', St1), + {Tag,St3} = new_ssa_var('@ssa_tag', St2), + {Is0,St4} = make_cond_branch({bif,is_tuple}, [Tuple], Succ, St3), + GetArity = #b_set{op={bif,tuple_size},dst=Arity,args=[Tuple]}, + {Is1,St5} = make_cond_branch({bif,'=:='}, [Arity,ArityVal], Succ, St4), + GetTag = #b_set{op=get_tuple_element,dst=Tag, + args=[Tuple,#b_literal{val=0}]}, + {Is2,St} = make_cond_branch({bif,'=:='}, [Tag,TagVal], Succ, St5), + Is3 = [make_uncond_branch(Fail),{label,Succ}], + Is = Is0 ++ [GetArity] ++ Is1 ++ [GetTag] ++ Is2 ++ Is3, + {Is,St}. + +%% protected_cg([Kexpr], [Ret], Fail, St) -> {[Ainstr],St}. +%% Do a protected. Protecteds without return values are just done +%% for effect, the return value is not checked, success passes on to +%% the next instruction and failure jumps to Fail. If there are +%% return values then these must be set to 'false' on failure, +%% control always passes to the next instruction. + +protected_cg(Ts, [], _, Fail, St0) -> + %% Protect these calls, revert when done. + {Tis,St1} = guard_cg(Ts, Fail, St0#cg{bfail=Fail}), + {Tis,St1#cg{bfail=St0#cg.bfail}}; +protected_cg(Ts, Rs, Inner0, _Fail, St0) -> + {Pfail,St1} = new_label(St0), + {Br,St2} = new_label(St1), + Prot = duplicate(length(Rs), #b_literal{val=false}), + {Tis,St3} = guard_cg(Ts, Pfail, St2#cg{break=Pfail,bfail=Pfail}), + Inner = ssa_args(Inner0, St3), + {BreakVars,St} = new_ssa_vars(Rs, St3), + Is = Tis ++ [#cg_break{args=Inner,phi=Br}, + {label,Pfail},#cg_break{args=Prot,phi=Br}, + {label,Br},#cg_phi{vars=BreakVars}], + {Is,St#cg{break=St0#cg.break,bfail=St0#cg.bfail}}. + +%% match_fmf(Fun, LastFail, State, [Clause]) -> {Is,State}. +%% This is a special flatmapfoldl for match code gen where we +%% generate a "failure" label for each clause. The last clause uses +%% an externally generated failure label, LastFail. N.B. We do not +%% know or care how the failure labels are used. + +match_fmf(F, LastFail, St, [H]) -> + F(H, LastFail, St); +match_fmf(F, LastFail, St0, [H|T]) -> + {Fail,St1} = new_label(St0), + {R,St2} = F(H, Fail, St1), + {Rs,St3} = match_fmf(F, LastFail, St2, T), + {R ++ [{label,Fail}] ++ Rs,St3}. + +%% fail_label(State) -> {Where,FailureLabel}. +%% Where = guard | no_catch | in_catch +%% Return an indication of which part of a function code is +%% being generated for and the appropriate failure label to +%% use. +%% +%% Where has the following meaning: +%% +%% guard - Inside a guard. +%% no_catch - In a function body, not in the scope of +%% a try/catch or catch. +%% in_catch - In the scope of a try/catch or catch. + +fail_label(#cg{catch_label=Catch,bfail=Fail,ultimate_failure=Ult}) -> + if + Fail =/= Ult -> + {guard,Fail}; + Catch =:= none -> + {no_catch,Fail}; + is_integer(Catch) -> + {in_catch,Catch} + end. + +%% bif_fail_label(State) -> FailureLabel. +%% Return the appropriate failure label for a guard BIF call or +%% primop that fails. + +bif_fail_label(St) -> + {_,Fail} = fail_label(St), + Fail. + +%% call_cg(Func, [Arg], [Ret], Le, State) -> +%% {[Ainstr],State}. +%% enter_cg(Func, [Arg], Le, St) -> {[Ainstr],St}. +%% Generate code for call and enter. + +call_cg(Func, As, [], Le, St) -> + call_cg(Func, As, [#k_var{name='@ssa_ignored'}], Le, St); +call_cg(Func0, As, [#k_var{name=R}|MoreRs]=Rs, Le, St0) -> + case fail_label(St0) of + {guard,Fail} -> + %% Inside a guard. The only allowed function call is to + %% erlang:error/1,2. We will generate a branch to the + %% failure branch. + #k_remote{mod=#k_atom{val=erlang}, + name=#k_atom{val=error}} = Func0, %Assertion. + [#k_var{name=DestVar}] = Rs, + St = set_ssa_var(DestVar, #b_literal{val=unused}, St0), + {[make_uncond_branch(Fail),#cg_unreachable{}],St}; + {Catch,Fail} -> + %% Ordinary function call in a function body. + Args = ssa_args(As, St0), + {Ret,St1} = new_ssa_var(R, St0), + Func = call_target(Func0, Args, St0), + Call = #b_set{anno=line_anno(Le),op=call,dst=Ret,args=[Func|Args]}, + + %% If this is a call to erlang:error(), MoreRs could be a + %% nonempty list of variables that each need a value. + St2 = foldl(fun(#k_var{name=Dummy}, S) -> + set_ssa_var(Dummy, #b_literal{val=unused}, S) + end, St1, MoreRs), + case Catch of + no_catch -> + {[Call],St2}; + in_catch -> + {TestIs,St} = make_cond_branch(succeeded, [Ret], Fail, St2), + {[Call|TestIs],St} + end + end. + +enter_cg(Func0, As0, Le, St0) -> + Anno = line_anno(Le), + Func = call_target(Func0, As0, St0), + As = ssa_args(As0, St0), + {Ret,St} = new_ssa_var('@ssa_ret', St0), + Call = #b_set{anno=Anno,op=call,dst=Ret,args=[Func|As]}, + {[Call,#b_ret{arg=Ret}],St}. + +call_target(Func, As, St) -> + Arity = length(As), + case Func of + #k_remote{mod=Mod0,name=Name0} -> + Mod = ssa_arg(Mod0, St), + Name = ssa_arg(Name0, St), + #b_remote{mod=Mod,name=Name,arity=Arity}; + #k_local{name=Name} when is_atom(Name) -> + #b_local{name=#b_literal{val=Name},arity=Arity}; + #k_var{}=Var -> + ssa_arg(Var, St) + end. + +%% bif_cg(#k_bif{}, Le,State) -> {[Ainstr],State}. +%% Generate code for a guard BIF or primop. + +bif_cg(#k_bif{op=#k_internal{name=Name},args=As,ret=Rs}, Le, St) -> + internal_cg(Name, As, Rs, Le, St); +bif_cg(#k_bif{op=#k_remote{mod=#k_atom{val=erlang},name=#k_atom{val=Name}}, + args=As,ret=Rs}, Le, St) -> + bif_cg(Name, As, Rs, Le, St). + +%% internal_cg(Bif, [Arg], [Ret], Le, State) -> +%% {[Ainstr],State}. + +internal_cg(make_fun, [Name0,Arity0|As], Rs, _Le, St0) -> + #k_atom{val=Name} = Name0, + #k_int{val=Arity} = Arity0, + [#k_var{name=Dst0}] = Rs, + {Dst,St} = new_ssa_var(Dst0, St0), + Args = ssa_args(As, St), + Local = #b_local{name=#b_literal{val=Name},arity=Arity}, + MakeFun = #b_set{op=make_fun,dst=Dst,args=[Local|Args]}, + {[MakeFun],St}; +internal_cg(bs_init_writable=I, As, [#k_var{name=Dst0}], _Le, St0) -> + %% This behaves like a function call. + {Dst,St} = new_ssa_var(Dst0, St0), + Args = ssa_args(As, St), + Set = #b_set{op=I,dst=Dst,args=Args}, + {[Set],St}; +internal_cg(build_stacktrace=I, As, [#k_var{name=Dst0}], _Le, St0) -> + {Dst,St} = new_ssa_var(Dst0, St0), + Args = ssa_args(As, St), + Set = #b_set{op=I,dst=Dst,args=Args}, + {[Set],St}; +internal_cg(raise, As, [#k_var{name=Dst0}], _Le, St0) -> + Args = ssa_args(As, St0), + {Dst,St} = new_ssa_var(Dst0, St0), + Resume = #b_set{op=resume,dst=Dst,args=Args}, + case St of + #cg{catch_label=none} -> + {[Resume],St}; + #cg{catch_label=Catch} when is_integer(Catch) -> + Is = [Resume,make_uncond_branch(Catch),#cg_unreachable{}], + {Is,St} + end; +internal_cg(raw_raise=I, As, [#k_var{name=Dst0}], _Le, St0) -> + %% This behaves like a function call. + {Dst,St} = new_ssa_var(Dst0, St0), + Args = ssa_args(As, St), + Set = #b_set{op=I,dst=Dst,args=Args}, + {[Set],St}. + +bif_cg(Bif, As0, [#k_var{name=Dst0}], Le, St0) -> + {Dst,St1} = new_ssa_var(Dst0, St0), + case {Bif,ssa_args(As0, St0)} of + {is_record,[Tuple,#b_literal{val=Atom}=Tag, + #b_literal{val=Int}=Arity]} + when is_atom(Atom), is_integer(Int) -> + bif_is_record_cg(Dst, Tuple, Tag, Arity, St1); + {_,As} -> + I = #b_set{anno=line_anno(Le),op={bif,Bif},dst=Dst,args=As}, + case erl_bifs:is_safe(erlang, Bif, length(As)) of + false -> + Fail = bif_fail_label(St1), + {Is,St} = make_cond_branch(succeeded, [Dst], Fail, St1), + {[I|Is],St}; + true-> + {[I],St1} + end + end. + +bif_is_record_cg(Dst, Tuple, TagVal, ArityVal, St0) -> + {Arity,St1} = new_ssa_var('@ssa_arity', St0), + {Tag,St2} = new_ssa_var('@ssa_tag', St1), + {Phi,St3} = new_label(St2), + {False,St4} = new_label(St3), + {Is0,St5} = make_cond_branch({bif,is_tuple}, [Tuple], False, St4), + GetArity = #b_set{op={bif,tuple_size},dst=Arity,args=[Tuple]}, + {Is1,St6} = make_cond_branch({bif,'=:='}, [Arity,ArityVal], False, St5), + GetTag = #b_set{op=get_tuple_element,dst=Tag, + args=[Tuple,#b_literal{val=0}]}, + {Is2,St} = make_cond_branch({bif,'=:='}, [Tag,TagVal], False, St6), + Is3 = [#cg_break{args=[#b_literal{val=true}],phi=Phi}, + {label,False}, + #cg_break{args=[#b_literal{val=false}],phi=Phi}, + {label,Phi}, + #cg_phi{vars=[Dst]}], + Is = Is0 ++ [GetArity] ++ Is1 ++ [GetTag] ++ Is2 ++ Is3, + {Is,St}. + +%% recv_loop_cg(TimeOut, ReceiveVar, ReceiveMatch, TimeOutExprs, +%% [Ret], Le, St) -> {[Ainstr],St}. + +recv_loop_cg(Te, Rvar, Rm, Tes, Rs, Le, St0) -> + %% Get labels. + {Rl,St1} = new_label(St0), + {Tl,St2} = new_label(St1), + {Bl,St3} = new_label(St2), + St4 = St3#cg{break=Bl,recv=Rl}, + {Ris,St5} = cg_recv_mesg(Rvar, Rm, Tl, Le, St4), + {Wis,St6} = cg_recv_wait(Te, Tes, St5), + {BreakVars,St} = new_ssa_vars(Rs, St6), + {Ris ++ [{label,Tl}] ++ Wis ++ + [{label,Bl},#cg_phi{vars=BreakVars}], + St#cg{break=St0#cg.break,recv=St0#cg.recv}}. + +%% cg_recv_mesg( ) -> {[Ainstr],St}. + +cg_recv_mesg(#k_var{name=R}, Rm, Tl, Le, St0) -> + {Dst,St1} = new_ssa_var(R, St0), + {Mis,St2} = match_cg(Rm, none, St1), + RecvLbl = St1#cg.recv, + {TestIs,St} = make_cond_branch(succeeded, [Dst], Tl, St2), + Is = [#b_br{anno=line_anno(Le),bool=#b_literal{val=true}, + succ=RecvLbl,fail=RecvLbl}, + {label,RecvLbl}, + #b_set{op=peek_message,dst=Dst}|TestIs], + {Is++Mis,St}. + +%% cg_recv_wait(Te, Tes, St) -> {[Ainstr],St}. + +cg_recv_wait(#k_int{val=0}, Es, St0) -> + {Tis,St} = cg(Es, St0), + {[#b_set{op=timeout}|Tis],St}; +cg_recv_wait(Te, Es, St0) -> + {Tis,St1} = cg(Es, St0), + Args = [ssa_arg(Te, St1)], + {WaitDst,St2} = new_ssa_var('@ssa_wait', St1), + {WaitIs,St} = make_cond_branch(succeeded, [WaitDst], St1#cg.recv, St2), + %% Infinite timeout will be optimized later. + Is = [#b_set{op=wait_timeout,dst=WaitDst,args=Args}] ++ WaitIs ++ + [#b_set{op=timeout}] ++ Tis, + {Is,St}. + +%% try_cg(TryBlock, [BodyVar], TryBody, [ExcpVar], TryHandler, [Ret], St) -> +%% {[Ainstr],St}. + +try_cg(Ta, Vs, Tb, Evs, Th, Rs, St0) -> + {B,St1} = new_label(St0), %Body label + {H,St2} = new_label(St1), %Handler label + {E,St3} = new_label(St2), %End label + {Next,St4} = new_label(St3), + {TryTag,St5} = new_ssa_var('@ssa_catch_tag', St4), + {SsaVs,St6} = new_ssa_vars(Vs, St5), + {SsaEvs,St7} = new_ssa_vars(Evs, St6), + {Ais,St8} = cg(Ta, St7#cg{break=B,catch_label=H}), + St9 = St8#cg{break=E,catch_label=St7#cg.catch_label}, + {Bis,St10} = cg(Tb, St9), + {His,St11} = cg(Th, St10), + {BreakVars,St12} = new_ssa_vars(Rs, St11), + {CatchedAgg,St} = new_ssa_var('@ssa_agg', St12), + ExtractVs = extract_vars(SsaEvs, CatchedAgg, 0), + KillTryTag = #b_set{op=kill_try_tag,args=[TryTag]}, + Args = [#b_literal{val='try'},TryTag], + Handler = [{label,H}, + #b_set{op=landingpad,dst=CatchedAgg,args=Args}] ++ + ExtractVs ++ [KillTryTag], + {[#b_set{op=new_try_tag,dst=TryTag,args=[#b_literal{val='try'}]}, + #b_br{bool=TryTag,succ=Next,fail=H}, + {label,Next}] ++ Ais ++ + [{label,B},#cg_phi{vars=SsaVs},KillTryTag] ++ Bis ++ + Handler ++ His ++ + [{label,E},#cg_phi{vars=BreakVars}], + St#cg{break=St0#cg.break}}. + +try_enter_cg(Ta, Vs, Tb, Evs, Th, St0) -> + {B,St1} = new_label(St0), %Body label + {H,St2} = new_label(St1), %Handler label + {Next,St3} = new_label(St2), + {TryTag,St4} = new_ssa_var('@ssa_catch_tag', St3), + {SsaVs,St5} = new_ssa_vars(Vs, St4), + {SsaEvs,St6} = new_ssa_vars(Evs, St5), + {Ais,St7} = cg(Ta, St6#cg{break=B,catch_label=H}), + St8 = St7#cg{catch_label=St6#cg.catch_label}, + {Bis,St9} = cg(Tb, St8), + {His,St10} = cg(Th, St9), + {CatchedAgg,St} = new_ssa_var('@ssa_agg', St10), + ExtractVs = extract_vars(SsaEvs, CatchedAgg, 0), + KillTryTag = #b_set{op=kill_try_tag,args=[TryTag]}, + Args = [#b_literal{val='try'},TryTag], + Handler = [{label,H}, + #b_set{op=landingpad,dst=CatchedAgg,args=Args}] ++ + ExtractVs ++ [KillTryTag], + {[#b_set{op=new_try_tag,dst=TryTag,args=[#b_literal{val='try'}]}, + #b_br{bool=TryTag,succ=Next,fail=H}, + {label,Next}] ++ Ais ++ + [{label,B},#cg_phi{vars=SsaVs},KillTryTag] ++ Bis ++ + Handler ++ His, + St#cg{break=St0#cg.break}}. + +extract_vars([V|Vs], Agg, N) -> + I = #b_set{op=extract,dst=V,args=[Agg,#b_literal{val=N}]}, + [I|extract_vars(Vs, Agg, N+1)]; +extract_vars([], _, _) -> []. + +%% do_catch_cg(CatchBlock, Ret, St) -> {[Ainstr],St}. + +do_catch_cg(Block, #k_var{name=R}, St0) -> + {B,St1} = new_label(St0), + {Next,St2} = new_label(St1), + {H,St3} = new_label(St2), + {CatchReg,St4} = new_ssa_var('@ssa_catch_tag', St3), + {Dst,St5} = new_ssa_var(R, St4), + {Succ,St6} = new_label(St5), + {Cis,St7} = cg(Block, St6#cg{break=Succ,catch_label=H}), + {CatchedVal,St8} = new_ssa_var('@catched_val', St7), + {SuccVal,St9} = new_ssa_var('@success_val', St8), + {CatchedAgg,St10} = new_ssa_var('@ssa_agg', St9), + {CatchEndVal,St} = new_ssa_var('@catch_end_val', St10), + Args = [#b_literal{val='catch'},CatchReg], + {[#b_set{op=new_try_tag,dst=CatchReg,args=[#b_literal{val='catch'}]}, + #b_br{bool=CatchReg,succ=Next,fail=H}, + {label,Next}] ++ Cis ++ + [{label,H}, + #b_set{op=landingpad,dst=CatchedAgg,args=Args}, + #b_set{op=extract,dst=CatchedVal, + args=[CatchedAgg,#b_literal{val=0}]}, + #cg_break{args=[CatchedVal],phi=B}, + {label,Succ}, + #cg_phi{vars=[SuccVal]}, + #cg_break{args=[SuccVal],phi=B}, + {label,B},#cg_phi{vars=[CatchEndVal]}, + #b_set{op=catch_end,dst=Dst,args=[CatchReg,CatchEndVal]}], + St#cg{break=St1#cg.break,catch_label=St1#cg.catch_label}}. + +%% put_cg([Var], Constr, Le, Vdb, Bef, St) -> {[Ainstr],St}. +%% Generate code for constructing terms. + +put_cg([#k_var{name=R}], #k_cons{hd=Hd,tl=Tl}, _Le, St0) -> + Args = ssa_args([Hd,Tl], St0), + {Dst,St} = new_ssa_var(R, St0), + PutList = #b_set{op=put_list,dst=Dst,args=Args}, + {[PutList],St}; +put_cg([#k_var{name=R}], #k_tuple{es=Es}, _Le, St0) -> + {Ret,St} = new_ssa_var(R, St0), + Args = ssa_args(Es, St), + PutTuple = #b_set{op=put_tuple,dst=Ret,args=Args}, + {[PutTuple],St}; +put_cg([#k_var{name=R}], #k_binary{segs=Segs}, Le, St0) -> + Fail = bif_fail_label(St0), + {Dst,St1} = new_ssa_var(R, St0), + cg_binary(Dst, Segs, Fail, Le, St1); +put_cg([#k_var{name=R}], #k_map{op=Op,var=Map, + es=[#k_map_pair{key=#k_var{}=K,val=V}]}, + Le, St0) -> + %% Map: single variable key. + SrcMap = ssa_arg(Map, St0), + LineAnno = line_anno(Le), + List = [ssa_arg(K, St0),ssa_arg(V, St0)], + {Dst,St1} = new_ssa_var(R, St0), + {Is,St} = put_cg_map(LineAnno, Op, SrcMap, Dst, List, St1), + {Is,St}; +put_cg([#k_var{name=R}], #k_map{op=Op,var=Map,es=Es}, Le, St0) -> + %% Map: one or more literal keys. + [] = [Var || #k_map_pair{key=#k_var{}=Var} <- Es], %Assertion + SrcMap = ssa_arg(Map, St0), + LineAnno = line_anno(Le), + List = flatmap(fun(#k_map_pair{key=K,val=V}) -> + [ssa_arg(K, St0),ssa_arg(V, St0)] + end, Es), + {Dst,St1} = new_ssa_var(R, St0), + {Is,St} = put_cg_map(LineAnno, Op, SrcMap, Dst, List, St1), + {Is,St}; +put_cg([#k_var{name=R}], Con0, _Le, St0) -> + %% Create an alias for a variable or literal. + Con = ssa_arg(Con0, St0), + St = set_ssa_var(R, Con, St0), + {[],St}. + +put_cg_map(LineAnno, Op, SrcMap, Dst, List, St0) -> + Fail = bif_fail_label(St0), + Args = [#b_literal{val=Op},SrcMap|List], + PutMap = #b_set{anno=LineAnno,op=put_map,dst=Dst,args=Args}, + if + Op =:= assoc -> + {[PutMap],St0}; + true -> + {Is,St} = make_cond_branch(succeeded, [Dst], Fail, St0), + {[PutMap|Is],St} + end. + +%%% +%%% Code generation for constructing binaries. +%%% + +cg_binary(Dst, Segs0, Fail, Le, St0) -> + {PutCode0,SzCalc0,St1} = cg_bin_put(Segs0, Fail, St0), + LineAnno = line_anno(Le), + Anno = Le#k.a, + case PutCode0 of + [#b_set{op=bs_put,dst=Bool,args=[_,_,Src,#b_literal{val=all}|_]}, + #b_br{bool=Bool}, + {label,_}|_] -> + #k_bin_seg{unit=Unit0,next=Segs} = Segs0, + Unit = #b_literal{val=Unit0}, + {PutCode,SzCalc1,St2} = cg_bin_put(Segs, Fail, St1), + {_,SzVar,SzCode0,St3} = cg_size_calc(1, SzCalc1, Fail, St2), + SzCode = cg_bin_anno(SzCode0, LineAnno), + Args = case member(single_use, Anno) of + true -> + [#b_literal{val=private_append},Src,SzVar,Unit]; + false -> + [#b_literal{val=append},Src,SzVar,Unit] + end, + BsInit = #b_set{anno=LineAnno,op=bs_init,dst=Dst,args=Args}, + {TestIs,St} = make_cond_branch(succeeded, [Dst], Fail, St3), + {SzCode ++ [BsInit] ++ TestIs ++ PutCode,St}; + [#b_set{op=bs_put}|_] -> + {Unit,SzVar,SzCode0,St2} = cg_size_calc(8, SzCalc0, Fail, St1), + SzCode = cg_bin_anno(SzCode0, LineAnno), + Args = [#b_literal{val=new},SzVar,Unit], + BsInit = #b_set{anno=LineAnno,op=bs_init,dst=Dst,args=Args}, + {TestIs,St} = make_cond_branch(succeeded, [Dst], Fail, St2), + {SzCode ++ [BsInit] ++ TestIs ++ PutCode0,St} + end. + +cg_bin_anno([Set|Sets], Anno) -> + [Set#b_set{anno=Anno}|Sets]; +cg_bin_anno([], _) -> []. + +%% cg_size_calc(PreferredUnit, SzCalc, Fail, St0) -> +%% {ActualUnit,SizeVariable,SizeCode,St}. +%% Generate size calculation code. + +cg_size_calc(Unit, error, _Fail, St) -> + {#b_literal{val=Unit},#b_literal{val=badarg},[],St}; +cg_size_calc(8, [{1,_}|_]=SzCalc, Fail, St) -> + cg_size_calc(1, SzCalc, Fail, St); +cg_size_calc(8, SzCalc, Fail, St0) -> + {Var,Pre,St} = cg_size_calc_1(SzCalc, Fail, St0), + {#b_literal{val=8},Var,Pre,St}; +cg_size_calc(1, SzCalc0, Fail, St0) -> + SzCalc = map(fun({8,#b_literal{val=Size}}) -> + {1,#b_literal{val=8*Size}}; + ({8,{{bif,byte_size},Src}}) -> + {1,{{bif,bit_size},Src}}; + ({8,{_,_}=UtfCalc}) -> + {1,{'*',#b_literal{val=8},UtfCalc}}; + ({_,_}=Pair) -> + Pair + end, SzCalc0), + {Var,Pre,St} = cg_size_calc_1(SzCalc, Fail, St0), + {#b_literal{val=1},Var,Pre,St}. + +cg_size_calc_1(SzCalc, Fail, St0) -> + cg_size_calc_2(SzCalc, #b_literal{val=0}, Fail, St0). + +cg_size_calc_2([{_,{'*',Unit,{_,_}=Bif}}|T], Sum0, Fail, St0) -> + {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, Fail, St0), + {BifDst,Pre1,St2} = cg_size_bif(Bif, Fail, St1), + {Sum,Pre2,St} = cg_size_add(Sum1, BifDst, Unit, Fail, St2), + {Sum,Pre0++Pre1++Pre2,St}; +cg_size_calc_2([{_,#b_literal{}=Sz}|T], Sum0, Fail, St0) -> + {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, Fail, St0), + {Sum,Pre,St} = cg_size_add(Sum1, Sz, #b_literal{val=1}, Fail, St1), + {Sum,Pre0++Pre,St}; +cg_size_calc_2([{_,#b_var{}=Sz}|T], Sum0, Fail, St0) -> + {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, Fail, St0), + {Sum,Pre,St} = cg_size_add(Sum1, Sz, #b_literal{val=1}, Fail, St1), + {Sum,Pre0++Pre,St}; +cg_size_calc_2([{_,{_,_}=Bif}|T], Sum0, Fail, St0) -> + {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, Fail, St0), + {BifDst,Pre1,St2} = cg_size_bif(Bif, Fail, St1), + {Sum,Pre2,St} = cg_size_add(Sum1, BifDst, #b_literal{val=1}, Fail, St2), + {Sum,Pre0++Pre1++Pre2,St}; +cg_size_calc_2([], Sum, _Fail, St) -> + {Sum,[],St}. + +cg_size_bif(#b_var{}=Var, _Fail, St) -> + {Var,[],St}; +cg_size_bif({Name,Src}, Fail, St0) -> + {Dst,St1} = new_ssa_var('@ssa_bif', St0), + Bif = #b_set{op=Name,dst=Dst,args=[Src]}, + {TestIs,St} = make_cond_branch(succeeded, [Dst], Fail, St1), + {Dst,[Bif|TestIs],St}. + +cg_size_add(#b_literal{val=0}, Val, #b_literal{val=1}, _Fail, St) -> + {Val,[],St}; +cg_size_add(A, B, Unit, Fail, St0) -> + {Dst,St1} = new_ssa_var('@ssa_sum', St0), + {TestIs,St} = make_cond_branch(succeeded, [Dst], Fail, St1), + BsAdd = #b_set{op=bs_add,dst=Dst,args=[A,B,Unit]}, + {Dst,[BsAdd|TestIs],St}. + +cg_bin_put(Seg, Fail, St) -> + cg_bin_put_1(Seg, Fail, [], [], St). + +cg_bin_put_1(#k_bin_seg{size=Size0,unit=U,type=T,flags=Fs,seg=Src0,next=Next}, + Fail, Acc, SzCalcAcc, St0) -> + [Src,Size] = ssa_args([Src0,Size0], St0), + NeedSize = bs_need_size(T), + TypeArg = #b_literal{val=T}, + Flags = #b_literal{val=Fs}, + Unit = #b_literal{val=U}, + Args = case NeedSize of + true -> [TypeArg,Flags,Src,Size,Unit]; + false -> [TypeArg,Flags,Src] + end, + {Is,St} = make_cond_branch(bs_put, Args, Fail, St0), + SzCalc = bin_size_calc(T, Src, Size, U), + cg_bin_put_1(Next, Fail, reverse(Is, Acc), [SzCalc|SzCalcAcc], St); +cg_bin_put_1(#k_bin_end{}, _, Acc, SzCalcAcc, St) -> + SzCalc = fold_size_calc(SzCalcAcc, 0, []), + {reverse(Acc),SzCalc,St}. + +bs_need_size(utf8) -> false; +bs_need_size(utf16) -> false; +bs_need_size(utf32) -> false; +bs_need_size(_) -> true. + +bin_size_calc(utf8, Src, _Size, _Unit) -> + {8,{bs_utf8_size,Src}}; +bin_size_calc(utf16, Src, _Size, _Unit) -> + {8,{bs_utf16_size,Src}}; +bin_size_calc(utf32, _Src, _Size, _Unit) -> + {8,#b_literal{val=4}}; +bin_size_calc(binary, Src, #b_literal{val=all}, Unit) -> + case Unit rem 8 of + 0 -> {8,{{bif,byte_size},Src}}; + _ -> {1,{{bif,bit_size},Src}} + end; +bin_size_calc(_Type, _Src, Size, Unit) -> + {Unit,Size}. + +fold_size_calc([{Unit,#b_literal{val=Size}}|T], Bits, Acc) -> + if + is_integer(Size) -> + fold_size_calc(T, Bits + Unit*Size, Acc); + true -> + error + end; +fold_size_calc([{U,#b_var{}}=H|T], Bits, Acc) when U =:= 1; U =:= 8 -> + fold_size_calc(T, Bits, [H|Acc]); +fold_size_calc([{U,#b_var{}=Var}|T], Bits, Acc) -> + fold_size_calc(T, Bits, [{1,{'*',#b_literal{val=U},Var}}|Acc]); +fold_size_calc([{_,_}=H|T], Bits, Acc) -> + fold_size_calc(T, Bits, [H|Acc]); +fold_size_calc([], Bits, Acc) -> + Bytes = Bits div 8, + RemBits = Bits rem 8, + Sizes = sort([{1,#b_literal{val=RemBits}},{8,#b_literal{val=Bytes}}|Acc]), + [Pair || {_,Sz}=Pair <- Sizes, Sz =/= #b_literal{val=0}]. + +%%% +%%% Utilities for creating the SSA types. +%%% + +ssa_args(As, St) -> + [ssa_arg(A, St) || A <- As]. + +ssa_arg(#k_var{name=V}, #cg{vars=Vars}) -> maps:get(V, Vars); +ssa_arg(#k_literal{val=V}, _) -> #b_literal{val=V}; +ssa_arg(#k_atom{val=V}, _) -> #b_literal{val=V}; +ssa_arg(#k_float{val=V}, _) -> #b_literal{val=V}; +ssa_arg(#k_int{val=V}, _) -> #b_literal{val=V}; +ssa_arg(#k_nil{}, _) -> #b_literal{val=[]}. + +new_ssa_vars(Vs, St) -> + mapfoldl(fun(#k_var{name=V}, S) -> + new_ssa_var(V, S) + end, St, Vs). + +new_ssa_var(VarBase, #cg{lcount=Uniq,vars=Vars}=St0) + when is_atom(VarBase); is_integer(VarBase) -> + case Vars of + #{VarBase:=_} -> + Var = #b_var{name={VarBase,Uniq}}, + St = St0#cg{lcount=Uniq+1,vars=Vars#{VarBase=>Var}}, + {Var,St}; + #{} -> + Var = #b_var{name=VarBase}, + St = St0#cg{vars=Vars#{VarBase=>Var}}, + {Var,St} + end. + +set_ssa_var(VarBase, Val, #cg{vars=Vars}=St) + when is_atom(VarBase); is_integer(VarBase) -> + St#cg{vars=Vars#{VarBase=>Val}}. + +%% new_label(St) -> {L,St}. + +new_label(#cg{lcount=Next}=St) -> + {Next,St#cg{lcount=Next+1}}. + +%% line_anno(Le) -> #{} | #{location:={File,Line}}. +%% Create a location annotation, containing information about the +%% current filename and line number. The annotation should be +%% included in any operation that could cause an exception. + +line_anno(#k{a=Anno}) -> + line_anno_1(Anno). + +line_anno_1([Line,{file,Name}]) when is_integer(Line) -> + line_anno_2(Name, Line); +line_anno_1([_|_]=A) -> + {Name,Line} = find_loc(A, no_file, 0), + line_anno_2(Name, Line); +line_anno_1([]) -> + #{}. + +line_anno_2(no_file, _) -> + #{}; +line_anno_2(_, 0) -> + %% Missing line number or line number 0. + #{}; +line_anno_2(Name, Line) -> + #{location=>{Name,Line}}. + +find_loc([Line|T], File, _) when is_integer(Line) -> + find_loc(T, File, Line); +find_loc([{file,File}|T], _, Line) -> + find_loc(T, File, Line); +find_loc([_|T], File, Line) -> + find_loc(T, File, Line); +find_loc([], File, Line) -> {File,Line}. + +flatmapfoldl(F, Accu0, [Hd|Tail]) -> + {R,Accu1} = F(Hd, Accu0), + {Rs,Accu2} = flatmapfoldl(F, Accu1, Tail), + {R++Rs,Accu2}; +flatmapfoldl(_, Accu, []) -> {[],Accu}. + +%%% +%%% Finalize the code. +%%% + +finalize(Asm0, St0) -> + Asm1 = fix_phis(Asm0), + {Asm,St} = fix_sets(Asm1, [], St0), + {build_map(Asm),St}. + +fix_phis(Is) -> + fix_phis_1(Is, none, #{}). + +fix_phis_1([{label,L},#cg_phi{vars=[]}=Phi|Is0], _Lbl, Map0) -> + case maps:is_key(L, Map0) of + false -> + %% No #cg_break{} references this label. Nothing else can + %% reference it, so it can be safely be removed. + {Is,Map} = drop_upto_label(Is0, Map0), + fix_phis_1(Is, none, Map); + true -> + %% There is a break referencing this label; probably caused + %% by a try/catch whose return value is ignored. + [{label,L}|fix_phis_1([Phi|Is0], L, Map0)] + end; +fix_phis_1([{label,L}=I|Is], _Lbl, Map) -> + [I|fix_phis_1(Is, L, Map)]; +fix_phis_1([#cg_unreachable{}|Is0], _Lbl, Map0) -> + {Is,Map} = drop_upto_label(Is0, Map0), + fix_phis_1(Is, none, Map); +fix_phis_1([#cg_break{args=Args,phi=Target}|Is], Lbl, Map) when is_integer(Lbl) -> + Pairs1 = case Map of + #{Target:=Pairs0} -> Pairs0; + #{} -> [] + end, + Pairs = [[{Arg,Lbl} || Arg <- Args]|Pairs1], + I = make_uncond_branch(Target), + [I|fix_phis_1(Is, none, Map#{Target=>Pairs})]; +fix_phis_1([#cg_phi{vars=Vars}|Is0], Lbl, Map0) -> + Pairs = maps:get(Lbl, Map0), + Map1 = maps:remove(Lbl, Map0), + case gen_phis(Vars, Pairs) of + [#b_set{op=phi,args=[]}] -> + {Is,Map} = drop_upto_label(Is0, Map1), + Ret = #b_ret{arg=#b_literal{val=unreachable}}, + [Ret|fix_phis_1(Is, none, Map)]; + Phis -> + Phis ++ fix_phis_1(Is0, Lbl, Map1) + end; +fix_phis_1([I|Is], Lbl, Map) -> + [I|fix_phis_1(Is, Lbl, Map)]; +fix_phis_1([], _, Map) -> + [] = maps:to_list(Map), %Assertion. + []. + +gen_phis([V|Vs], Preds0) -> + {Pairs,Preds} = collect_preds(Preds0, [], []), + [#b_set{op=phi,dst=V,args=Pairs}|gen_phis(Vs, Preds)]; +gen_phis([], _) -> []. + +collect_preds([[First|Rest]|T], ColAcc, RestAcc) -> + collect_preds(T, [First|ColAcc], [Rest|RestAcc]); +collect_preds([], ColAcc, RestAcc) -> + {keysort(2, ColAcc),RestAcc}. + +fix_sets([#b_set{dst=none}=Set|Is], Acc, St0) -> + {Dst,St} = new_ssa_var('@ssa_ignored', St0), + I = Set#b_set{dst=Dst}, + fix_sets(Is, [I|Acc], St); +fix_sets([I|Is], Acc, St) -> + fix_sets(Is, [I|Acc], St); +fix_sets([], Acc, St) -> + {reverse(Acc),St}. + +build_map(Is) -> + Blocks = build_graph_1(Is, [], []), + maps:from_list(Blocks). + +build_graph_1([{label,L}|Is], Lbls, []) -> + build_graph_1(Is, [L|Lbls], []); +build_graph_1([{label,L}|Is], Lbls, [_|_]=BlockAcc) -> + make_blocks(Lbls, BlockAcc) ++ build_graph_1(Is, [L], []); +build_graph_1([I|Is], Lbls, BlockAcc) -> + build_graph_1(Is, Lbls, [I|BlockAcc]); +build_graph_1([], Lbls, BlockAcc) -> + make_blocks(Lbls, BlockAcc). + +make_blocks(Lbls, [Last|Is0]) -> + Is = reverse(Is0), + Block = #b_blk{is=Is,last=Last}, + [{L,Block} || L <- Lbls]. + +drop_upto_label([{label,_}|_]=Is, Map) -> + {Is,Map}; +drop_upto_label([#cg_break{phi=Target}|Is], Map) -> + Pairs = case Map of + #{Target:=Pairs0} -> Pairs0; + #{} -> [] + end, + drop_upto_label(Is, Map#{Target=>Pairs}); +drop_upto_label([_|Is], Map) -> + drop_upto_label(Is, Map). + +k_get_anno(Thing) -> element(2, Thing). diff --git a/lib/compiler/src/beam_listing.erl b/lib/compiler/src/beam_listing.erl index 518b958794..6121593b11 100644 --- a/lib/compiler/src/beam_listing.erl +++ b/lib/compiler/src/beam_listing.erl @@ -23,6 +23,7 @@ -include("core_parse.hrl"). -include("v3_kernel.hrl"). +-include("beam_ssa.hrl"). -include("beam_disasm.hrl"). -import(lists, [foreach/2]). @@ -41,6 +42,12 @@ module(File, #k_mdef{}=Kern) -> %% This is a kernel module. io:put_chars(File, v3_kernel_pp:format(Kern)); %%io:put_chars(File, io_lib:format("~p~n", [Kern])); +module(File, #b_module{name=Mod,exports=Exp,attributes=Attr,body=Fs}) -> + io:format(File, "module ~p.\n", [Mod]), + io:format(File, "exports ~p.\n", [Exp]), + io:format(File, "attributes ~p.\n\n", [Attr]), + PP = [beam_ssa_pp:format_function(F) || F <- Fs], + io:put_chars(File, lists:join($\n, PP)); module(Stream, {Mod,Exp,Attr,Code,NumLabels}) -> %% This is output from v3_codegen. io:format(Stream, "{module, ~p}. %% version = ~w\n", @@ -59,7 +66,7 @@ module(Stream, [_|_]=Fs) -> foreach(fun (F) -> io:format(Stream, "~p.\n", [F]) end, Fs). format_asm([{label,L}|Is]) -> - [" {label,",integer_to_list(L),"}.\n"|format_asm(Is)]; + [io_lib:format(" {label,~p}.\n", [L])|format_asm(Is)]; format_asm([I|Is]) -> [io_lib:format(" ~p", [I]),".\n"|format_asm(Is)]; format_asm([]) -> []. diff --git a/lib/compiler/src/beam_peep.erl b/lib/compiler/src/beam_peep.erl index 2b8dd40e29..5730e9704e 100644 --- a/lib/compiler/src/beam_peep.erl +++ b/lib/compiler/src/beam_peep.erl @@ -94,38 +94,43 @@ peep([{gc_bif,_,_,_,_,Dst}=I|Is], SeenTests0, Acc) -> peep([{jump,{f,L}},{label,L}=I|Is], _, Acc) -> %% Sometimes beam_jump has missed this optimization. peep(Is, gb_sets:empty(), [I|Acc]); -peep([{select,Op,R,F,Vls0}|Is], SeenTests0, Acc0) -> +peep([{select,select_val,R,F,Vls0}|Is], SeenTests0, Acc0) -> case prune_redundant_values(Vls0, F) of [] -> %% No values left. Must convert to plain jump. I = {jump,F}, peep([I|Is], gb_sets:empty(), Acc0); - [{atom,_}=Value,Lbl] when Op =:= select_val -> - %% Single value left. Convert to regular test and pop redundant tests. + [{atom,_}=Value,Lbl] -> + %% Single value left. Convert to regular test. Is1 = [{test,is_eq_exact,F,[R,Value]},{jump,Lbl}|Is], - case Acc0 of - [{test,is_atom,F,[R]}|Acc] -> - peep(Is1, SeenTests0, Acc); - _ -> - peep(Is1, SeenTests0, Acc0) - end; - [{integer,_}=Value,Lbl] when Op =:= select_val -> - %% Single value left. Convert to regular test and pop redundant tests. + peep(Is1, SeenTests0, Acc0); + [{integer,_}=Value,Lbl] -> + %% Single value left. Convert to regular test. Is1 = [{test,is_eq_exact,F,[R,Value]},{jump,Lbl}|Is], - case Acc0 of - [{test,is_integer,F,[R]}|Acc] -> - peep(Is1, SeenTests0, Acc); - _ -> - peep(Is1, SeenTests0, Acc0) - end; - [Arity,Lbl] when Op =:= select_tuple_arity -> - %% Single value left. Convert to regular test - Is1 = [{test,test_arity,F,[R,Arity]},{jump,Lbl}|Is], + peep(Is1, SeenTests0, Acc0); + [{atom,B1},Lbl,{atom,B2},Lbl] when B1 =:= not B2 -> + %% Replace with is_boolean test. + Is1 = [{test,is_boolean,F,[R]},{jump,Lbl}|Is], peep(Is1, SeenTests0, Acc0); [_|_]=Vls -> - I = {select,Op,R,F,Vls}, + I = {select,select_val,R,F,Vls}, peep(Is, gb_sets:empty(), [I|Acc0]) end; +peep([{get_map_elements,Fail,Src,List}=I|Is], _SeenTests, Acc0) -> + SeenTests = gb_sets:empty(), + case simplify_get_map_elements(Fail, Src, List, Acc0) of + {ok,Acc} -> + peep(Is, SeenTests, Acc); + error -> + peep(Is, SeenTests, [I|Acc0]) + end; +peep([{test,has_map_fields,Fail,Ops}=I|Is], SeenTests, Acc0) -> + case simplify_has_map_fields(Fail, Ops, Acc0) of + {ok,Acc} -> + peep(Is, SeenTests, Acc); + error -> + peep(Is, SeenTests, [I|Acc0]) + end; peep([{test,Op,_,Ops}=I|Is], SeenTests0, Acc) -> case beam_utils:is_pure_test(I) of false -> @@ -176,3 +181,39 @@ prune_redundant_values([_Val,F|Vls], F) -> prune_redundant_values([Val,Lbl|Vls], F) -> [Val,Lbl|prune_redundant_values(Vls, F)]; prune_redundant_values([], _) -> []. + +simplify_get_map_elements(Fail, Src, {list,[Key,Dst]}, + [{get_map_elements,Fail,Src,{list,List1}}|Acc]) -> + case are_keys_literals([Key]) andalso are_keys_literals(List1) of + true -> + case member(Key, List1) of + true -> + %% The key is already in the other list. That is + %% very unusual, because there are optimizations to get + %% rid of duplicate keys. Therefore, don't try to + %% do anything smart here; just keep the + %% get_map_elements instructions separate. + error; + false -> + List = [Key,Dst|List1], + {ok,[{get_map_elements,Fail,Src,{list,List}}|Acc]} + end; + false -> + error + end; +simplify_get_map_elements(_, _, _, _) -> error. + +simplify_has_map_fields(Fail, [Src|Keys0], + [{test,has_map_fields,Fail,[Src|Keys1]}|Acc]) -> + case are_keys_literals(Keys0) andalso are_keys_literals(Keys1) of + true -> + Keys = Keys0 ++ Keys1, + {ok,[{test,has_map_fields,Fail,[Src|Keys]}|Acc]}; + false -> + error + end; +simplify_has_map_fields(_, _, _) -> error. + +are_keys_literals([{x,_}|_]) -> false; +are_keys_literals([{y,_}|_]) -> false; +are_keys_literals([_|_]) -> true. diff --git a/lib/compiler/src/beam_receive.erl b/lib/compiler/src/beam_receive.erl deleted file mode 100644 index ddbe67605a..0000000000 --- a/lib/compiler/src/beam_receive.erl +++ /dev/null @@ -1,416 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2010-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(beam_receive). --export([module/2]). --import(lists, [foldl/3,reverse/1,reverse/2]). - -%%% -%%% In code such as: -%%% -%%% Ref = make_ref(), %Or erlang:monitor(process, Pid) -%%% . -%%% . -%%% . -%%% receive -%%% {Ref,Reply} -> Reply -%%% end. -%%% -%%% we know that none of the messages that exist in the message queue -%%% before the call to make_ref/0 can be matched out in the receive -%%% statement. Therefore we can avoid going through the entire message -%%% queue if we introduce two new instructions (here written as -%%% BIFs in pseudo-Erlang): -%%% -%%% recv_mark(SomeUniqInteger), -%%% Ref = make_ref(), -%%% . -%%% . -%%% . -%%% recv_set(SomeUniqInteger), -%%% receive -%%% {Ref,Reply} -> Reply -%%% end. -%%% -%%% The recv_mark/1 instruction will save the current position and -%%% SomeUniqInteger in the process context. The recv_set -%%% instruction will verify that SomeUniqInteger is still stored -%%% in the process context. If it is, it will set the current pointer -%%% for the message queue (the next message to be read out) to the -%%% position that was saved by recv_mark/1. -%%% -%%% The remove_message instruction must be modified to invalidate -%%% the information stored by the previous recv_mark/1, in case there -%%% is another receive executed between the calls to recv_mark/1 and -%%% recv_set/1. -%%% -%%% We use a reference to a label (i.e. a position in the loaded code) -%%% as the SomeUniqInteger. -%%% - --spec module(beam_utils:module_code(), [compile:option()]) -> - {'ok',beam_utils:module_code()}. - -module({Mod,Exp,Attr,Fs0,Lc}, _Opts) -> - Fs = [function(F) || F <- Fs0], - Code = {Mod,Exp,Attr,Fs,Lc}, - {ok,Code}. - -%%% -%%% Local functions. -%%% - -function({function,Name,Arity,Entry,Is}) -> - try - D = beam_utils:index_labels(Is), - {function,Name,Arity,Entry,opt(Is, D, [])} - catch - Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [Name,Arity]), - erlang:raise(Class, Error, Stack) - end. - -opt([{call_ext,A,{extfunc,erlang,spawn_monitor,A}}=I0|Is0], D, Acc) - when A =:= 1; A =:= 3 -> - case ref_in_tuple(Is0) of - no -> - opt(Is0, D, [I0|Acc]); - {yes,Regs,Is1,MatchReversed} -> - %% The call creates a brand new reference. Now - %% search for a receive statement in the same - %% function that will match against the reference. - case opt_recv(Is1, Regs, D) of - no -> - opt(Is0, D, [I0|Acc]); - {yes,Is,Lbl} -> - opt(Is, D, MatchReversed++[I0,{recv_mark,{f,Lbl}}|Acc]) - end - end; -opt([{call_ext,Arity,{extfunc,erlang,Name,Arity}}=I|Is0], D, Acc) -> - case creates_new_ref(Name, Arity) of - true -> - %% The call creates a brand new reference. Now - %% search for a receive statement in the same - %% function that will match against the reference. - case opt_recv(Is0, regs_init_x0(), D) of - no -> - opt(Is0, D, [I|Acc]); - {yes,Is,Lbl} -> - opt(Is, D, [I,{recv_mark,{f,Lbl}}|Acc]) - end; - false -> - opt(Is0, D, [I|Acc]) - end; -opt([I|Is], D, Acc) -> - opt(Is, D, [I|Acc]); -opt([], _, Acc) -> - reverse(Acc). - -ref_in_tuple([{test,is_tuple,_,[{x,0}]}=I1, - {test,test_arity,_,[{x,0},2]}=I2, - {block,[{set,[_],[{x,0}],{get_tuple_element,0}}, - {set,[Dst],[{x,0}],{get_tuple_element,1}}|Bl]}=I3|Is]) -> - ref_in_tuple_1(Bl, Dst, Is, [I3,I2,I1]); -ref_in_tuple([{test,is_tuple,_,[{x,0}]}=I1, - {test,test_arity,_,[{x,0},2]}=I2, - {block,[{set,[Dst],[{x,0}],{get_tuple_element,1}}|Bl]}=I3|Is]) -> - ref_in_tuple_1(Bl, Dst, Is, [I3,I2,I1]); -ref_in_tuple(_) -> no. - -ref_in_tuple_1(Bl, Dst, Is, MatchReversed) -> - Regs0 = regs_init_singleton(Dst), - Regs = opt_update_regs_bl(Bl, Regs0), - {yes,Regs,Is,MatchReversed}. - -%% creates_new_ref(Name, Arity) -> true|false. -%% Return 'true' if the BIF Name/Arity will create a new reference. -creates_new_ref(monitor, 2) -> true; -creates_new_ref(make_ref, 0) -> true; -creates_new_ref(_, _) -> false. - -%% opt_recv([Instruction], Regs, LabelIndex) -> no|{yes,[Instruction]} -%% Search for a receive statement that will only retrieve messages -%% that contain the newly created reference (which is currently in {x,0}). -opt_recv(Is, Regs, D) -> - L = gb_sets:empty(), - opt_recv(Is, D, Regs, L, []). - -opt_recv([{label,L}=Lbl,{loop_rec,{f,Fail},_}=Loop|Is], D, R0, _, Acc) -> - R = regs_kill_not_live(0, R0), - case regs_empty(R) of - false -> - %% We now have the new reference in Y registers - %% and the current instruction is the beginning of a - %% receive statement. We must now verify that only messages - %% that contain the reference will be matched. - case opt_ref_used(Is, R, Fail, D) of - false -> - no; - true -> - RecvSet = {recv_set,{f,L}}, - {yes,reverse(Acc, [RecvSet,Lbl,Loop|Is]),L} - end; - true -> - no - end; -opt_recv([I|Is], D, R0, L0, Acc) -> - {R,L} = opt_update_regs(I, R0, L0), - case regs_empty(R) of - true -> - %% The reference is no longer alive. There is no - %% point in continuing the search. - no; - false -> - opt_recv(Is, D, R, L, [I|Acc]) - end; -opt_recv([], _, _, _, _) -> no. - -opt_update_regs({block,Bl}, R, L) -> - {opt_update_regs_bl(Bl, R),L}; -opt_update_regs({call,_,_}, R, L) -> - {regs_kill_not_live(0, R),L}; -opt_update_regs({call_ext,_,_}, R, L) -> - {regs_kill_not_live(0, R),L}; -opt_update_regs({call_fun,_}, R, L) -> - {regs_kill_not_live(0, R),L}; -opt_update_regs({kill,Y}, R, L) -> - {regs_kill([Y], R),L}; -opt_update_regs({'catch',_,{f,Lbl}}, R, L) -> - {R,gb_sets:add(Lbl, L)}; -opt_update_regs({catch_end,_}, R, L) -> - {R,L}; -opt_update_regs({label,Lbl}, R, L) -> - case gb_sets:is_member(Lbl, L) of - false -> - %% We can't allow arbitrary labels (since the receive - %% could be entered without first creating the reference). - {regs_init(),L}; - true -> - %% A catch label for a previously seen catch instruction is OK. - {R,L} - end; -opt_update_regs({'try',_,{f,Lbl}}, R, L) -> - {R,gb_sets:add(Lbl, L)}; -opt_update_regs({try_end,_}, R, L) -> - {R,L}; -opt_update_regs({line,_}, R, L) -> - {R,L}; -opt_update_regs(_I, _R, L) -> - %% Unrecognized instruction. Abort the search. - {regs_init(),L}. - -opt_update_regs_bl([{set,Ds,_,{alloc,Live,_}}|Is], Regs0) -> - Regs1 = regs_kill_not_live(Live, Regs0), - Regs = regs_kill(Ds, Regs1), - opt_update_regs_bl(Is, Regs); -opt_update_regs_bl([{set,[Dst]=Ds,[Src],move}|Is], Regs0) -> - Regs1 = regs_kill(Ds, Regs0), - Regs = case regs_is_member(Src, Regs1) of - false -> Regs1; - true -> regs_add(Dst, Regs1) - end, - opt_update_regs_bl(Is, Regs); -opt_update_regs_bl([{set,Ds,_,_}|Is], Regs0) -> - Regs = regs_kill(Ds, Regs0), - opt_update_regs_bl(Is, Regs); -opt_update_regs_bl([], Regs) -> Regs. - -%% opt_ref_used([Instruction], RefRegs, FailLabel, LabelIndex) -> true|false -%% Return 'true' if it is certain that only messages that contain the same -%% reference as in RefRegs can be matched out. Otherwise return 'false'. -%% -%% Basically, we follow all possible paths through the receive statement. -%% If all paths are safe, we return 'true'. -%% -%% A branch to FailLabel is safe, because it exits the receive statement -%% and no further message may be matched out. -%% -%% If a path hits an comparision between RefRegs and part of the message, -%% that path is safe (any messages that may be matched further down the -%% path is guaranteed to contain the reference). -%% -%% Otherwise, if we hit a 'remove_message' instruction, we give up -%% and return 'false' (the optimization is definitely unsafe). If -%% we hit an unrecognized instruction, we also give up and return -%% 'false' (the optimization may be unsafe). - -opt_ref_used(Is, RefRegs, Fail, D) -> - Done = gb_sets:singleton(Fail), - Regs = regs_init_x0(), - try - _ = opt_ref_used_1(Is, RefRegs, D, Done, Regs), - true - catch - throw:not_used -> - false - end. - -%% This functions only returns if all paths through the receive -%% statement are safe, and throws an 'not_used' term otherwise. -opt_ref_used_1([{block,Bl}|Is], RefRegs, D, Done, Regs0) -> - Regs = opt_ref_used_bl(Bl, Regs0), - opt_ref_used_1(Is, RefRegs, D, Done, Regs); -opt_ref_used_1([{test,is_eq_exact,{f,Fail},Args}|Is], - RefRegs, D, Done0, Regs) -> - Done = opt_ref_used_at(Fail, RefRegs, D, Done0, Regs), - case is_ref_msg_comparison(Args, RefRegs, Regs) of - false -> - opt_ref_used_1(Is, RefRegs, D, Done, Regs); - true -> - %% The instructions that follow (Is) can only be executed - %% if the message contains the same reference as in RefRegs. - Done - end; -opt_ref_used_1([{test,is_ne_exact,{f,Fail},Args}|Is], - RefRegs, D, Done0, Regs) -> - Done = opt_ref_used_1(Is, RefRegs, D, Done0, Regs), - case is_ref_msg_comparison(Args, RefRegs, Regs) of - false -> - opt_ref_used_at(Fail, RefRegs, D, Done, Regs); - true -> - Done - end; -opt_ref_used_1([{test,_,{f,Fail},_}|Is], RefRegs, D, Done0, Regs) -> - Done = opt_ref_used_at(Fail, RefRegs, D, Done0, Regs), - opt_ref_used_1(Is, RefRegs, D, Done, Regs); -opt_ref_used_1([{select,_,_,{f,Fail},List}|_], RefRegs, D, Done, Regs) -> - Lbls = [F || {f,F} <- List] ++ [Fail], - opt_ref_used_in_all(Lbls, RefRegs, D, Done, Regs); -opt_ref_used_1([{label,Lbl}|Is], RefRegs, D, Done, Regs) -> - case gb_sets:is_member(Lbl, Done) of - true -> Done; - false -> opt_ref_used_1(Is, RefRegs, D, Done, Regs) - end; -opt_ref_used_1([{loop_rec_end,_}|_], _, _, Done, _) -> - Done; -opt_ref_used_1([_I|_], _RefReg, _D, _Done, _Regs) -> - %% The optimization may be unsafe. - throw(not_used). - -%% is_ref_msg_comparison(Args, RefRegs, RegisterSet) -> true|false. -%% Return 'true' if Args denotes a comparison between the -%% reference and message or part of the message. -is_ref_msg_comparison([R1,R2], RefRegs, Regs) -> - (regs_is_member(R2, RefRegs) andalso regs_is_member(R1, Regs)) orelse - (regs_is_member(R1, RefRegs) andalso regs_is_member(R2, Regs)). - -opt_ref_used_in_all([L|Ls], RefRegs, D, Done0, Regs) -> - Done = opt_ref_used_at(L, RefRegs, D, Done0, Regs), - opt_ref_used_in_all(Ls, RefRegs, D, Done, Regs); -opt_ref_used_in_all([], _, _, Done, _) -> Done. - -opt_ref_used_at(Fail, RefRegs, D, Done0, Regs) -> - case gb_sets:is_member(Fail, Done0) of - true -> - Done0; - false -> - Is = beam_utils:code_at(Fail, D), - Done = opt_ref_used_1(Is, RefRegs, D, Done0, Regs), - gb_sets:add(Fail, Done) - end. - -opt_ref_used_bl([{set,[],[],remove_message}|_], _) -> - %% We have proved that a message that does not depend on the - %% reference can be matched out. - throw(not_used); -opt_ref_used_bl([{set,Ds,Ss,_}|Is], Regs0) -> - case regs_all_members(Ss, Regs0) of - false -> - %% The destination registers may be assigned values that - %% are not dependent on the message being matched. - Regs = regs_kill(Ds, Regs0), - opt_ref_used_bl(Is, Regs); - true -> - %% All the sources depend on the message directly or - %% indirectly. - Regs = regs_add_list(Ds, Regs0), - opt_ref_used_bl(Is, Regs) - end; -opt_ref_used_bl([], Regs) -> Regs. - -%%% -%%% Functions for keeping track of a set of registers. -%%% - -%% regs_init() -> RegisterSet -%% Return an empty set of registers. - -regs_init() -> - {0,0}. - -%% regs_init_singleton(Register) -> RegisterSet -%% Return a set that only contains one register. - -regs_init_singleton(Reg) -> - regs_add(Reg, regs_init()). - -%% regs_init_x0() -> RegisterSet -%% Return a set that only contains the {x,0} register. - -regs_init_x0() -> - {1 bsl 0,0}. - -%% regs_empty(Register) -> true|false -%% Test whether the register set is empty. - -regs_empty(R) -> - R =:= {0,0}. - -%% regs_kill_not_live(Live, RegisterSet) -> RegisterSet' -%% Kill all registers indicated not live by Live. - -regs_kill_not_live(Live, {Xregs,Yregs}) -> - {Xregs band ((1 bsl Live)-1),Yregs}. - -%% regs_kill([Register], RegisterSet) -> RegisterSet' -%% Kill all registers mentioned in the list of registers. - -regs_kill([{x,N}|Rs], {Xregs,Yregs}) -> - regs_kill(Rs, {Xregs band (bnot (1 bsl N)),Yregs}); -regs_kill([{y,N}|Rs], {Xregs,Yregs}) -> - regs_kill(Rs, {Xregs,Yregs band (bnot (1 bsl N))}); -regs_kill([{fr,_}|Rs], Regs) -> - regs_kill(Rs, Regs); -regs_kill([], Regs) -> Regs. - -regs_add_list(List, Regs) -> - foldl(fun(R, A) -> regs_add(R, A) end, Regs, List). - -%% regs_add(Register, RegisterSet) -> RegisterSet' -%% Add a new register to the set of registers. - -regs_add({x,N}, {Xregs,Yregs}) -> - {Xregs bor (1 bsl N),Yregs}; -regs_add({y,N}, {Xregs,Yregs}) -> - {Xregs,Yregs bor (1 bsl N)}. - -%% regs_all_members([Register], RegisterSet) -> true|false -%% Test whether all of the registers are part of the register set. - -regs_all_members([R|Rs], Regs) -> - regs_is_member(R, Regs) andalso regs_all_members(Rs, Regs); -regs_all_members([], _) -> true. - -%% regs_is_member(Register, RegisterSet) -> true|false -%% Test whether Register is part of the register set. - -regs_is_member({x,N}, {Regs,_}) -> Regs band (1 bsl N) =/= 0; -regs_is_member({y,N}, {_,Regs}) -> Regs band (1 bsl N) =/= 0; -regs_is_member(_, _) -> false. diff --git a/lib/compiler/src/beam_record.erl b/lib/compiler/src/beam_record.erl deleted file mode 100644 index 58a6de6775..0000000000 --- a/lib/compiler/src/beam_record.erl +++ /dev/null @@ -1,131 +0,0 @@ -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2014-2017. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% - -%% Rewrite the instruction stream on tagged tuple tests. -%% Tagged tuples means a tuple of any arity with an atom as its -%% first element, such as records and error tuples. -%% -%% From: -%% ... -%% {test,is_tuple,Fail,[Src]}. -%% {test,test_arity,Fail,[Src,Sz]}. -%% ... -%% {get_tuple_element,Src,0,Dst}. -%% ... -%% {test,is_eq_exact,Fail,[Dst,Atom]}. -%% ... -%% To: -%% ... -%% {test,is_tagged_tuple,Fail,[Src,Sz,Atom]}. -%% ... -%% - --module(beam_record). --export([module/2]). - --import(lists, [reverse/1,reverse/2]). - --spec module(beam_utils:module_code(), [compile:option()]) -> - {'ok',beam_utils:module_code()}. - -module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> - Fs = [function(F) || F <- Fs0], - {ok,{Mod,Exp,Attr,Fs,Lc}}. - -function({function,Name,Arity,CLabel,Is0}) -> - try - Is1 = beam_utils:anno_defs(Is0), - Idx = beam_utils:index_labels(Is1), - Is = rewrite(reverse(Is1), Idx), - {function,Name,Arity,CLabel,Is} - catch - Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [Name,Arity]), - erlang:raise(Class, Error, Stack) - end. - -rewrite(Is, Idx) -> - rewrite(Is, Idx, 0, []). - -rewrite([{test,test_arity,Fail,[Src,N]}=TA, - {test,is_tuple,Fail,[Src]}=TT|Is], Idx, Def, Acc0) -> - case is_tagged_tuple(Acc0, Def, Fail, Src, Idx) of - no -> - rewrite(Is, Idx, 0, [TT,TA|Acc0]); - {yes,Atom,Acc} -> - I = {test,is_tagged_tuple,Fail,[Src,N,Atom]}, - rewrite(Is, Idx, Def, [I|Acc]) - end; -rewrite([{block,[{'%anno',{def,Def}}|Bl]}|Is], Idx, _Def, Acc) -> - rewrite(Is, Idx, Def, [{block,Bl}|Acc]); -rewrite([{label,L}=I|Is], Idx0, Def, Acc) -> - Idx = beam_utils:index_label(L, Acc, Idx0), - rewrite(Is, Idx, Def, [I|Acc]); -rewrite([I|Is], Idx, Def, Acc) -> - rewrite(Is, Idx, Def, [I|Acc]); -rewrite([], _, _, Acc) -> Acc. - -is_tagged_tuple([{block,Bl}, - {test,is_eq_exact,Fail,[Dst,{atom,_}=Atom]}|Is], - Def, Fail, Src, Idx) -> - case is_tagged_tuple_1(Bl, Is, Fail, Src, Dst, Idx, Def, []) of - no -> - no; - {yes,[]} -> - {yes,Atom,Is}; - {yes,[_|_]=Block} -> - {yes,Atom,[{block,Block}|Is]} - end; -is_tagged_tuple(_, _, _, _, _) -> - no. - -is_tagged_tuple_1([{set,[Dst],[Src],{get_tuple_element,0}}=I|Bl], - Is, Fail, Src, Dst, Idx, Def, Acc) -> - %% Check usage of Dst to find out whether the get_tuple_element - %% is needed. - case usage(Dst, Is, Fail, Idx) of - killed -> - %% Safe to remove the get_tuple_element instruction. - {yes,reverse(Acc, Bl)}; - used -> - %% Actively used. Must keep instruction. - {yes,reverse(Acc, [I|Bl])}; - not_used -> - %% Not actually used (but must be initialized). - case is_defined(Dst, Def) of - false -> - %% Dst must be initialized, but the - %% actual value does not matter. - Kill = {set,[Dst],[nil],move}, - {yes,reverse(Acc, [Kill|Bl])}; - true -> - %% The register is previously initialized. - %% We can remove the instruction. - {yes,reverse(Acc, Bl)} - end - end; -is_tagged_tuple_1([I|Bl], Is, Fail, Src, Dst, Idx, Def, Acc) -> - is_tagged_tuple_1(Bl, Is, Fail, Src, Dst, Idx, Def, [I|Acc]); -is_tagged_tuple_1(_, _, _, _, _, _, _, _) -> - no. - -usage(Dst, Is, Fail, Idx) -> - beam_utils:usage(Dst, [{test,is_number,Fail,[nil]}|Is], Idx). - -is_defined({x,X}, Def) -> - (Def bsr X) band 1 =:= 1. diff --git a/lib/compiler/src/beam_reorder.erl b/lib/compiler/src/beam_reorder.erl deleted file mode 100644 index 8d2ef5a431..0000000000 --- a/lib/compiler/src/beam_reorder.erl +++ /dev/null @@ -1,150 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1999-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(beam_reorder). - --export([module/2]). --import(lists, [member/2,reverse/1]). - --spec module(beam_utils:module_code(), [compile:option()]) -> - {'ok',beam_utils:module_code()}. - -module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> - Fs = [function(F) || F <- Fs0], - {ok,{Mod,Exp,Attr,Fs,Lc}}. - -function({function,Name,Arity,CLabel,Is0}) -> - try - Is = reorder(Is0), - {function,Name,Arity,CLabel,Is} - catch - Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [Name,Arity]), - erlang:raise(Class, Error, Stack) - end. - -%% reorder(Instructions0) -> Instructions -%% Reorder instructions before the beam_block pass, because reordering -%% will be more cumbersome when the blocks are in place. -%% -%% Execution of get_tuple_element instructions can be delayed until -%% they are actually needed. Consider the sequence: -%% -%% get_tuple_element Tuple Pos Dst -%% test Test Fail Operands -%% -%% If Dst is killed at label Fail (and not referenced in Operands), -%% we can can swap the instructions: -%% -%% test Test Fail Operands -%% get_tuple_element Tuple Pos Dst -%% -%% That can be beneficial in two ways: Firstly, if the branch is taken -%% we have avoided execution of the get_tuple_element instruction. -%% Secondly, even if the branch is not taken, subsequent optimization -%% (opt_blocks/1) may be able to change Dst to the final destination -%% register and eliminate a 'move' instruction. - -reorder(Is) -> - D = beam_utils:index_labels(Is), - reorder_1(Is, D, []). - -reorder_1([{Op,_,_}=TryCatch|[I|Is]=Is0], D, Acc) - when Op =:= 'catch'; Op =:= 'try' -> - %% Don't allow 'try' or 'catch' instructions to split blocks if - %% it can be avoided. - case is_safe(I) of - false -> - reorder_1(Is0, D, [TryCatch|Acc]); - true -> - reorder_1([TryCatch|Is], D, [I|Acc]) - end; -reorder_1([{label,L}=I|_], D, Acc) -> - Is = beam_utils:code_at(L, D), - reorder_1(Is, D, [I|Acc]); -reorder_1([{test,is_nonempty_list,_,_}=I|Is], D, Acc) -> - %% The run-time system may combine the is_nonempty_list test with - %% the following get_list instruction. - reorder_1(Is, D, [I|Acc]); -reorder_1([{test,_,_,_}=I, - {select,_,_,_,_}=S|Is], D, Acc) -> - %% There is nothing to gain by inserting a get_tuple_element - %% instruction between the test instruction and the select - %% instruction. - reorder_1(Is, D, [S,I|Acc]); -reorder_1([{test,_,{f,_},[Src|_]}=I|Is], D, - [{get_tuple_element,Src,_,_}|_]=Acc) -> - %% We want to avoid code that can confuse beam_validator such as: - %% is_tuple Fail Src - %% test_arity Fail Src Arity - %% is_map Fail Src - %% get_tuple_element Src Pos Dst - %% Therefore, don't reorder the instructions in such cases. - reorder_1(Is, D, [I|Acc]); -reorder_1([{test,_,{f,L},Ss}=I|Is0], D0, - [{get_tuple_element,_,_,El}=G|Acc0]=Acc) -> - case member(El, Ss) of - true -> - reorder_1(Is0, D0, [I|Acc]); - false -> - case beam_utils:is_killed_at(El, L, D0) of - true -> - Is = [I,G|Is0], - reorder_1(Is, D0, Acc0); - false -> - case beam_utils:is_killed(El, Is0, D0) of - true -> - Code0 = beam_utils:code_at(L, D0), - Code = [G|Code0], - D = beam_utils:index_label(L, Code, D0), - Is = [I|Is0], - reorder_1(Is, D, Acc0); - false -> - reorder_1(Is0, D0, [I|Acc]) - end - end - end; -reorder_1([{allocate_zero,N,Live}=I0|Is], D, - [{get_tuple_element,{x,Tup},_,{x,Dst}}=G|Acc]=Acc0) -> - case Tup < Dst andalso Dst+1 =:= Live of - true -> - %% Move allocation instruction upwards past - %% get_tuple_element instructions to create more - %% opportunities for moving get_tuple_element - %% instructions. - I = {allocate_zero,N,Dst}, - reorder_1([I,G|Is], D, Acc); - false -> - reorder_1(Is, D, [I0|Acc0]) - end; -reorder_1([I|Is], D, Acc) -> - reorder_1(Is, D, [I|Acc]); -reorder_1([], _, Acc) -> reverse(Acc). - -%% is_safe(Instruction) -> true|false -%% Test whether an instruction is safe (cannot cause an exception). - -is_safe({kill,_}) -> true; -is_safe({move,_,_}) -> true; -is_safe({put,_}) -> true; -is_safe({put_list,_,_,_}) -> true; -is_safe({put_tuple,_,_}) -> true; -is_safe({test_heap,_,_}) -> true; -is_safe(_) -> false. diff --git a/lib/compiler/src/beam_split.erl b/lib/compiler/src/beam_split.erl deleted file mode 100644 index 809e49b3d0..0000000000 --- a/lib/compiler/src/beam_split.erl +++ /dev/null @@ -1,94 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2011-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(beam_split). --export([module/2]). - --import(lists, [reverse/1]). - --spec module(beam_utils:module_code(), [compile:option()]) -> - {'ok',beam_utils:module_code()}. - -module({Mod,Exp,Attr,Fs0,Lc}, _Opts) -> - Fs = [split_blocks(F) || F <- Fs0], - {ok,{Mod,Exp,Attr,Fs,Lc}}. - -%% We must split the basic block when we encounter instructions with labels, -%% such as catches and BIFs. All labels must be visible outside the blocks. - -split_blocks({function,Name,Arity,CLabel,Is0}) -> - Is = split_blocks(Is0, []), - {function,Name,Arity,CLabel,Is}. - -split_blocks([{block,Bl}|Is], Acc0) -> - Acc = split_block(Bl, [], Acc0), - split_blocks(Is, Acc); -split_blocks([I|Is], Acc) -> - split_blocks(Is, [I|Acc]); -split_blocks([], Acc) -> reverse(Acc). - -split_block([{set,[R],[_,_,_]=As,{bif,is_record,{f,Lbl}}}|Is], Bl, Acc) -> - %% is_record/3 must be translated by beam_clean; therefore, - %% it must be outside of any block. - split_block(Is, [], [{bif,is_record,{f,Lbl},As,R}|make_block(Bl, Acc)]); -split_block([{set,[R],As,{bif,N,{f,Lbl}=Fail}}|Is], Bl, Acc) when Lbl =/= 0 -> - split_block(Is, [], [{bif,N,Fail,As,R}|make_block(Bl, Acc)]); -split_block([{set,[],[],{line,_}=Line}, - {set,[R],As,{bif,raise,{f,_}=Fail}}|Is], Bl, Acc) -> - split_block(Is, [], [{bif,raise,Fail,As,R},Line|make_block(Bl, Acc)]); -split_block([{set,[R],As,{alloc,Live,{gc_bif,N,{f,Lbl}=Fail}}}|Is], Bl, Acc) - when Lbl =/= 0 -> - split_block(Is, [], [{gc_bif,N,Fail,Live,As,R}|make_block(Bl, Acc)]); -split_block([{set,[D],[S|Puts],{alloc,R,{put_map,Op,{f,Lbl}=Fail}}}|Is], - Bl, Acc) when Lbl =/= 0 -> - split_block(Is, [], [{put_map,Fail,Op,S,D,R,{list,Puts}}| - make_block(Bl, Acc)]); -split_block([{set,[R],[],{try_catch,Op,L}}|Is], Bl, Acc) -> - split_block(Is, [], [{Op,R,L}|make_block(Bl, Acc)]); -split_block([I|Is], Bl, Acc) -> - split_block(Is, [I|Bl], Acc); -split_block([], Bl, Acc) -> make_block(Bl, Acc). - -make_block([], Acc) -> Acc; -make_block([{set,[D],Ss,{bif,Op,Fail}}|Bl]=Bl0, Acc) -> - %% If the last instruction in the block is a comparison or boolean operator - %% (such as '=:='), move it out of the block to facilitate further - %% optimizations. - Arity = length(Ss), - case erl_internal:comp_op(Op, Arity) orelse - erl_internal:new_type_test(Op, Arity) orelse - erl_internal:bool_op(Op, Arity) of - false -> - [{block,reverse(Bl0)}|Acc]; - true -> - I = {bif,Op,Fail,Ss,D}, - case Bl =:= [] of - true -> [I|Acc]; - false -> [I,{block,reverse(Bl)}|Acc] - end - end; -make_block([{set,[Dst],[Src],move}|Bl], Acc) -> - %% Make optimization of {move,Src,Dst}, {jump,...} possible. - I = {move,Src,Dst}, - case Bl =:= [] of - true -> [I|Acc]; - false -> [I,{block,reverse(Bl)}|Acc] - end; -make_block(Bl, Acc) -> [{block,reverse(Bl)}|Acc]. diff --git a/lib/compiler/src/beam_ssa.erl b/lib/compiler/src/beam_ssa.erl new file mode 100644 index 0000000000..a9977b0b1d --- /dev/null +++ b/lib/compiler/src/beam_ssa.erl @@ -0,0 +1,890 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% Purpose: Type definitions and utilities for the SSA format. + +-module(beam_ssa). +-export([add_anno/3,get_anno/2,get_anno/3, + clobbers_xregs/1,def/2,def_used/2, + definitions/1, + dominators/1,common_dominators/3, + flatmapfold_instrs_rpo/4, + fold_po/3,fold_po/4,fold_rpo/3,fold_rpo/4, + fold_instrs_rpo/4, + linearize/1, + mapfold_blocks_rpo/4, + mapfold_instrs_rpo/4, + normalize/1, + no_side_effect/1, + predecessors/1, + rename_vars/3, + rpo/1,rpo/2, + split_blocks/3, + successors/1,successors/2, + trim_unreachable/1, + update_phi_labels/4,used/1, + uses/1,uses/2]). + +-export_type([b_module/0,b_function/0,b_blk/0,b_set/0, + b_ret/0,b_br/0,b_switch/0,terminator/0, + b_var/0,b_literal/0,b_remote/0,b_local/0, + value/0,argument/0,label/0, + var_name/0,var_base/0,literal_value/0, + op/0,anno/0,block_map/0,dominator_map/0, + rename_map/0,rename_proplist/0,usage_map/0, + definition_map/0]). + +-include("beam_ssa.hrl"). + +-type b_module() :: #b_module{}. +-type b_function() :: #b_function{}. +-type b_blk() :: #b_blk{}. +-type b_set() :: #b_set{}. + +-type b_br() :: #b_br{}. +-type b_ret() :: #b_ret{}. +-type b_switch() :: #b_switch{}. +-type terminator() :: b_br() | b_ret() | b_switch(). + +-type construct() :: b_module() | b_function() | b_blk() | b_set() | + terminator(). + +-type b_var() :: #b_var{}. +-type b_literal() :: #b_literal{}. +-type b_remote() :: #b_remote{}. +-type b_local() :: #b_local{}. + +-type value() :: b_var() | b_literal(). +-type phi_value() :: {value(),label()}. +-type argument() :: value() | b_remote() | b_local() | phi_value(). +-type label() :: non_neg_integer(). + +-type var_name() :: var_base() | {var_base(),non_neg_integer()}. +-type var_base() :: atom() | non_neg_integer(). + +-type literal_value() :: atom() | integer() | float() | list() | + nil() | tuple() | map() | binary(). + +-type op() :: {'bif',atom()} | {'float',float_op()} | prim_op() | cg_prim_op(). +-type anno() :: #{atom() := any()}. + +-type block_map() :: #{label():=b_blk()}. +-type dominator_map() :: #{label():=[label()]}. +-type numbering_map() :: #{label():=non_neg_integer()}. +-type usage_map() :: #{b_var():=[{label(),b_set() | terminator()}]}. +-type definition_map() :: #{b_var():=b_set()}. +-type rename_map() :: #{b_var():=value()}. +-type rename_proplist() :: [{b_var(),value()}]. + +%% Note: By default, dialyzer will collapse this type to atom(). +%% To avoid the collapsing, change the value of SET_LIMIT to 50 in the +%% file erl_types.erl in the hipe application. + +-type prim_op() :: 'bs_add' | 'bs_extract' | 'bs_init' | 'bs_init_writable' | + 'bs_match' | 'bs_put' | 'bs_start_match' | 'bs_test_tail' | + 'bs_utf16_size' | 'bs_utf8_size' | 'build_stacktrace' | + 'call' | 'catch_end' | + 'extract' | + 'get_hd' | 'get_map_element' | 'get_tl' | 'get_tuple_element' | + 'has_map_field' | + 'is_nonempty_list' | 'is_tagged_tuple' | + 'kill_try_tag' | + 'landingpad' | + 'make_fun' | 'new_try_tag' | + 'peek_message' | 'phi' | 'put_list' | 'put_map' | 'put_tuple' | + 'raw_raise' | 'recv_next' | 'remove_message' | 'resume' | + 'succeeded' | + 'timeout' | + 'wait' | 'wait_timeout'. + +-type float_op() :: 'checkerror' | 'clearerror' | 'convert' | 'get' | 'put' | + '+' | '-' | '*' | '/'. + +%% Primops only used internally during code generation. +-type cg_prim_op() :: 'bs_get' | 'bs_match_string' | 'bs_restore' | 'bs_skip' | + 'copy' | 'put_tuple_arity' | 'put_tuple_element' | + 'set_tuple_element'. + +-import(lists, [foldl/3,keyfind/3,mapfoldl/3,member/2,reverse/1]). + +-spec add_anno(Key, Value, Construct) -> Construct when + Key :: atom(), + Value :: any(), + Construct :: construct(). + +add_anno(Key, Val, #b_function{anno=Anno}=Bl) -> + Bl#b_function{anno=Anno#{Key=>Val}}; +add_anno(Key, Val, #b_blk{anno=Anno}=Bl) -> + Bl#b_blk{anno=Anno#{Key=>Val}}; +add_anno(Key, Val, #b_set{anno=Anno}=Bl) -> + Bl#b_set{anno=Anno#{Key=>Val}}; +add_anno(Key, Val, #b_br{anno=Anno}=Bl) -> + Bl#b_br{anno=Anno#{Key=>Val}}; +add_anno(Key, Val, #b_ret{anno=Anno}=Bl) -> + Bl#b_ret{anno=Anno#{Key=>Val}}; +add_anno(Key, Val, #b_switch{anno=Anno}=Bl) -> + Bl#b_switch{anno=Anno#{Key=>Val}}. + +-spec get_anno(atom(), construct()) -> any(). + +get_anno(Key, Construct) -> + map_get(Key, get_anno(Construct)). + +-spec get_anno(atom(), construct(),any()) -> any(). + +get_anno(Key, Construct, Default) -> + maps:get(Key, get_anno(Construct), Default). + +get_anno(#b_function{anno=Anno}) -> Anno; +get_anno(#b_blk{anno=Anno}) -> Anno; +get_anno(#b_set{anno=Anno}) -> Anno; +get_anno(#b_br{anno=Anno}) -> Anno; +get_anno(#b_ret{anno=Anno}) -> Anno; +get_anno(#b_switch{anno=Anno}) -> Anno. + +%% clobbers_xregs(#b_set{}) -> true|false. +%% Test whether the instruction invalidates all X registers. + +-spec clobbers_xregs(b_set()) -> boolean(). + +clobbers_xregs(#b_set{op=Op}) -> + case Op of + bs_init_writable -> true; + build_stacktrace -> true; + call -> true; + landingpad -> true; + make_fun -> true; + peek_message -> true; + raw_raise -> true; + _ -> false + end. + +%% no_side_effect(#b_set{}) -> true|false. +%% Test whether this instruction has no side effect and thus is safe +%% not to execute if its value is not used. Note that even if `true` +%% is returned, the instruction could still be impure (e.g. bif:get). + +-spec no_side_effect(b_set()) -> boolean(). + +no_side_effect(#b_set{op=Op}) -> + case Op of + {bif,_} -> true; + {float,get} -> true; + bs_init -> true; + bs_extract -> true; + bs_match -> true; + bs_start_match -> true; + bs_test_tail -> true; + bs_get_tail -> true; + bs_put -> true; + extract -> true; + get_hd -> true; + get_tl -> true; + get_map_element -> true; + get_tuple_element -> true; + has_map_field -> true; + is_nonempty_list -> true; + is_tagged_tuple -> true; + make_fun -> true; + put_map -> true; + put_list -> true; + put_tuple -> true; + succeeded -> true; + _ -> false + end. + +-spec predecessors(Blocks) -> #{BlockNumber:=[Predecessor]} when + Blocks :: block_map(), + BlockNumber :: label(), + Predecessor :: label(). + +predecessors(Blocks) -> + P0 = [{S,L} || {L,Blk} <- maps:to_list(Blocks), + S <- successors(Blk)], + P1 = sofs:relation(P0), + P2 = sofs:rel2fam(P1), + P3 = sofs:to_external(P2), + P = [{0,[]}|P3], + maps:from_list(P). + +-spec successors(b_blk()) -> [label()]. + +successors(#b_blk{last=Terminator}) -> + case Terminator of + #b_br{bool=#b_literal{val=true},succ=Succ} -> + [Succ]; + #b_br{bool=#b_literal{val=false},fail=Fail} -> + [Fail]; + #b_br{succ=Succ,fail=Fail} -> + [Fail,Succ]; + #b_switch{fail=Fail,list=List} -> + [Fail|[L || {_,L} <- List]]; + #b_ret{} -> + [] + end. + +%% normalize(Instr0) -> Instr. +%% Normalize instructions to help optimizations. +%% +%% For commutative operators (such as '+' and 'or'), always +%% place a variable operand before a literal operand. +%% +%% Normalize #b_br{} to one of the following forms: +%% +%% #b_br{b_literal{val=true},succ=Label,fail=Label} +%% #b_br{b_var{},succ=Label1,fail=Label2} where Label1 =/= Label2 +%% +%% Simplify a #b_switch{} with a literal argument to a #b_br{}. +%% +%% Simplify a #b_switch{} with a variable argument and an empty +%% switch list to a #b_br{}. + +-spec normalize(b_set() | terminator()) -> + b_set() | terminator(). + +normalize(#b_set{op={bif,Bif},args=Args}=Set) -> + case {is_commutative(Bif),Args} of + {false,_} -> + Set; + {true,[#b_literal{}=Lit,#b_var{}=Var]} -> + Set#b_set{args=[Var,Lit]}; + {true,_} -> + Set + end; +normalize(#b_set{}=Set) -> + Set; +normalize(#b_br{}=Br) -> + case Br of + #b_br{bool=Bool,succ=Same,fail=Same} -> + case Bool of + #b_literal{val=true} -> + Br; + _ -> + Br#b_br{bool=#b_literal{val=true}} + end; + #b_br{bool=#b_literal{val=true},succ=Succ} -> + Br#b_br{fail=Succ}; + #b_br{bool=#b_literal{val=false},fail=Fail} -> + Br#b_br{bool=#b_literal{val=true},succ=Fail}; + #b_br{} -> + Br + end; +normalize(#b_switch{arg=Arg,fail=Fail,list=List}=Sw) -> + case Arg of + #b_literal{} -> + case keyfind(Arg, 1, List) of + false -> + #b_br{bool=#b_literal{val=true},succ=Fail,fail=Fail}; + {Arg,L} -> + #b_br{bool=#b_literal{val=true},succ=L,fail=L} + end; + #b_var{} when List =:= [] -> + #b_br{bool=#b_literal{val=true},succ=Fail,fail=Fail}; + #b_var{} -> + Sw + end; +normalize(#b_ret{}=Ret) -> + Ret. + +-spec successors(label(), block_map()) -> [label()]. + +successors(L, Blocks) -> + successors(map_get(L, Blocks)). + +-spec def(Ls, Blocks) -> Def when + Ls :: [label()], + Blocks :: block_map(), + Def :: ordsets:ordset(var_name()). + +def(Ls, Blocks) -> + Top = rpo(Ls, Blocks), + Blks = [map_get(L, Blocks) || L <- Top], + def_1(Blks, []). + +-spec def_used(Ls, Blocks) -> {Def,Used} when + Ls :: [label()], + Blocks :: block_map(), + Def :: ordsets:ordset(var_name()), + Used :: ordsets:ordset(var_name()). + +def_used(Ls, Blocks) -> + Top = rpo(Ls, Blocks), + Blks = [map_get(L, Blocks) || L <- Top], + Preds = cerl_sets:from_list(Top), + def_used_1(Blks, Preds, [], []). + +%% dominators(BlockMap) -> {Dominators,Numbering}. +%% Calculate the dominator tree, returning a map where each entry +%% in the map is a list that gives the path from that block to +%% the top of the dominator tree. (Note that the suffixes of the +%% paths are shared with each other, which make the representation +%% of the dominator tree highly memory-efficient.) +%% +%% The implementation is based on: +%% +%% http://www.hipersoft.rice.edu/grads/publications/dom14.pdf +%% Cooper, Keith D.; Harvey, Timothy J; Kennedy, Ken (2001). +%% A Simple, Fast Dominance Algorithm. + +-spec dominators(Blocks) -> Result when + Blocks :: block_map(), + Result :: {dominator_map(), numbering_map()}. +dominators(Blocks) -> + Preds = predecessors(Blocks), + Top0 = rpo(Blocks), + Df = maps:from_list(number(Top0, 0)), + [{0,[]}|Top] = [{L,map_get(L, Preds)} || L <- Top0], + + %% The flow graph for an Erlang function is reducible, and + %% therefore one traversal in reverse postorder is sufficient. + Acc = #{0=>[0]}, + {dominators_1(Top, Df, Acc),Df}. + +%% common_dominators([Label], Dominators, Numbering) -> [Label]. +%% Calculate the common dominators for the given list of blocks +%% and Dominators and Numbering as returned from dominators/1. + +-spec common_dominators([label()], dominator_map(), numbering_map()) -> [label()]. +common_dominators(Ls, Dom, Numbering) -> + Doms = [map_get(L, Dom) || L <- Ls], + dom_intersection(Doms, Numbering). + +-spec fold_instrs_rpo(Fun, From, Acc0, Blocks) -> any() when + Fun :: fun((b_blk()|terminator(), any()) -> any()), + From :: [label()], + Acc0 :: any(), + Blocks :: block_map(). + +fold_instrs_rpo(Fun, From, Acc0, Blocks) -> + Top = rpo(From, Blocks), + fold_instrs_rpo_1(Top, Fun, Blocks, Acc0). + +%% Like mapfold_instrs_rpo but at the block level to support lookahead and +%% scope-dependent transformations. +-spec mapfold_blocks_rpo(Fun, From, Acc, Blocks) -> Result when + Fun :: fun((label(), b_blk(), any()) -> {b_blk(), any()}), + From :: [label()], + Acc :: any(), + Blocks :: block_map(), + Result :: {block_map(), any()}. +mapfold_blocks_rpo(Fun, From, Acc, Blocks) -> + Successors = rpo(From, Blocks), + foldl(fun(Lbl, A) -> + mapfold_blocks_rpo_1(Fun, Lbl, A) + end, {Blocks, Acc}, Successors). + +mapfold_blocks_rpo_1(Fun, Lbl, {Blocks0, Acc0}) -> + Block0 = map_get(Lbl, Blocks0), + {Block, Acc} = Fun(Lbl, Block0, Acc0), + Blocks = Blocks0#{Lbl:=Block}, + {Blocks, Acc}. + +-spec mapfold_instrs_rpo(Fun, From, Acc0, Blocks0) -> {Blocks,Acc} when + Fun :: fun((b_blk()|terminator(), any()) -> any()), + From :: [label()], + Acc0 :: any(), + Acc :: any(), + Blocks0 :: block_map(), + Blocks :: block_map(). + +mapfold_instrs_rpo(Fun, From, Acc0, Blocks) -> + Top = rpo(From, Blocks), + mapfold_instrs_rpo_1(Top, Fun, Blocks, Acc0). + +-spec flatmapfold_instrs_rpo(Fun, From, Acc0, Blocks0) -> {Blocks,Acc} when + Fun :: fun((b_blk()|terminator(), any()) -> any()), + From :: [label()], + Acc0 :: any(), + Acc :: any(), + Blocks0 :: block_map(), + Blocks :: block_map(). + +flatmapfold_instrs_rpo(Fun, From, Acc0, Blocks) -> + Top = rpo(From, Blocks), + flatmapfold_instrs_rpo_1(Top, Fun, Blocks, Acc0). + +-type fold_fun() :: fun((label(), b_blk(), any()) -> any()). + +%% fold_rpo(Fun, [Label], Acc0, Blocks) -> Acc. +%% Fold over all blocks a reverse postorder traversal of the block +%% graph; that is, first visit a block, then visit its successors. + +-spec fold_rpo(Fun, Acc0, Blocks) -> any() when + Fun :: fold_fun(), + Acc0 :: any(), + Blocks :: #{label():=b_blk()}. + +fold_rpo(Fun, Acc0, Blocks) -> + fold_rpo(Fun, [0], Acc0, Blocks). + +%% fold_rpo(Fun, [Label], Acc0, Blocks) -> Acc. Fold over all blocks +%% reachable from a given set of labels in a reverse postorder +%% traversal of the block graph; that is, first visit a block, then +%% visit its successors. + +-spec fold_rpo(Fun, Labels, Acc0, Blocks) -> any() when + Fun :: fold_fun(), + Labels :: [label()], + Acc0 :: any(), + Blocks :: #{label():=b_blk()}. + +fold_rpo(Fun, From, Acc0, Blocks) -> + Top = rpo(From, Blocks), + fold_rpo_1(Top, Fun, Blocks, Acc0). + +%% fold_po(Fun, Acc0, Blocks) -> Acc. +%% Fold over all blocks in a postorder traversal of the block graph; +%% that is, first visit all successors of block, then the block +%% itself. + +-spec fold_po(Fun, Acc0, Blocks) -> any() when + Fun :: fold_fun(), + Acc0 :: any(), + Blocks :: #{label():=b_blk()}. + +%% fold_po(Fun, From, Acc0, Blocks) -> Acc. +%% Fold over the blocks reachable from the block numbers given +%% by From in a postorder traversal of the block graph. + +fold_po(Fun, Acc0, Blocks) -> + fold_po(Fun, [0], Acc0, Blocks). + +-spec fold_po(Fun, Labels, Acc0, Blocks) -> any() when + Fun :: fold_fun(), + Labels :: [label()], + Acc0 :: any(), + Blocks :: block_map(). + +fold_po(Fun, From, Acc0, Blocks) -> + Top = reverse(rpo(From, Blocks)), + fold_rpo_1(Top, Fun, Blocks, Acc0). + +%% linearize(Blocks) -> [{BlockLabel,#b_blk{}}]. +%% Linearize the intermediate representation of the code. +%% Unreachable blocks will be discarded, and phi nodes will +%% be adjusted so that they no longer refers to discarded +%% blocks or to blocks that no longer are predecessors of +%% the phi node block. + +-spec linearize(Blocks) -> Linear when + Blocks :: block_map(), + Linear :: [{label(),b_blk()}]. + +linearize(Blocks) -> + Seen = cerl_sets:new(), + {Linear0,_} = linearize_1([0], Blocks, Seen, []), + Linear = fix_phis(Linear0, #{}), + Linear. + +-spec rpo(Blocks) -> [Label] when + Blocks :: block_map(), + Label :: label(). + +rpo(Blocks) -> + rpo([0], Blocks). + +-spec rpo(From, Blocks) -> Labels when + From :: [label()], + Blocks :: block_map(), + Labels :: [label()]. + +rpo(From, Blocks) -> + Seen = cerl_sets:new(), + {Ls,_} = rpo_1(From, Blocks, Seen, []), + Ls. + +-spec rename_vars(Rename, [label()], block_map()) -> block_map() when + Rename :: rename_map() | rename_proplist(). + +rename_vars(Rename, From, Blocks) when is_list(Rename) -> + rename_vars(maps:from_list(Rename), From, Blocks); +rename_vars(Rename, From, Blocks) when is_map(Rename)-> + Top = rpo(From, Blocks), + Preds = cerl_sets:from_list(Top), + F = fun(#b_set{op=phi,args=Args0}=Set) -> + Args = rename_phi_vars(Args0, Preds, Rename), + Set#b_set{args=Args}; + (#b_set{args=Args0}=Set) -> + Args = [rename_var(A, Rename) || A <- Args0], + Set#b_set{args=Args}; + (#b_switch{arg=Bool}=Sw) -> + Sw#b_switch{arg=rename_var(Bool, Rename)}; + (#b_br{bool=Bool}=Br) -> + Br#b_br{bool=rename_var(Bool, Rename)}; + (#b_ret{arg=Arg}=Ret) -> + Ret#b_ret{arg=rename_var(Arg, Rename)} + end, + map_instrs_1(Top, F, Blocks). + +%% split_blocks(Predicate, Blocks0, Count0) -> {Blocks,Count}. +%% Call Predicate(Instruction) for each instruction in all +%% blocks. If Predicate/1 returns true, split the block +%% before this instruction. + +-spec split_blocks(Pred, Blocks0, Count0) -> {Blocks,Count} when + Pred :: fun((b_set()) -> boolean()), + Blocks :: block_map(), + Count0 :: beam_ssa:label(), + Blocks0 :: block_map(), + Blocks :: block_map(), + Count :: beam_ssa:label(). + +split_blocks(P, Blocks, Count) -> + Ls = beam_ssa:rpo(Blocks), + split_blocks_1(Ls, P, Blocks, Count). + +-spec trim_unreachable(Blocks0) -> Blocks when + Blocks0 :: block_map(), + Blocks :: block_map(). + +%% trim_unreachable(Blocks0) -> Blocks. +%% Remove all unreachable blocks. Adjust all phi nodes so +%% they don't refer to blocks that has been removed or no +%% no longer branch to the phi node in question. + +trim_unreachable(Blocks) -> + %% Could perhaps be optimized if there is any need. + maps:from_list(linearize(Blocks)). + +%% update_phi_labels([BlockLabel], Old, New, Blocks0) -> Blocks. +%% In the given blocks, replace label Old in with New in all +%% phi nodes. This is useful after merging or splitting +%% blocks. + +-spec update_phi_labels(From, Old, New, Blocks0) -> Blocks when + From :: [label()], + Old :: label(), + New :: label(), + Blocks0 :: block_map(), + Blocks :: block_map(). + +update_phi_labels([L|Ls], Old, New, Blocks0) -> + case Blocks0 of + #{L:=#b_blk{is=[#b_set{op=phi}|_]=Is0}=Blk0} -> + Is = update_phi_labels_is(Is0, Old, New), + Blk = Blk0#b_blk{is=Is}, + Blocks = Blocks0#{L:=Blk}, + update_phi_labels(Ls, Old, New, Blocks); + #{L:=#b_blk{}} -> + %% No phi nodes in this block. + update_phi_labels(Ls, Old, New, Blocks0) + end; +update_phi_labels([], _, _, Blocks) -> Blocks. + +-spec used(b_blk() | b_set() | terminator()) -> [var_name()]. + +used(#b_blk{is=Is,last=Last}) -> + used_1([Last|Is], ordsets:new()); +used(#b_br{bool=#b_var{}=V}) -> + [V]; +used(#b_ret{arg=#b_var{}=V}) -> + [V]; +used(#b_set{op=phi,args=Args}) -> + ordsets:from_list([V || {#b_var{}=V,_} <- Args]); +used(#b_set{args=Args}) -> + ordsets:from_list(used_args(Args)); +used(#b_switch{arg=#b_var{}=V}) -> + [V]; +used(_) -> []. + +-spec definitions(Blocks :: block_map()) -> definition_map(). +definitions(Blocks) -> + fold_instrs_rpo(fun(#b_set{ dst = Var }=I, Acc) -> + Acc#{Var => I}; + (_Terminator, Acc) -> + Acc + end, [0], #{}, Blocks). + +-spec uses(Blocks :: block_map()) -> usage_map(). +uses(Blocks) -> + uses([0], Blocks). + +-spec uses(From, Blocks) -> usage_map() when + From :: [label()], + Blocks :: block_map(). +uses(From, Blocks) -> + fold_rpo(fun fold_uses_block/3, From, #{}, Blocks). + +fold_uses_block(Lbl, #b_blk{is=Is,last=Last}, UseMap0) -> + F = fun(I, UseMap) -> + foldl(fun(Var, Acc) -> + Uses0 = maps:get(Var, Acc, []), + Uses = [{Lbl, I} | Uses0], + maps:put(Var, Uses, Acc) + end, UseMap, used(I)) + end, + F(Last, foldl(F, UseMap0, Is)). + +%%% +%%% Internal functions. +%%% + +is_commutative('and') -> true; +is_commutative('or') -> true; +is_commutative('xor') -> true; +is_commutative('band') -> true; +is_commutative('bor') -> true; +is_commutative('bxor') -> true; +is_commutative('+') -> true; +is_commutative('*') -> true; +is_commutative('=:=') -> true; +is_commutative('==') -> true; +is_commutative('=/=') -> true; +is_commutative('/=') -> true; +is_commutative(_) -> false. + +def_used_1([#b_blk{is=Is,last=Last}|Bs], Preds, Def0, Used0) -> + {Def,Used1} = def_used_is(Is, Preds, Def0, Used0), + Used = ordsets:union(used(Last), Used1), + def_used_1(Bs, Preds, Def, Used); +def_used_1([], _Preds, Def, Used) -> + {ordsets:from_list(Def),Used}. + +def_used_is([#b_set{op=phi,dst=Dst,args=Args}|Is], + Preds, Def0, Used0) -> + Def = [Dst|Def0], + %% We must be careful to only include variables that will + %% be used when arriving from one of the predecessor blocks + %% in Preds. + Used1 = [V || {#b_var{}=V,L} <- Args, cerl_sets:is_element(L, Preds)], + Used = ordsets:union(ordsets:from_list(Used1), Used0), + def_used_is(Is, Preds, Def, Used); +def_used_is([#b_set{dst=Dst}=I|Is], Preds, Def0, Used0) -> + Def = [Dst|Def0], + Used = ordsets:union(used(I), Used0), + def_used_is(Is, Preds, Def, Used); +def_used_is([], _Preds, Def, Used) -> + {Def,Used}. + +def_1([#b_blk{is=Is}|Bs], Def0) -> + Def = def_is(Is, Def0), + def_1(Bs, Def); +def_1([], Def) -> + ordsets:from_list(Def). + +def_is([#b_set{dst=Dst}|Is], Def) -> + def_is(Is, [Dst|Def]); +def_is([], Def) -> Def. + +dominators_1([{L,Preds}|Ls], Df, Doms) -> + DomPreds = [map_get(P, Doms) || P <- Preds, is_map_key(P, Doms)], + Dom = [L|dom_intersection(DomPreds, Df)], + dominators_1(Ls, Df, Doms#{L=>Dom}); +dominators_1([], _Df, Doms) -> Doms. + +dom_intersection([S], _Df) -> + S; +dom_intersection([S|Ss], Df) -> + dom_intersection(S, Ss, Df). + +dom_intersection(S1, [S2|Ss], Df) -> + dom_intersection(dom_intersection_1(S1, S2, Df), Ss, Df); +dom_intersection(S, [], _Df) -> S. + +dom_intersection_1([E1|Es1]=Set1, [E2|Es2]=Set2, Df) -> + %% Blocks are numbered in the order they are found in + %% reverse postorder. + #{E1:=Df1,E2:=Df2} = Df, + if Df1 > Df2 -> + dom_intersection_1(Es1, Set2, Df); + Df2 > Df1 -> + dom_intersection_1(Es2, Set1, Df); %switch arguments! + true -> %Set1 == Set2 + %% The common suffix of the sets is the intersection. + Set1 + end. + +number([L|Ls], N) -> + [{L,N}|number(Ls, N+1)]; +number([], _) -> []. + +fold_rpo_1([L|Ls], Fun, Blocks, Acc0) -> + Block = map_get(L, Blocks), + Acc = Fun(L, Block, Acc0), + fold_rpo_1(Ls, Fun, Blocks, Acc); +fold_rpo_1([], _, _, Acc) -> Acc. + +fold_instrs_rpo_1([L|Ls], Fun, Blocks, Acc0) -> + #b_blk{is=Is,last=Last} = map_get(L, Blocks), + Acc1 = foldl(Fun, Acc0, Is), + Acc = Fun(Last, Acc1), + fold_instrs_rpo_1(Ls, Fun, Blocks, Acc); +fold_instrs_rpo_1([], _, _, Acc) -> Acc. + +mapfold_instrs_rpo_1([L|Ls], Fun, Blocks0, Acc0) -> + #b_blk{is=Is0,last=Last0} = Block0 = map_get(L, Blocks0), + {Is,Acc1} = mapfoldl(Fun, Acc0, Is0), + {Last,Acc} = Fun(Last0, Acc1), + Block = Block0#b_blk{is=Is,last=Last}, + Blocks = Blocks0#{L:=Block}, + mapfold_instrs_rpo_1(Ls, Fun, Blocks, Acc); +mapfold_instrs_rpo_1([], _, Blocks, Acc) -> + {Blocks,Acc}. + +flatmapfold_instrs_rpo_1([L|Ls], Fun, Blocks0, Acc0) -> + #b_blk{is=Is0,last=Last0} = Block0 = map_get(L, Blocks0), + {Is,Acc1} = flatmapfoldl(Fun, Acc0, Is0), + {[Last],Acc} = Fun(Last0, Acc1), + Block = Block0#b_blk{is=Is,last=Last}, + Blocks = Blocks0#{L:=Block}, + flatmapfold_instrs_rpo_1(Ls, Fun, Blocks, Acc); +flatmapfold_instrs_rpo_1([], _, Blocks, Acc) -> + {Blocks,Acc}. + +linearize_1([L|Ls], Blocks, Seen0, Acc0) -> + case cerl_sets:is_element(L, Seen0) of + true -> + linearize_1(Ls, Blocks, Seen0, Acc0); + false -> + Seen1 = cerl_sets:add_element(L, Seen0), + Block = map_get(L, Blocks), + Successors = successors(Block), + {Acc,Seen} = linearize_1(Successors, Blocks, Seen1, Acc0), + linearize_1(Ls, Blocks, Seen, [{L,Block}|Acc]) + end; +linearize_1([], _, Seen, Acc) -> + {Acc,Seen}. + +fix_phis([{L,Blk0}|Bs], S) -> + Blk = case Blk0 of + #b_blk{is=[#b_set{op=phi}|_]=Is0} -> + Is = fix_phis_1(Is0, L, S), + Blk0#b_blk{is=Is}; + #b_blk{} -> + Blk0 + end, + Successors = successors(Blk), + [{L,Blk}|fix_phis(Bs, S#{L=>Successors})]; +fix_phis([], _) -> []. + +fix_phis_1([#b_set{op=phi,args=Args0}=I|Is], L, S) -> + Args = [{Val,Pred} || {Val,Pred} <- Args0, + is_successor(L, Pred, S)], + [I#b_set{args=Args}|fix_phis_1(Is, L, S)]; +fix_phis_1(Is, _, _) -> Is. + +is_successor(L, Pred, S) -> + case S of + #{Pred:=Successors} -> + member(L, Successors); + #{} -> + %% This block has been removed. + false + end. + +rpo_1([L|Ls], Blocks, Seen0, Acc0) -> + case cerl_sets:is_element(L, Seen0) of + true -> + rpo_1(Ls, Blocks, Seen0, Acc0); + false -> + Block = map_get(L, Blocks), + Seen1 = cerl_sets:add_element(L, Seen0), + Successors = successors(Block), + {Acc,Seen} = rpo_1(Successors, Blocks, Seen1, Acc0), + rpo_1(Ls, Blocks, Seen, [L|Acc]) + end; +rpo_1([], _, Seen, Acc) -> + {Acc,Seen}. + +rename_var(#b_var{}=Old, Rename) -> + case Rename of + #{Old:=New} -> New; + #{} -> Old + end; +rename_var(#b_remote{mod=Mod0,name=Name0}=Remote, Rename) -> + Mod = rename_var(Mod0, Rename), + Name = rename_var(Name0, Rename), + Remote#b_remote{mod=Mod,name=Name}; +rename_var(Old, _) -> Old. + +rename_phi_vars([{Var,L}|As], Preds, Ren) -> + case cerl_sets:is_element(L, Preds) of + true -> + [{rename_var(Var, Ren),L}|rename_phi_vars(As, Preds, Ren)]; + false -> + [{Var,L}|rename_phi_vars(As, Preds, Ren)] + end; +rename_phi_vars([], _, _) -> []. + +map_instrs_1([L|Ls], Fun, Blocks0) -> + #b_blk{is=Is0,last=Last0} = Blk0 = map_get(L, Blocks0), + Is = [Fun(I) || I <- Is0], + Last = Fun(Last0), + Blk = Blk0#b_blk{is=Is,last=Last}, + Blocks = Blocks0#{L:=Blk}, + map_instrs_1(Ls, Fun, Blocks); +map_instrs_1([], _, Blocks) -> Blocks. + +flatmapfoldl(F, Accu0, [Hd|Tail]) -> + {R,Accu1} = F(Hd, Accu0), + {Rs,Accu2} = flatmapfoldl(F, Accu1, Tail), + {R++Rs,Accu2}; +flatmapfoldl(_, Accu, []) -> {[],Accu}. + +split_blocks_1([L|Ls], P, Blocks0, Count0) -> + #b_blk{is=Is0} = Blk = map_get(L, Blocks0), + case split_blocks_is(Is0, P, []) of + {yes,Bef,Aft} -> + NewLbl = Count0, + Count = Count0 + 1, + Br = #b_br{bool=#b_literal{val=true},succ=NewLbl,fail=NewLbl}, + BefBlk = Blk#b_blk{is=Bef,last=Br}, + NewBlk = Blk#b_blk{is=Aft}, + Blocks1 = Blocks0#{L:=BefBlk,NewLbl=>NewBlk}, + Successors = successors(NewBlk), + Blocks = update_phi_labels(Successors, L, NewLbl, Blocks1), + split_blocks_1([NewLbl|Ls], P, Blocks, Count); + no -> + split_blocks_1(Ls, P, Blocks0, Count0) + end; +split_blocks_1([], _, Blocks, Count) -> + {Blocks,Count}. + +split_blocks_is([I|Is], P, []) -> + split_blocks_is(Is, P, [I]); +split_blocks_is([I|Is], P, Acc) -> + case P(I) of + true -> + {yes,reverse(Acc),[I|Is]}; + false -> + split_blocks_is(Is, P, [I|Acc]) + end; +split_blocks_is([], _, _) -> no. + +update_phi_labels_is([#b_set{op=phi,args=Args0}=I0|Is], Old, New) -> + Args = [{Arg,rename_label(Lbl, Old, New)} || {Arg,Lbl} <- Args0], + I = I0#b_set{args=Args}, + [I|update_phi_labels_is(Is, Old, New)]; +update_phi_labels_is(Is, _, _) -> Is. + +rename_label(Old, Old, New) -> New; +rename_label(Lbl, _Old, _New) -> Lbl. + +used_args([#b_var{}=V|As]) -> + [V|used_args(As)]; +used_args([#b_remote{mod=Mod,name=Name}|As]) -> + used_args([Mod,Name|As]); +used_args([_|As]) -> + used_args(As); +used_args([]) -> []. + +used_1([H|T], Used0) -> + Used = ordsets:union(used(H), Used0), + used_1(T, Used); +used_1([], Used) -> Used. diff --git a/lib/compiler/src/beam_ssa.hrl b/lib/compiler/src/beam_ssa.hrl new file mode 100644 index 0000000000..fa76b08453 --- /dev/null +++ b/lib/compiler/src/beam_ssa.hrl @@ -0,0 +1,66 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-record(b_module, {anno=#{} :: beam_ssa:anno(), + name :: module(), + exports :: [{atom(),arity()}], + attributes :: list(), + body :: [beam_ssa:b_function()]}). +-record(b_function, {anno=#{} :: beam_ssa:anno(), + args :: [beam_ssa:b_var()], + bs :: #{beam_ssa:label():=beam_ssa:b_blk()}, + cnt :: beam_ssa:label()}). + +-record(b_blk, {anno=#{} :: beam_ssa:anno(), + is :: [beam_ssa:b_set()], + last :: beam_ssa:terminator()}). +-record(b_set, {anno=#{} :: beam_ssa:anno(), + dst=none :: 'none'|beam_ssa:b_var(), + op :: beam_ssa:op(), + args=[] :: [beam_ssa:argument()]}). + +%% Terminators. +-record(b_ret, {anno=#{} :: beam_ssa:anno(), + arg :: beam_ssa:value()}). + +-record(b_br, {anno=#{}, + bool :: beam_ssa:value(), + succ :: beam_ssa:label(), + fail :: beam_ssa:label()}). + +-record(b_switch, {anno=#{} :: beam_ssa:anno(), + arg :: beam_ssa:value(), + fail :: beam_ssa:label(), + list :: [{beam_ssa:b_literal(),beam_ssa:label()}]}). + +%% Values. +-record(b_var, {name :: beam_ssa:var_name()}). + +-record(b_literal, {val :: beam_ssa:literal_value()}). + +-record(b_remote, {mod :: beam_ssa:value(), + name :: beam_ssa:value(), + arity :: non_neg_integer()}). + +-record(b_local, {name :: beam_ssa:b_literal(), + arity :: non_neg_integer()}). + +%% If this block exists, it calls erlang:error(badarg). +-define(BADARG_BLOCK, 1). diff --git a/lib/compiler/src/beam_ssa_bsm.erl b/lib/compiler/src/beam_ssa_bsm.erl new file mode 100644 index 0000000000..382e6f635e --- /dev/null +++ b/lib/compiler/src/beam_ssa_bsm.erl @@ -0,0 +1,1046 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%%% +%%% This pass optimizes bit syntax matching, and is centered around the concept +%%% of "match context reuse" which is best explained through example. To put it +%%% shortly we attempt to turn this: +%%% +%%% <<0,B/bits>> = A, +%%% <<1,C/bits>> = B, +%%% <<D,_/bits>> = C, +%%% D. +%%% +%%% ... Into this: +%%% +%%% <<0,1,D,_/bits>>=A, +%%% D. +%%% +%%% Which is much faster as it avoids the creation of intermediate terms. This +%%% is especially noticeable in loops where such garbage is generated on each +%%% iteration. +%%% +%%% The optimization itself is very simple and can be applied whenever there's +%%% matching on the tail end of a binary; instead of creating a new binary and +%%% starting a new match context on it, we reuse the match context used to +%%% extract the tail and avoid the creation of both objects. +%%% +%%% The catch is that a match context isn't a proper type and nothing outside +%%% of bit syntax match operations can handle them. We therefore need to make +%%% sure that they never "leak" into other instructions, and most of the pass +%%% revolves around getting around that limitation. +%%% +%%% Unlike most other passes we look at the whole module so we can combine +%%% matches across function boundaries, greatly increasing the performance of +%%% complex matches and loops. +%%% + +-module(beam_ssa_bsm). + +-export([module/2, format_error/1]). + +-include("beam_ssa.hrl"). + +-import(lists, [member/2, reverse/1, splitwith/2, map/2, foldl/3, mapfoldl/3, + nth/2, max/1, unzip/1]). + +-spec format_error(term()) -> nonempty_string(). + +format_error(OptInfo) -> + format_opt_info(OptInfo). + +-spec module(Module, Options) -> Result when + Module :: beam_ssa:b_module(), + Options :: [compile:option()], + Result :: {ok, beam_ssa:b_module(), list()}. + +-define(PASS(N), {N,fun N/1}). + +module(#b_module{body=Fs0}=Module, Opts) -> + ModInfo = analyze_module(Module), + + %% combine_matches is repeated after accept_context_args as the control + %% flow changes can enable further optimizations, as in the example below: + %% + %% a(<<0,X/binary>>) -> a(X); + %% a(A) when bit_size(A) =:= 52 -> bar; + %% a(<<1,X/binary>>) -> X. %% Match context will be reused here when + %% %% when repeated. + + {Fs, _} = compile:run_sub_passes( + [?PASS(combine_matches), + ?PASS(accept_context_args), + ?PASS(combine_matches), + ?PASS(allow_context_passthrough), + ?PASS(skip_outgoing_tail_extraction), + ?PASS(annotate_context_parameters)], + {Fs0, ModInfo}), + + Ws = case proplists:get_bool(bin_opt_info, Opts) of + true -> collect_opt_info(Fs); + false -> [] + end, + + {ok, Module#b_module{body=Fs}, Ws}. + +-type module_info() :: #{ func_id() => func_info() }. + +-type func_id() :: {Name :: atom(), Arity :: non_neg_integer()}. + +-type func_info() :: #{ has_bsm_ops => boolean(), + parameters => [#b_var{}], + parameter_info => #{ #b_var{} => param_info() } }. + +-type param_info() :: suitable_for_reuse | + {Problem :: atom(), Where :: term()}. + +-spec analyze_module(#b_module{}) -> module_info(). + +analyze_module(#b_module{body=Fs}) -> + foldl(fun(#b_function{args=Parameters}=F, I) -> + FuncInfo = #{ has_bsm_ops => has_bsm_ops(F), + parameters => Parameters, + parameter_info => #{} }, + FuncId = get_fa(F), + I#{ FuncId => FuncInfo } + end, #{}, Fs). + +has_bsm_ops(#b_function{bs=Blocks}) -> + hbo_blocks(maps:to_list(Blocks)). + +hbo_blocks([{_,#b_blk{is=Is}} | Blocks]) -> + case hbo_is(Is) of + false -> hbo_blocks(Blocks); + true -> true + end; +hbo_blocks([]) -> + false. + +hbo_is([#b_set{op=bs_start_match} | _]) -> true; +hbo_is([_I | Is]) -> hbo_is(Is); +hbo_is([]) -> false. + +%% Checks whether it's legal to make a call with the given argument as a match +%% context, returning the param_info() of the relevant parameter. +-spec check_context_call(#b_set{}, Arg, CtxChain, ModInfo) -> param_info() when + Arg :: #b_var{}, + CtxChain :: [#b_var{}], + ModInfo :: module_info(). +check_context_call(#b_set{args=Args}, Arg, CtxChain, ModInfo) -> + Aliases = [Arg | CtxChain], + ccc_1(Args, Arg, Aliases, ModInfo). + +ccc_1([#b_local{}=Call | Args], Ctx, Aliases, ModInfo) -> + %% Matching operations assume that their context isn't aliased (as in + %% pointer aliasing), so we must reject calls whose arguments contain more + %% than one reference to the context. + %% + %% TODO: Try to fall back to passing binaries in these cases. Partial reuse + %% is better than nothing. + UseCount = foldl(fun(Arg, C) -> + case member(Arg, Aliases) of + true -> C + 1; + false -> C + end + end, 0, Args), + if + UseCount =:= 1 -> + #b_local{name=#b_literal{val=Name},arity=Arity} = Call, + Callee = {Name, Arity}, + + ParamInfo = funcinfo_get(Callee, parameter_info, ModInfo), + Parameters = funcinfo_get(Callee, parameters, ModInfo), + Parameter = nth(1 + arg_index(Ctx, Args), Parameters), + + case maps:find(Parameter, ParamInfo) of + {ok, suitable_for_reuse} -> + suitable_for_reuse; + {ok, Other} -> + {unsuitable_call, {Call, Other}}; + error -> + {no_match_on_entry, Call} + end; + UseCount > 1 -> + {multiple_uses_in_call, Call} + end; +ccc_1([#b_remote{}=Call | _Args], _Ctx, _CtxChain, _ModInfo) -> + {remote_call, Call}; +ccc_1([Fun | _Args], _Ctx, _CtxChain, _ModInfo) -> + %% TODO: It may be possible to support this in the future for locally + %% defined funs, including ones with free variables. + {fun_call, Fun}. + +%% Returns the index of Var in Args. +arg_index(Var, Args) -> arg_index_1(Var, Args, 0). + +arg_index_1(Var, [Var | _Args], Index) -> Index; +arg_index_1(Var, [_Arg | Args], Index) -> arg_index_1(Var, Args, Index + 1). + +is_tail_binary(#b_set{op=bs_match,args=[#b_literal{val=binary} | Rest]}) -> + member(#b_literal{val=all}, Rest); +is_tail_binary(#b_set{op=bs_get_tail}) -> + true; +is_tail_binary(_) -> + false. + +is_tail_binary(#b_var{}=Var, Defs) -> + case find_match_definition(Var, Defs) of + {ok, Def} -> is_tail_binary(Def); + _ -> false + end; +is_tail_binary(_Literal, _Defs) -> + false. + +assert_match_context(#b_var{}=Var, Defs) -> + case maps:find(Var, Defs) of + {ok, #b_set{op=bs_match,args=[_,#b_var{}=Ctx|_]}} -> + assert_match_context(Ctx, Defs); + {ok, #b_set{op=bs_start_match}} -> + ok + end. + +find_match_definition(#b_var{}=Var, Defs) -> + case maps:find(Var, Defs) of + {ok, #b_set{op=bs_extract,args=[Ctx]}} -> maps:find(Ctx, Defs); + {ok, #b_set{op=bs_get_tail}=Def} -> {ok, Def}; + _ -> error + end. + +%% Returns a list of all contexts that were used to extract Var. +context_chain_of(#b_var{}=Var, Defs) -> + case maps:find(Var, Defs) of + {ok, #b_set{op=bs_match,args=[_,#b_var{}=Ctx|_]}} -> + [Ctx | context_chain_of(Ctx, Defs)]; + {ok, #b_set{op=bs_get_tail,args=[Ctx]}} -> + [Ctx | context_chain_of(Ctx, Defs)]; + {ok, #b_set{op=bs_extract,args=[Ctx]}} -> + [Ctx | context_chain_of(Ctx, Defs)]; + _ -> + [] + end. + +%% Grabs the match context used to produce the given variable. +match_context_of(#b_var{}=Var, Defs) -> + Ctx = match_context_of_1(Var, Defs), + assert_match_context(Ctx, Defs), + Ctx. + +match_context_of_1(Var, Defs) -> + case maps:get(Var, Defs) of + #b_set{op=bs_extract,args=[#b_var{}=Ctx0]} -> + #b_set{op=bs_match, + args=[_,#b_var{}=Ctx|_]} = maps:get(Ctx0, Defs), + Ctx; + #b_set{op=bs_get_tail,args=[#b_var{}=Ctx]} -> + Ctx + end. + +funcinfo_get(#b_function{}=F, Attribute, ModInfo) -> + funcinfo_get(get_fa(F), Attribute, ModInfo); +funcinfo_get({_,_}=Key, Attribute, ModInfo) -> + FuncInfo = maps:get(Key, ModInfo), + maps:get(Attribute, FuncInfo). + +funcinfo_set(#b_function{}=F, Attribute, Value, ModInfo) -> + funcinfo_set(get_fa(F), Attribute, Value, ModInfo); +funcinfo_set(Key, Attribute, Value, ModInfo) -> + FuncInfo = maps:put(Attribute, Value, maps:get(Key, ModInfo, #{})), + maps:put(Key, FuncInfo, ModInfo). + +get_fa(#b_function{ anno = Anno }) -> + {_,Name,Arity} = maps:get(func_info, Anno), + {Name,Arity}. + +%% Replaces matched-out binaries with aliases that are lazily converted to +%% binary form when used, allowing us to keep the "match path" free of binary +%% creation. + +-spec alias_matched_binaries(Blocks, Counter, AliasMap) -> Result when + Blocks :: beam_ssa:block_map(), + Counter :: non_neg_integer(), + AliasMap :: match_alias_map(), + Result :: {Blocks, Counter}. + +-type match_alias_map() :: + #{ Binary :: #b_var{} => + { %% Replace all uses of Binary with an alias after this + %% label. + AliasAfter :: beam_ssa:label(), + %% The match context whose tail is equal to Binary. + Context :: #b_var{} } }. + +%% Keeps track of the promotions we need to insert. They're partially keyed by +%% location because they may not be valid on all execution paths and we may +%% need to add redundant promotions in some cases. +-type promotion_map() :: + #{ { PromoteAt :: beam_ssa:label(), + Variable :: #b_var{} } => + Instruction :: #b_set{} }. + +-record(amb, { dominators :: beam_ssa:dominator_map(), + match_aliases :: match_alias_map(), + cnt :: non_neg_integer(), + promotions = #{} :: promotion_map() }). + +alias_matched_binaries(Blocks0, Counter, AliasMap) when AliasMap =/= #{} -> + {Dominators, _} = beam_ssa:dominators(Blocks0), + State0 = #amb{ dominators = Dominators, + match_aliases = AliasMap, + cnt = Counter }, + {Blocks, State} = beam_ssa:mapfold_blocks_rpo(fun amb_1/3, [0], State0, + Blocks0), + {amb_insert_promotions(Blocks, State), State#amb.cnt}; +alias_matched_binaries(Blocks, Counter, _AliasMap) -> + {Blocks, Counter}. + +amb_1(Lbl, #b_blk{is=Is0,last=Last0}=Block, State0) -> + {Is, State1} = mapfoldl(fun(I, State) -> + amb_assign_set(I, Lbl, State) + end, State0, Is0), + {Last, State} = amb_assign_last(Last0, Lbl, State1), + {Block#b_blk{is=Is,last=Last}, State}. + +amb_assign_set(#b_set{op=phi,args=Args0}=I, _Lbl, State0) -> + %% Phi node aliases are relative to their source block, not their + %% containing block. + {Args, State} = + mapfoldl(fun({Arg0, Lbl}, Acc) -> + {Arg, State} = amb_get_alias(Arg0, Lbl, Acc), + {{Arg, Lbl}, State} + end, State0, Args0), + {I#b_set{args=Args}, State}; +amb_assign_set(#b_set{args=Args0}=I, Lbl, State0) -> + {Args, State} = mapfoldl(fun(Arg0, Acc) -> + amb_get_alias(Arg0, Lbl, Acc) + end, State0, Args0), + {I#b_set{args=Args}, State}. + +amb_assign_last(#b_ret{arg=Arg0}=T, Lbl, State0) -> + {Arg, State} = amb_get_alias(Arg0, Lbl, State0), + {T#b_ret{arg=Arg}, State}; +amb_assign_last(#b_switch{arg=Arg0}=T, Lbl, State0) -> + {Arg, State} = amb_get_alias(Arg0, Lbl, State0), + {T#b_switch{arg=Arg}, State}; +amb_assign_last(#b_br{bool=Arg0}=T, Lbl, State0) -> + {Arg, State} = amb_get_alias(Arg0, Lbl, State0), + {T#b_br{bool=Arg}, State}. + +amb_get_alias(#b_var{}=Arg, Lbl, State) -> + case maps:find(Arg, State#amb.match_aliases) of + {ok, {AliasAfter, Context}} -> + %% Our context may not have been created yet, so we skip assigning + %% an alias unless the given block is among our dominators. + Dominators = maps:get(Lbl, State#amb.dominators), + case member(AliasAfter, Dominators) of + true -> amb_create_alias(Arg, Context, Lbl, State); + false -> {Arg, State} + end; + error -> + {Arg, State} + end; +amb_get_alias(#b_remote{mod=Mod0,name=Name0}=Arg0, Lbl, State0) -> + {Mod, State1} = amb_get_alias(Mod0, Lbl, State0), + {Name, State} = amb_get_alias(Name0, Lbl, State1), + Arg = Arg0#b_remote{mod=Mod,name=Name}, + {Arg, State}; +amb_get_alias(Arg, _Lbl, State) -> + {Arg, State}. + +amb_create_alias(#b_var{}=Arg0, Context, Lbl, State0) -> + Dominators = maps:get(Lbl, State0#amb.dominators), + Promotions0 = State0#amb.promotions, + + PrevPromotions = + [maps:get({Dom, Arg0}, Promotions0) + || Dom <- Dominators, is_map_key({Dom, Arg0}, Promotions0)], + + case PrevPromotions of + [_|_] -> + %% We've already created an alias prior to this block, so we'll + %% grab the most recent one to minimize stack use. + + #b_set{dst=Alias} = max(PrevPromotions), + {Alias, State0}; + [] -> + %% If we haven't created an alias we need to do so now. The + %% promotion will be inserted later by amb_insert_promotions/2. + + Counter = State0#amb.cnt, + Alias = #b_var{name={'@ssa_bsm_alias', Counter}}, + Promotion = #b_set{op=bs_get_tail,dst=Alias,args=[Context]}, + + Promotions = maps:put({Lbl, Arg0}, Promotion, Promotions0), + State = State0#amb{ promotions=Promotions, cnt=Counter+1 }, + + {Alias, State} + end. + +amb_insert_promotions(Blocks0, State) -> + F = fun({Lbl, #b_var{}}, Promotion, Blocks) -> + Block = maps:get(Lbl, Blocks), + + Alias = Promotion#b_set.dst, + {Before, After} = splitwith( + fun(#b_set{args=Args}) -> + not is_var_in_args(Alias, Args) + end, Block#b_blk.is), + Is = Before ++ [Promotion | After], + + maps:put(Lbl, Block#b_blk{is=Is}, Blocks) + end, + maps:fold(F, Blocks0, State#amb.promotions). + +is_var_in_args(Var, [Var | _]) -> true; +is_var_in_args(Var, [#b_remote{name=Var} | _]) -> true; +is_var_in_args(Var, [#b_remote{mod=Var} | _]) -> true; +is_var_in_args(Var, [_ | Args]) -> is_var_in_args(Var, Args); +is_var_in_args(_Var, []) -> false. + +%%% +%%% Subpasses +%%% + +%% Removes superflous chained bs_start_match instructions in the same +%% function. When matching on an extracted tail binary, or on a binary we've +%% already matched on, we reuse the original match context. +%% +%% This pass runs first since it makes subsequent optimizations more effective +%% by removing spots where promotion would be required. + +-type prior_match_map() :: + #{ Binary :: #b_var{} => + [{ %% The context and success label of a previous + %% bs_start_match made on this binary. + ValidAfter :: beam_ssa:label(), + Context :: #b_var{} }] }. + +-record(cm, { definitions :: beam_ssa:definition_map(), + dominators :: beam_ssa:dominator_map(), + blocks :: beam_ssa:block_map(), + match_aliases = #{} :: match_alias_map(), + prior_matches = #{} :: prior_match_map(), + renames = #{} :: beam_ssa:rename_map() }). + +combine_matches({Fs0, ModInfo}) -> + Fs = map(fun(F) -> combine_matches(F, ModInfo) end, Fs0), + {Fs, ModInfo}. + +combine_matches(#b_function{bs=Blocks0,cnt=Counter0}=F, ModInfo) -> + case funcinfo_get(F, has_bsm_ops, ModInfo) of + true -> + {Dominators, _} = beam_ssa:dominators(Blocks0), + {Blocks1, State} = + beam_ssa:mapfold_blocks_rpo( + fun(Lbl, #b_blk{is=Is0}=Block0, State0) -> + {Is, State} = cm_1(Is0, [], Lbl, State0), + {Block0#b_blk{is=Is}, State} + end, [0], + #cm{ definitions = beam_ssa:definitions(Blocks0), + dominators = Dominators, + blocks = Blocks0 }, + Blocks0), + + Blocks2 = beam_ssa:rename_vars(State#cm.renames, [0], Blocks1), + + {Blocks, Counter} = alias_matched_binaries(Blocks2, Counter0, + State#cm.match_aliases), + + F#b_function{ bs=Blocks, cnt=Counter }; + false -> + F + end. + +cm_1([#b_set{ op=bs_start_match, + dst=Ctx, + args=[Src] }, + #b_set{ op=succeeded, + dst=Bool, + args=[Ctx] }]=MatchSeq, Acc0, Lbl, State0) -> + Acc = reverse(Acc0), + case is_tail_binary(Src, State0#cm.definitions) of + true -> cm_combine_tail(Src, Ctx, Bool, Acc, State0); + false -> cm_handle_priors(Src, Ctx, Bool, Acc, MatchSeq, Lbl, State0) + end; +cm_1([I | Is], Acc, Lbl, State) -> + cm_1(Is, [I | Acc], Lbl, State); +cm_1([], Acc, _Lbl, State) -> + {reverse(Acc), State}. + +%% If we're dominated by at least one match on the same source, we can reuse +%% the context created by that match. +cm_handle_priors(Src, DstCtx, Bool, Acc, MatchSeq, Lbl, State0) -> + PriorCtxs = case maps:find(Src, State0#cm.prior_matches) of + {ok, Priors} -> + %% We've seen other match contexts on this source, but + %% we can only consider the ones whose success path + %% dominate us. + Dominators = maps:get(Lbl, State0#cm.dominators, []), + [Ctx || {ValidAfter, Ctx} <- Priors, + member(ValidAfter, Dominators)]; + error -> + [] + end, + case PriorCtxs of + [Ctx|_] -> + Renames0 = State0#cm.renames, + Renames = Renames0#{ Bool => #b_literal{val=true}, DstCtx => Ctx }, + {Acc, State0#cm{ renames = Renames }}; + [] -> + %% Since we lack a prior match, we need to register this one in + %% case we dominate another. + State = cm_register_prior(Src, DstCtx, Lbl, State0), + {Acc ++ MatchSeq, State} + end. + +cm_register_prior(Src, DstCtx, Lbl, State) -> + Block = maps:get(Lbl, State#cm.blocks), + #b_br{succ=ValidAfter} = Block#b_blk.last, + + Priors0 = maps:get(Src, State#cm.prior_matches, []), + Priors = [{ValidAfter, DstCtx} | Priors0], + + PriorMatches = maps:put(Src, Priors, State#cm.prior_matches), + State#cm{ prior_matches = PriorMatches }. + +cm_combine_tail(Src, DstCtx, Bool, Acc, State0) -> + SrcCtx = match_context_of(Src, State0#cm.definitions), + + %% We replace the source with a context alias as it normally won't be used + %% on the happy path after being matched, and the added cost of conversion + %% is negligible if it is. + Aliases = maps:put(Src, {0, SrcCtx}, State0#cm.match_aliases), + + Renames0 = State0#cm.renames, + Renames = Renames0#{ Bool => #b_literal{val=true}, DstCtx => SrcCtx }, + + State = State0#cm{ match_aliases = Aliases, renames = Renames }, + + {Acc, State}. + +%% Lets functions accept match contexts as arguments. The parameter must be +%% unused before the bs_start_match instruction, and it must be matched in the +%% first block. + +-record(aca, { unused_parameters :: ordsets:ordset(#b_var{}), + counter :: non_neg_integer(), + parameter_info = #{} :: #{ #b_var{} => param_info() }, + match_aliases = #{} :: match_alias_map() }). + +accept_context_args({Fs, ModInfo}) -> + mapfoldl(fun accept_context_args/2, ModInfo, Fs). + +accept_context_args(#b_function{bs=Blocks0}=F, ModInfo0) -> + case funcinfo_get(F, has_bsm_ops, ModInfo0) of + true -> + Parameters = ordsets:from_list(funcinfo_get(F, parameters, ModInfo0)), + State0 = #aca{ unused_parameters = Parameters, + counter = F#b_function.cnt }, + + {Blocks1, State} = aca_1(Blocks0, State0), + {Blocks, Counter} = alias_matched_binaries(Blocks1, + State#aca.counter, + State#aca.match_aliases), + + ModInfo = funcinfo_set(F, parameter_info, State#aca.parameter_info, + ModInfo0), + + {F#b_function{bs=Blocks,cnt=Counter}, ModInfo}; + false -> + {F, ModInfo0} + end. + +aca_1(Blocks, State) -> + %% We only handle block 0 as we don't yet support starting a match after a + %% test. This is generally good enough as the sys_core_bsm pass makes the + %% match instruction come first if possible, and it's rare for a function + %% to binary-match several parameters at once. + EntryBlock = maps:get(0, Blocks), + aca_enable_reuse(EntryBlock#b_blk.is, EntryBlock, Blocks, [], State). + +aca_enable_reuse([#b_set{op=bs_start_match,args=[Src]}=I0 | Rest], + EntryBlock, Blocks0, Acc, State0) -> + case aca_is_reuse_safe(Src, State0) of + true -> + {I, Last, Blocks1, State} = + aca_reuse_context(I0, EntryBlock, Blocks0, State0), + + Is = reverse([I|Acc]) ++ Rest, + Blocks = maps:put(0, EntryBlock#b_blk{is=Is,last=Last}, Blocks1), + + {Blocks, State}; + false -> + {Blocks0, State0} + end; +aca_enable_reuse([I | Is], EntryBlock, Blocks, Acc, State0) -> + UnusedParams0 = State0#aca.unused_parameters, + case ordsets:intersection(UnusedParams0, beam_ssa:used(I)) of + [] -> + aca_enable_reuse(Is, EntryBlock, Blocks, [I | Acc], State0); + PrematureUses -> + UnusedParams = ordsets:subtract(UnusedParams0, PrematureUses), + + %% Mark the offending parameters as unsuitable for context reuse. + ParamInfo = foldl(fun(A, Ps) -> + maps:put(A, {used_before_match, I}, Ps) + end, State0#aca.parameter_info, PrematureUses), + + State = State0#aca{ unused_parameters = UnusedParams, + parameter_info = ParamInfo }, + aca_enable_reuse(Is, EntryBlock, Blocks, [I | Acc], State) + end; +aca_enable_reuse([], _EntryBlock, Blocks, _Acc, State) -> + {Blocks, State}. + +aca_is_reuse_safe(Src, State) -> + %% Context reuse is unsafe unless all uses are dominated by the start_match + %% instruction. Since we only process block 0 it's enough to check if + %% they're unused so far. + ordsets:is_element(Src, State#aca.unused_parameters). + +aca_reuse_context(#b_set{dst=Dst, args=[Src]}=I0, Block, Blocks0, State0) -> + %% When matching fails on a reused context it needs to be converted back + %% to a binary. We only need to do this on the success path since it can't + %% be a context on the type failure path, but it's very common for these + %% to converge which requires special handling. + {State1, Last, Blocks} = + aca_handle_convergence(Src, State0, Block#b_blk.last, Blocks0), + + Aliases = maps:put(Src, {Last#b_br.succ, Dst}, State1#aca.match_aliases), + ParamInfo = maps:put(Src, suitable_for_reuse, State1#aca.parameter_info), + + State = State1#aca{ match_aliases = Aliases, + parameter_info = ParamInfo }, + + I = beam_ssa:add_anno(accepts_match_contexts, true, I0), + + {I, Last, Blocks, State}. + +aca_handle_convergence(Src, State0, Last0, Blocks0) -> + #b_br{fail=Fail0,succ=Succ0} = Last0, + + SuccPath = beam_ssa:rpo([Succ0], Blocks0), + FailPath = beam_ssa:rpo([Fail0], Blocks0), + + %% The promotion logic in alias_matched_binaries breaks down if the source + %% is used after the fail/success paths converge, as we have no way to tell + %% whether the source is a match context or something else past that point. + %% + %% We could handle this through clever insertion of phi nodes but it's + %% far simpler to copy either branch in its entirety. It doesn't matter + %% which one as long as they become disjoint. + ConvergedPaths = ordsets:intersection( + ordsets:from_list(SuccPath), + ordsets:from_list(FailPath)), + + case maps:is_key(Src, beam_ssa:uses(ConvergedPaths, Blocks0)) of + true -> + case shortest(SuccPath, FailPath) of + left -> + {Succ, Blocks, Counter} = + aca_copy_successors(Succ0, Blocks0, State0#aca.counter), + State = State0#aca{ counter = Counter }, + {State, Last0#b_br{succ=Succ}, Blocks}; + right -> + {Fail, Blocks, Counter} = + aca_copy_successors(Fail0, Blocks0, State0#aca.counter), + State = State0#aca{ counter = Counter }, + {State, Last0#b_br{fail=Fail}, Blocks} + end; + false -> + {State0, Last0, Blocks0} + end. + +shortest([_|As], [_|Bs]) -> shortest(As, Bs); +shortest([], _) -> left; +shortest(_, []) -> right. + +%% Copies all successor blocks of Lbl, returning the label to the entry block +%% of this copy. Since the copied blocks aren't referenced anywhere else, they +%% are all guaranteed to be dominated by Lbl. +aca_copy_successors(Lbl0, Blocks0, Counter0) -> + %% Building the block rename map up front greatly simplifies phi node + %% handling. + Path = beam_ssa:rpo([Lbl0], Blocks0), + {BRs, Counter1} = aca_cs_build_brs(Path, Counter0, #{}), + {Blocks, Counter} = aca_cs_1(Path, Blocks0, Counter1, #{}, BRs, #{}), + Lbl = maps:get(Lbl0, BRs), + {Lbl, Blocks, Counter}. + +aca_cs_build_brs([Lbl | Path], Counter0, Acc) -> + aca_cs_build_brs(Path, Counter0 + 1, maps:put(Lbl, Counter0, Acc)); +aca_cs_build_brs([], Counter, Acc) -> + {Acc, Counter}. + +aca_cs_1([Lbl0 | Path], Blocks, Counter0, VRs0, BRs, Acc0) -> + Block0 = maps:get(Lbl0, Blocks), + Lbl = maps:get(Lbl0, BRs), + {VRs, Block, Counter} = aca_cs_block(Block0, Counter0, VRs0, BRs), + Acc = maps:put(Lbl, Block, Acc0), + aca_cs_1(Path, Blocks, Counter, VRs, BRs, Acc); +aca_cs_1([], Blocks, Counter, _VRs, _BRs, Acc) -> + {maps:merge(Blocks, Acc), Counter}. + +aca_cs_block(#b_blk{is=Is0,last=Last0}=Block0, Counter0, VRs0, BRs) -> + {VRs, Is, Counter} = aca_cs_is(Is0, Counter0, VRs0, BRs, []), + Last = aca_cs_last(Last0, VRs, BRs), + Block = Block0#b_blk{is=Is,last=Last}, + {VRs, Block, Counter}. + +aca_cs_is([#b_set{op=Op, + dst=Dst0, + args=Args0}=I0 | Is], + Counter0, VRs0, BRs, Acc) -> + Args = case Op of + phi -> aca_cs_args_phi(Args0, VRs0, BRs); + _ -> aca_cs_args(Args0, VRs0) + end, + Counter = Counter0 + 1, + Dst = #b_var{name={'@ssa_bsm_aca',Counter}}, + I = I0#b_set{dst=Dst,args=Args}, + VRs = maps:put(Dst0, Dst, VRs0), + aca_cs_is(Is, Counter, VRs, BRs, [I | Acc]); +aca_cs_is([], Counter, VRs, _BRs, Acc) -> + {VRs, reverse(Acc), Counter}. + +aca_cs_last(#b_switch{arg=Arg0,list=Switch0,fail=Fail0}=Sw, VRs, BRs) -> + Switch = [{Literal, maps:get(Lbl, BRs)} || {Literal, Lbl} <- Switch0], + Sw#b_switch{arg=aca_cs_arg(Arg0, VRs), + fail=maps:get(Fail0, BRs), + list=Switch}; +aca_cs_last(#b_br{bool=Arg0,succ=Succ0,fail=Fail0}=Br, VRs, BRs) -> + Br#b_br{bool=aca_cs_arg(Arg0, VRs), + succ=maps:get(Succ0, BRs), + fail=maps:get(Fail0, BRs)}; +aca_cs_last(#b_ret{arg=Arg0}=Ret, VRs, _BRs) -> + Ret#b_ret{arg=aca_cs_arg(Arg0, VRs)}. + +aca_cs_args_phi([{Arg, Lbl} | Args], VRs, BRs) -> + case BRs of + #{ Lbl := New } -> + [{aca_cs_arg(Arg, VRs), New} | aca_cs_args_phi(Args, VRs, BRs)]; + #{} -> + aca_cs_args_phi(Args, VRs, BRs) + end; +aca_cs_args_phi([], _VRs, _BRs) -> + []. + +aca_cs_args([Arg | Args], VRs) -> + [aca_cs_arg(Arg, VRs) | aca_cs_args(Args, VRs)]; +aca_cs_args([], _VRs) -> + []. + +aca_cs_arg(#b_remote{mod=Mod0,name=Name0}=Rem, VRs) -> + Mod = aca_cs_arg(Mod0, VRs), + Name = aca_cs_arg(Name0, VRs), + Rem#b_remote{mod=Mod,name=Name}; +aca_cs_arg(Arg, VRs) -> + case VRs of + #{ Arg := New } -> New; + #{} -> Arg + end. + +%% Allows contexts to pass through "wrapper functions" where the context is +%% passed directly to a function that accepts match contexts (including other +%% wrappers). +%% +%% This does not alter the function in any way, it only changes parameter info +%% so that skip_outgoing_tail_extraction is aware that it's safe to pass +%% contexts to us. + +allow_context_passthrough({Fs, ModInfo0}) -> + ModInfo = + acp_forward_params([{F, beam_ssa:uses(F#b_function.bs)} || F <- Fs], + ModInfo0), + {Fs, ModInfo}. + +acp_forward_params(FsUses, ModInfo0) -> + F = fun({#b_function{args=Parameters}=Func, UseMap}, ModInfo) -> + ParamInfo = + foldl(fun(Param, ParamInfo) -> + Uses = maps:get(Param, UseMap, []), + acp_1(Param, Uses, ModInfo, ParamInfo) + end, + funcinfo_get(Func, parameter_info, ModInfo), + Parameters), + funcinfo_set(Func, parameter_info, ParamInfo, ModInfo) + end, + %% Allowing context passthrough on one function may make it possible to + %% enable it on another, so it needs to be repeated for maximum effect. + case foldl(F, ModInfo0, FsUses) of + ModInfo0 -> ModInfo0; + Changed -> acp_forward_params(FsUses, Changed) + end. + +%% We have no way to know if an argument is a context, so it's only safe to +%% forward them if they're passed exactly once in the first block. Any other +%% uses are unsafe, including function_clause errors. +acp_1(Param, [{0, #b_set{op=call}=I}], ModInfo, ParamInfo) -> + %% We don't need to provide a context chain as our callers make sure that + %% multiple arguments never reference the same context. + case check_context_call(I, Param, [], ModInfo) of + {no_match_on_entry, _} -> ParamInfo; + Other -> maps:put(Param, Other, ParamInfo) + end; +acp_1(_Param, _Uses, _ModInfo, ParamInfo) -> + ParamInfo. + +%% This is conceptually similar to combine_matches but operates across +%% functions. Whenever a tail binary is passed to a parameter that accepts +%% match contexts we'll pass the context instead, improving performance by +%% avoiding the creation of a new match context in the callee. +%% +%% We also create an alias to delay extraction until it's needed as an actual +%% binary, which is often rare on the happy path. The cost of being wrong is +%% negligible (`bs_test_unit + bs_get_tail` vs `bs_get_binary`) so we're +%% applying it unconditionally to keep things simple. + +-record(sote, { definitions :: beam_ssa:definition_map(), + mod_info :: module_info(), + match_aliases = #{} :: match_alias_map() }). + +skip_outgoing_tail_extraction({Fs0, ModInfo}) -> + Fs = map(fun(F) -> skip_outgoing_tail_extraction(F, ModInfo) end, Fs0), + {Fs, ModInfo}. + +skip_outgoing_tail_extraction(#b_function{bs=Blocks0}=F, ModInfo) -> + case funcinfo_get(F, has_bsm_ops, ModInfo) of + true -> + State0 = #sote{ definitions = beam_ssa:definitions(Blocks0), + mod_info = ModInfo }, + + {Blocks1, State} = beam_ssa:mapfold_instrs_rpo( + fun sote_rewrite_calls/2, [0], State0, Blocks0), + + {Blocks, Counter} = alias_matched_binaries(Blocks1, + F#b_function.cnt, + State#sote.match_aliases), + + F#b_function{bs=Blocks,cnt=Counter}; + false -> + F + end. + +sote_rewrite_calls(#b_set{op=call,args=Args}=Call, State) -> + sote_rewrite_call(Call, Args, [], State); +sote_rewrite_calls(I, State) -> + {I, State}. + +sote_rewrite_call(Call, [], ArgsOut, State) -> + {Call#b_set{args=reverse(ArgsOut)}, State}; +sote_rewrite_call(Call0, [Arg | ArgsIn], ArgsOut, State0) -> + case is_tail_binary(Arg, State0#sote.definitions) of + true -> + CtxChain = context_chain_of(Arg, State0#sote.definitions), + case check_context_call(Call0, Arg, CtxChain, State0#sote.mod_info) of + suitable_for_reuse -> + Ctx = match_context_of(Arg, State0#sote.definitions), + + MatchAliases0 = State0#sote.match_aliases, + MatchAliases = maps:put(Arg, {0, Ctx}, MatchAliases0), + State = State0#sote{ match_aliases = MatchAliases }, + + Call = beam_ssa:add_anno(bsm_info, context_reused, Call0), + sote_rewrite_call(Call, ArgsIn, [Ctx | ArgsOut], State); + Other -> + Call = beam_ssa:add_anno(bsm_info, Other, Call0), + sote_rewrite_call(Call, ArgsIn, [Arg | ArgsOut], State0) + end; + false -> + sote_rewrite_call(Call0, ArgsIn, [Arg | ArgsOut], State0) + end. + +%% Adds parameter_type_info annotations to help the validator determine whether +%% our optimizations were safe. + +annotate_context_parameters({Fs, ModInfo}) -> + mapfoldl(fun annotate_context_parameters/2, ModInfo, Fs). + +annotate_context_parameters(F, ModInfo) -> + ParamInfo = funcinfo_get(F, parameter_info, ModInfo), + TypeAnno0 = beam_ssa:get_anno(parameter_type_info, F, #{}), + TypeAnno = maps:fold(fun(K, _V, Acc) when is_map_key(K, Acc) -> + %% Assertion. + error(conflicting_parameter_types); + (K, suitable_for_reuse, Acc) -> + T = beam_validator:type_anno(match_context), + Acc#{ K => T }; + (_K, _V, Acc) -> + Acc + end, TypeAnno0, ParamInfo), + {beam_ssa:add_anno(parameter_type_info, TypeAnno, F), ModInfo}. + +%%% +%%% +bin_opt_info +%%% + +collect_opt_info(Fs) -> + foldl(fun(#b_function{bs=Blocks}=F, Acc0) -> + UseMap = beam_ssa:uses(Blocks), + Where = beam_ssa:get_anno(location, F, []), + beam_ssa:fold_instrs_rpo( + fun(I, Acc) -> + collect_opt_info_1(I, Where, UseMap, Acc) + end, [0], Acc0, Blocks) + end, [], Fs). + +collect_opt_info_1(#b_set{op=Op,anno=Anno,dst=Dst}=I, Where, UseMap, Acc0) -> + case is_tail_binary(I) of + true when Op =:= bs_match -> + %% The uses include when the context is passed raw, so we discard + %% everything but the bs_extract instruction to limit warnings to + %% unoptimized uses. + Uses0 = maps:get(Dst, UseMap, []), + case [E || {_, #b_set{op=bs_extract}=E} <- Uses0] of + [Use] -> add_unopt_binary_info(Use, false, Where, UseMap, Acc0); + [] -> Acc0 + end; + true -> + %% Add a warning for each use. Note that we don't do anything + %% special if unused as a later pass will remove this instruction + %% anyway. + Uses = maps:get(Dst, UseMap, []), + foldl(fun({_Lbl, Use}, Acc) -> + add_unopt_binary_info(Use, false, Where, UseMap, Acc) + end, Acc0, Uses); + false -> + add_opt_info(Anno, Where, Acc0) + end; +collect_opt_info_1(#b_ret{anno=Anno}, Where, _UseMap, Acc) -> + add_opt_info(Anno, Where, Acc); +collect_opt_info_1(_I, _Where, _Uses, Acc) -> + Acc. + +add_opt_info(Anno, Where, Acc) -> + case maps:find(bsm_info, Anno) of + {ok, Term} -> [make_warning(Term, Anno, Where) | Acc]; + error -> Acc + end. + +%% When an alias is promoted we need to figure out where it goes to ignore +%% warnings for compiler-generated things, and provide more useful warnings in +%% general. +%% +%% We track whether the binary has been used to build another term because it +%% can be helpful when there's no line information. + +add_unopt_binary_info(#b_set{op=Follow,dst=Dst}, _Nested, Where, UseMap, Acc0) + when Follow =:= put_tuple; + Follow =:= put_list; + Follow =:= put_map -> + %% Term-building instructions. + {_, Uses} = unzip(maps:get(Dst, UseMap, [])), + foldl(fun(Use, Acc) -> + add_unopt_binary_info(Use, true, Where, UseMap, Acc) + end, Acc0, Uses); +add_unopt_binary_info(#b_set{op=Follow,dst=Dst}, Nested, Where, UseMap, Acc0) + when Follow =:= bs_extract; + Follow =:= phi -> + %% Non-building instructions that need to be followed. + {_, Uses} = unzip(maps:get(Dst, UseMap, [])), + foldl(fun(Use, Acc) -> + add_unopt_binary_info(Use, Nested, Where, UseMap, Acc) + end, Acc0, Uses); +add_unopt_binary_info(#b_set{op=call, + args=[#b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=error}} | + _Ignored]}, + _Nested, _Where, _UseMap, Acc) -> + %% There's no nice way to tell compiler-generated exceptions apart from + %% user ones so we ignore them all. I doubt anyone cares. + Acc; +add_unopt_binary_info(#b_switch{anno=Anno}=I, Nested, Where, _UseMap, Acc) -> + [make_promotion_warning(I, Nested, Anno, Where) | Acc]; +add_unopt_binary_info(#b_set{anno=Anno}=I, Nested, Where, _UseMap, Acc) -> + [make_promotion_warning(I, Nested, Anno, Where) | Acc]; +add_unopt_binary_info(#b_ret{anno=Anno}=I, Nested, Where, _UseMap, Acc) -> + [make_promotion_warning(I, Nested, Anno, Where) | Acc]; +add_unopt_binary_info(#b_br{anno=Anno}=I, Nested, Where, _UseMap, Acc) -> + [make_promotion_warning(I, Nested, Anno, Where) | Acc]. + +make_promotion_warning(I, Nested, Anno, Where) -> + make_warning({binary_created, I, Nested}, Anno, Where). + +make_warning(Term, Anno, Where) -> + {File, Line} = maps:get(location, Anno, Where), + {File,[{Line,?MODULE,Term}]}. + +format_opt_info(context_reused) -> + "OPTIMIZED: match context reused"; +format_opt_info({binary_created, _, _}=Promotion) -> + io_lib:format("BINARY CREATED: ~s", [format_opt_info_1(Promotion)]); +format_opt_info(Other) -> + io_lib:format("NOT OPTIMIZED: ~s", [format_opt_info_1(Other)]). + +format_opt_info_1({binary_created, #b_set{op=call,args=[Call|_]}, false}) -> + io_lib:format("binary is used in call to ~s which doesn't support " + "context reuse", [format_call(Call)]); +format_opt_info_1({binary_created, #b_set{op=call,args=[Call|_]}, true}) -> + io_lib:format("binary is used in term passed to ~s", + [format_call(Call)]); +format_opt_info_1({binary_created, #b_set{op={bif, BIF},args=Args}, false}) -> + io_lib:format("binary is used in ~p/~p which doesn't support context " + "reuse", [BIF, length(Args)]); +format_opt_info_1({binary_created, #b_set{op={bif, BIF},args=Args}, true}) -> + io_lib:format("binary is used in term passed to ~p/~p", + [BIF, length(Args)]); +format_opt_info_1({binary_created, #b_set{op=Op}, false}) -> + io_lib:format("binary is used in '~p' which doesn't support context " + "reuse", [Op]); +format_opt_info_1({binary_created, #b_set{op=Op}, true}) -> + io_lib:format("binary is used in term passed to '~p'", [Op]); +format_opt_info_1({binary_created, #b_ret{}, false}) -> + io_lib:format("binary is returned from the function", []); +format_opt_info_1({binary_created, #b_ret{}, true}) -> + io_lib:format("binary is used in a term that is returned from the " + "function", []); +format_opt_info_1({unsuitable_call, {Call, Inner}}) -> + io_lib:format("binary used in call to ~s, where ~s", + [format_call(Call), format_opt_info_1(Inner)]); +format_opt_info_1({remote_call, Call}) -> + io_lib:format("binary is used in remote call to ~s", [format_call(Call)]); +format_opt_info_1({fun_call, Call}) -> + io_lib:format("binary is used in fun call (~s)", + [format_call(Call)]); +format_opt_info_1({multiple_uses_in_call, Call}) -> + io_lib:format("binary is passed as multiple arguments to ~s", + [format_call(Call)]); +format_opt_info_1({no_match_on_entry, Call}) -> + io_lib:format("binary is used in call to ~s which does not begin with a " + "suitable binary match", [format_call(Call)]); +format_opt_info_1({used_before_match, #b_set{op=call,args=[Call|_]}}) -> + io_lib:format("binary is used in call to ~s before being matched", + [format_call(Call)]); +format_opt_info_1({used_before_match, #b_set{op={bif, BIF},args=Args}}) -> + io_lib:format("binary is used in ~p/~p before being matched", + [BIF, length(Args)]); +format_opt_info_1({used_before_match, #b_set{op=phi}}) -> + io_lib:format("binary is returned from an expression before being " + "matched", []); +format_opt_info_1({used_before_match, #b_set{op=Op}}) -> + io_lib:format("binary is used in '~p' before being matched",[Op]); +format_opt_info_1(Term) -> + io_lib:format("~w", [Term]). + +format_call(#b_local{name=#b_literal{val=F},arity=A}) -> + io_lib:format("~p/~p", [F, A]); +format_call(#b_remote{mod=#b_literal{val=M},name=#b_literal{val=F},arity=A}) -> + io_lib:format("~p:~p/~p", [M, F, A]); +format_call(Fun) -> + io_lib:format("~p", [Fun]). diff --git a/lib/compiler/src/beam_ssa_codegen.erl b/lib/compiler/src/beam_ssa_codegen.erl new file mode 100644 index 0000000000..c2d5035b19 --- /dev/null +++ b/lib/compiler/src/beam_ssa_codegen.erl @@ -0,0 +1,2077 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% Purpose: Generate BEAM assembly code from the SSA format. + +-module(beam_ssa_codegen). + +-export([module/2]). +-export([classify_heap_need/2]). %Called from beam_ssa_pre_codegen. + +-export_type([ssa_register/0]). + +-include("beam_ssa.hrl"). + +-import(lists, [foldl/3,keymember/3,keysort/2,last/1,map/2,mapfoldl/3, + reverse/1,reverse/2,sort/1,splitwith/2,takewhile/2]). + +-record(cg, {lcount=1 :: beam_label(), %Label counter + functable=#{} :: #{fa()=>beam_label()}, + labels=#{} :: #{ssa_label()=>0|beam_label()}, + used_labels=gb_sets:empty() :: gb_sets:set(ssa_label()), + regs=#{} :: #{beam_ssa:var_name()=>ssa_register()}, + ultimate_fail=1 :: beam_label(), + catches=gb_sets:empty() :: gb_sets:set(ssa_label()) + }). + +-spec module(beam_ssa:b_module(), [compile:option()]) -> + {'ok',beam_asm:module_code()}. + +module(#b_module{name=Mod,exports=Es,attributes=Attrs,body=Fs}, _Opts) -> + {Asm,St} = functions(Fs, {atom,Mod}), + {ok,{Mod,Es,Attrs,Asm,St#cg.lcount}}. + +-record(need, {h=0 :: non_neg_integer(), + f=0 :: non_neg_integer()}). + +-record(cg_blk, {anno=#{} :: anno(), + is=[] :: [instruction()], + last :: terminator()}). + +-record(cg_set, {anno=#{} :: anno(), + dst :: b_var(), + op :: beam_ssa:op(), + args :: [beam_ssa:argument() | xreg()]}). + +-record(cg_alloc, {anno=#{} :: anno(), + stack=none :: 'none' | pos_integer(), + words=#need{} :: #need{}, + live :: 'undefined' | pos_integer(), + def_yregs=[] :: [yreg()] + }). + +-record(cg_br, {bool :: beam_ssa:value(), + succ :: ssa_label(), + fail :: ssa_label() + }). +-record(cg_ret, {arg :: cg_value(), + dealloc=none :: 'none' | pos_integer() + }). +-record(cg_switch, {arg :: cg_value(), + fail :: ssa_label(), + list :: [sw_list_item()] + }). + +-type fa() :: {beam_asm:function_name(),arity()}. +-type ssa_label() :: beam_ssa:label(). +-type beam_label() :: beam_asm:label(). + +-type anno() :: beam_ssa:anno(). + +-type b_var() :: beam_ssa:b_var(). +-type b_literal() :: beam_ssa:b_literal(). + +-type cg_value() :: beam_ssa:value() | xreg(). + +-type cg_set() :: #cg_set{}. +-type cg_alloc() :: #cg_alloc{}. + +-type instruction() :: cg_set() | cg_alloc(). + +-type cg_br() :: #cg_br{}. +-type cg_ret() :: #cg_ret{}. +-type cg_switch() :: #cg_switch{}. +-type terminator() :: cg_br() | cg_ret() | cg_switch(). + +-type sw_list_item() :: {b_literal(),ssa_label()}. + +-type reg_num() :: beam_asm:reg_num(). +-type xreg() :: {'x',reg_num()}. +-type yreg() :: {'y',reg_num()}. + +-type ssa_register() :: xreg() | yreg() | {'fr',reg_num()} | {'z',reg_num()}. + +functions(Forms, AtomMod) -> + mapfoldl(fun (F, St) -> function(F, AtomMod, St) end, + #cg{lcount=1}, Forms). + +function(#b_function{anno=Anno,bs=Blocks}, AtomMod, St0) -> + #{func_info:={_,Name,Arity}} = Anno, + try + assert_badarg_block(Blocks), %Assertion. + Regs = maps:get(registers, Anno), + St1 = St0#cg{labels=#{},used_labels=gb_sets:empty(), + regs=Regs}, + {Fi,St2} = new_label(St1), %FuncInfo label + {Entry,St3} = local_func_label(Name, Arity, St2), + {Ult,St4} = new_label(St3), %Ultimate failure + Labels = (St4#cg.labels)#{0=>Entry,?BADARG_BLOCK=>0}, + St5 = St4#cg{labels=Labels,used_labels=gb_sets:singleton(Entry), + ultimate_fail=Ult}, + {Body,St} = cg_fun(Blocks, St5), + Asm = [{label,Fi},line(Anno), + {func_info,AtomMod,{atom,Name},Arity}] ++ + add_parameter_annos(Body, Anno) ++ + [{label,Ult},if_end], + Func = {function,Name,Arity,Entry,Asm}, + {Func,St} + catch + Class:Error:Stack -> + io:fwrite("Function: ~w/~w\n", [Name,Arity]), + erlang:raise(Class, Error, Stack) + end. + +assert_badarg_block(Blocks) -> + %% Assertion: ?BADARG_BLOCK must be the call erlang:error(badarg). + case Blocks of + #{?BADARG_BLOCK:=Blk} -> + #b_blk{is=[#b_set{op=call,dst=Ret, + args=[#b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=error}}, + #b_literal{val=badarg}]}], + last=#b_ret{arg=Ret}} = Blk, + ok; + #{} -> + %% ?BADARG_BLOCK has been removed because it was never used. + ok + end. + +add_parameter_annos([{label, _}=Entry | Body], Anno) -> + ParamInfo = maps:get(parameter_type_info, Anno, #{}), + Annos = maps:fold( + fun(K, V, Acc) when is_map_key(K, ParamInfo) -> + TypeInfo = maps:get(K, ParamInfo), + [{'%', {type_info, V, TypeInfo}} | Acc]; + (_K, _V, Acc) -> + Acc + end, [], maps:get(registers, Anno)), + [Entry | sort(Annos)] ++ Body. + +cg_fun(Blocks, St0) -> + Linear0 = linearize(Blocks), + St = collect_catch_labels(Linear0, St0), + Linear1 = need_heap(Linear0), + Linear2 = prefer_xregs(Linear1, St), + Linear3 = liveness(Linear2, St), + Linear4 = defined(Linear3, St), + Linear = opt_allocate(Linear4, St), + cg_linear(Linear, St). + +%% collect_catch_labels(Linear, St) -> St. +%% Collect all catch labels (labels for blocks that begin +%% with 'landingpad' instructions) for later use. + +collect_catch_labels(Linear, St) -> + Labels = collect_catch_labels_1(Linear), + St#cg{catches=gb_sets:from_list(Labels)}. + +collect_catch_labels_1([{L,#cg_blk{is=[#cg_set{op=landingpad}|_]}}|Bs]) -> + [L|collect_catch_labels_1(Bs)]; +collect_catch_labels_1([_|Bs]) -> + collect_catch_labels_1(Bs); +collect_catch_labels_1([]) -> []. + +%% need_heap([{BlockLabel,Block]) -> [{BlockLabel,Block}]. +%% Insert need_heap instructions in the instruction list. Try to be smart and +%% collect them together as much as possible. + +need_heap(Bs0) -> + Bs1 = need_heap_allocs(Bs0, #{}), + {Bs,#need{h=0,f=0}} = need_heap_blks(reverse(Bs1), #need{}, []), + Bs. + +need_heap_allocs([{L,#cg_blk{is=Is0,last=Terminator}=Blk0}|Bs], Counts0) -> + Next = next_block(Bs), + Successors = successors(Terminator), + Counts = foldl(fun(S, Cnts) -> + case Cnts of + #{S:=C} -> Cnts#{S:=C+1}; + #{} when S =:= Next -> Cnts#{S=>1}; + #{} -> Cnts#{S=>42} + end + end, Counts0, Successors), + case Counts of + #{L:=1} -> + [{L,Blk0}|need_heap_allocs(Bs, Counts)]; + #{L:=_} -> + %% This block has multiple predecessors. Force an allocation + %% in this block so that the predecessors don't need to do + %% an allocation on behalf of this block. + Is = case need_heap_never(Is0) of + true -> Is0; + false -> [#cg_alloc{}|Is0] + end, + Blk = Blk0#cg_blk{is=Is}, + [{L,Blk}|need_heap_allocs(Bs, Counts)]; + #{} -> + [{L,Blk0}|need_heap_allocs(Bs, Counts)] + end; +need_heap_allocs([], _) -> []. + +need_heap_never([#cg_alloc{}|_]) -> true; +need_heap_never([#cg_set{op=recv_next}|_]) -> true; +need_heap_never([#cg_set{op=wait}|_]) -> true; +need_heap_never(_) -> false. + +need_heap_blks([{L,#cg_blk{is=Is0}=Blk0}|Bs], H0, Acc) -> + {Is1,H1} = need_heap_is(reverse(Is0), H0, []), + {Ns,H} = need_heap_terminator(Bs, L, H1), + Is = Ns ++ Is1, + Blk = Blk0#cg_blk{is=Is}, + need_heap_blks(Bs, H, [{L,Blk}|Acc]); +need_heap_blks([], H, Acc) -> + {Acc,H}. + +need_heap_is([#cg_alloc{words=Words}=Alloc0|Is], N, Acc) -> + Alloc = Alloc0#cg_alloc{words=add_heap_words(N, Words)}, + need_heap_is(Is, #need{}, [Alloc|Acc]); +need_heap_is([#cg_set{anno=Anno,op=bs_init}=I0|Is], N, Acc) -> + Alloc = case need_heap_need(N) of + [#cg_alloc{words=Need}] -> alloc(Need); + [] -> 0 + end, + I = I0#cg_set{anno=Anno#{alloc=>Alloc}}, + need_heap_is(Is, #need{}, [I|Acc]); +need_heap_is([#cg_set{op=Op,args=Args}=I|Is], N, Acc) -> + case classify_heap_need(Op, Args) of + {put,Words} -> + %% Pass through adding to needed heap. + need_heap_is(Is, add_heap_words(N, Words), [I|Acc]); + put_float -> + need_heap_is(Is, add_heap_float(N), [I|Acc]); + neutral -> + need_heap_is(Is, N, [I|Acc]); + gc -> + need_heap_is(Is, #need{}, [I]++need_heap_need(N)++Acc) + end; +need_heap_is([], N, Acc) -> + {Acc,N}. + +need_heap_terminator([{_,#cg_blk{last=#cg_br{succ=L,fail=L}}}|_], L, N) -> + %% Fallthrough. + {[],N}; +need_heap_terminator([{_,#cg_blk{is=Is,last=#cg_br{succ=L}}}|_], L, N) -> + case need_heap_need(N) of + [] -> + {[],#need{}}; + [_|_]=Alloc -> + %% If the preceding instructions are a binary construction, + %% hoist the allocation and incorporate into the bs_init + %% instruction. + case reverse(Is) of + [#cg_set{op=succeeded},#cg_set{op=bs_init}|_] -> + {[],N}; + [#cg_set{op=bs_put}|_] -> + {[],N}; + _ -> + %% Not binary construction. Must emit an allocation + %% instruction in this block. + {Alloc,#need{}} + end + end; +need_heap_terminator([{_,#cg_blk{}}|_], _, N) -> + {need_heap_need(N),#need{}}; +need_heap_terminator([], _, H) -> + {need_heap_need(H),#need{}}. + +need_heap_need(#need{h=0,f=0}) -> []; +need_heap_need(#need{}=N) -> [#cg_alloc{words=N}]. + +add_heap_words(#need{h=H1,f=F1}, #need{h=H2,f=F2}) -> + #need{h=H1+H2,f=F1+F2}; +add_heap_words(#need{h=Heap}=N, Words) when is_integer(Words) -> + N#need{h=Heap+Words}. + +add_heap_float(#need{f=F}=N) -> + N#need{f=F+1}. + +%% classify_heap_need(Operation, Arguments) -> +%% gc | neutral | {put,Words} | put_float. +%% Classify the heap need for this instruction. The return +%% values have the following meaning. +%% +%% {put,Words} means that the instruction will use Words words to build +%% something on the heap. +%% +%% 'put_float' means that the instruction will build one floating point +%% number on the heap. +%% +%% 'gc' means that that the instruction can potentially do a GC or throw an +%% exception. That means that an allocation instruction for any building +%% must be placed after this instruction. +%% +%% 'neutral' means that the instruction does nothing to disturb the heap. + +-spec classify_heap_need(beam_ssa:op(), [beam_ssa:value()]) -> + 'gc' | 'neutral' | + {'put',non_neg_integer()} | 'put_float'. + +classify_heap_need(put_list, _) -> + {put,2}; +classify_heap_need(put_tuple_arity, [#b_literal{val=Words}]) -> + {put,Words+1}; +classify_heap_need(put_tuple, Elements) -> + {put,length(Elements)+1}; +classify_heap_need({bif,Name}, Args) -> + case is_gc_bif(Name, Args) of + false -> neutral; + true -> gc + end; +classify_heap_need({float,Op}, _Args) -> + case Op of + get -> put_float; + _ -> neutral + end; +classify_heap_need(Name, _Args) -> + classify_heap_need(Name). + +%% classify_heap_need(Operation) -> gc | neutral. +%% Return either 'gc' or 'neutral'. +%% +%% 'gc' means that that the instruction can potentially do a GC or throw an +%% exception. That means that an allocation instruction for any building +%% must be placed after this instruction. +%% +%% 'neutral' means that the instruction does nothing to disturb the heap. +%% +%% Note: Only handle operations in this function that are not handled +%% by classify_heap_need/2. + +classify_heap_need(bs_add) -> gc; +classify_heap_need(bs_get) -> gc; +classify_heap_need(bs_get_tail) -> gc; +classify_heap_need(bs_init) -> gc; +classify_heap_need(bs_init_writable) -> gc; +classify_heap_need(bs_match_string) -> gc; +classify_heap_need(bs_put) -> neutral; +classify_heap_need(bs_restore) -> neutral; +classify_heap_need(bs_save) -> neutral; +classify_heap_need(bs_get_position) -> gc; +classify_heap_need(bs_set_position) -> neutral; +classify_heap_need(bs_skip) -> gc; +classify_heap_need(bs_start_match) -> neutral; +classify_heap_need(bs_test_tail) -> neutral; +classify_heap_need(bs_utf16_size) -> neutral; +classify_heap_need(bs_utf8_size) -> neutral; +classify_heap_need(build_stacktrace) -> gc; +classify_heap_need(call) -> gc; +classify_heap_need(catch_end) -> gc; +classify_heap_need(copy) -> neutral; +classify_heap_need(extract) -> gc; +classify_heap_need(get_hd) -> neutral; +classify_heap_need(get_map_element) -> neutral; +classify_heap_need(get_tl) -> neutral; +classify_heap_need(get_tuple_element) -> neutral; +classify_heap_need(has_map_field) -> neutral; +classify_heap_need(is_nonempty_list) -> neutral; +classify_heap_need(is_tagged_tuple) -> neutral; +classify_heap_need(kill_try_tag) -> gc; +classify_heap_need(landingpad) -> gc; +classify_heap_need(make_fun) -> gc; +classify_heap_need(new_try_tag) -> gc; +classify_heap_need(peek_message) -> gc; +classify_heap_need(put_map) -> gc; +classify_heap_need(put_tuple_elements) -> neutral; +classify_heap_need(raw_raise) -> gc; +classify_heap_need(recv_next) -> gc; +classify_heap_need(remove_message) -> neutral; +classify_heap_need(resume) -> gc; +classify_heap_need(set_tuple_element) -> gc; +classify_heap_need(succeeded) -> neutral; +classify_heap_need(timeout) -> gc; +classify_heap_need(wait) -> gc; +classify_heap_need(wait_timeout) -> gc. + +%%% +%%% Because beam_ssa_pre_codegen has inserted 'copy' instructions to copy +%%% variables that must be saved on the stack, a value can for some time +%%% be in both an X register and a Y register. +%%% +%%% Here we will keep track of variables that have the same value and +%%% rewrite instructions to use the variable that refers to the X +%%% register instead of the Y register. That could improve performance, +%%% since the BEAM interpreter have more optimized instructions +%%% operating on X registers than on Y registers. +%%% +%%% 'call' and 'make_fun' are handled somewhat specially. If a value +%%% already is in the correct X register, the X register will always +%%% be used instead of the Y register. However, if there are one or more +%%% values in the wrong X registers, the X registers variables will be +%%% used only if that does not cause more 'move' instructions to be +%%% be emitted than if the Y register variables were used. +%%% +%%% Here are some examples. The first example shows how a 'move' from +%%% an Y register is eliminated: +%%% +%%% move x0 y1 +%%% move y1 x0 %%Will be eliminated. +%%% +%%% call f/1 +%%% +%%% Here is an example when x0 and x1 must be swapped to load the argument +%%% registers. Here the 'call' instruction will use the Y registers to +%%% avoid introducing an extra 'move' insruction: +%%% +%%% move x0 y0 +%%% move x1 y1 +%%% +%%% move y0 x1 +%%% move y1 x0 +%%% +%%% call f/2 +%%% +%%% Using the X register to load the argument registers would need +%%% an extra 'move' instruction like this: +%%% +%%% move x0 y0 +%%% move x1 y1 +%%% +%%% move x1 x2 +%%% move x0 x1 +%%% move x2 x0 +%%% +%%% call f/2 +%%% + +prefer_xregs(Linear, St) -> + prefer_xregs(Linear, St, #{0=>#{}}). + +prefer_xregs([{L,#cg_blk{is=Is0,last=Last0}=Blk0}|Bs], St, Map0) -> + Copies0 = maps:get(L, Map0), + {Is,Copies} = prefer_xregs_is(Is0, St, Copies0, []), + Last = prefer_xregs_terminator(Last0, Copies, St), + Blk = Blk0#cg_blk{is=Is,last=Last}, + Successors = successors(Last), + Map = prefer_xregs_successors(Successors, Copies, Map0), + [{L,Blk}|prefer_xregs(Bs, St, Map)]; +prefer_xregs([], _St, _Map) -> []. + +prefer_xregs_successors([L|Ls], Copies0, Map0) -> + case Map0 of + #{L:=Copies1} -> + Copies = merge_copies(Copies0, Copies1), + Map = Map0#{L:=Copies}, + prefer_xregs_successors(Ls, Copies0, Map); + #{} -> + Map = Map0#{L=>Copies0}, + prefer_xregs_successors(Ls, Copies0, Map) + end; +prefer_xregs_successors([], _, Map) -> Map. + +prefer_xregs_is([#cg_alloc{}=I|Is], St, Copies0, Acc) -> + Copies = case I of + #cg_alloc{stack=none,words=#need{h=0,f=0}} -> + Copies0; + #cg_alloc{} -> + #{} + end, + prefer_xregs_is(Is, St, Copies, [I|Acc]); +prefer_xregs_is([#cg_set{op=copy,dst=Dst,args=[Src]}=I|Is], St, Copies0, Acc) -> + Copies1 = prefer_xregs_prune(I, Copies0, St), + Copies = case beam_args([Src,Dst], St) of + [Same,Same] -> Copies1; + [_,_] -> Copies1#{Dst=>Src} + end, + prefer_xregs_is(Is, St, Copies, [I|Acc]); +prefer_xregs_is([#cg_set{op=call,dst=Dst}=I0|Is], St, Copies, Acc) -> + I = prefer_xregs_call(I0, Copies, St), + prefer_xregs_is(Is, St, #{Dst=>{x,0}}, [I|Acc]); +prefer_xregs_is([#cg_set{op=make_fun,dst=Dst}=I0|Is], St, Copies, Acc) -> + I = prefer_xregs_call(I0, Copies, St), + prefer_xregs_is(Is, St, #{Dst=>{x,0}}, [I|Acc]); +prefer_xregs_is([#cg_set{op=set_tuple_element}=I|Is], St, Copies, Acc) -> + %% FIXME: HiPE translates the following code segment incorrectly: + %% {call_ext,3,{extfunc,erlang,setelement,3}}. + %% {move,{x,0},{y,3}}. + %% {set_tuple_element,{y,1},{y,3},1}. + %% Therefore, skip the translation of the arguments for set_tuple_element. + prefer_xregs_is(Is, St, Copies, [I|Acc]); +prefer_xregs_is([#cg_set{args=Args0}=I0|Is], St, Copies0, Acc) -> + Args = [do_prefer_xreg(A, Copies0, St) || A <- Args0], + I = I0#cg_set{args=Args}, + Copies = prefer_xregs_prune(I, Copies0, St), + prefer_xregs_is(Is, St, Copies, [I|Acc]); +prefer_xregs_is([], _St, Copies, Acc) -> + {reverse(Acc),Copies}. + +prefer_xregs_terminator(#cg_br{bool=Arg0}=I, Copies, St) -> + Arg = do_prefer_xreg(Arg0, Copies, St), + I#cg_br{bool=Arg}; +prefer_xregs_terminator(#cg_ret{arg=Arg0}=I, Copies, St) -> + Arg = do_prefer_xreg(Arg0, Copies, St), + I#cg_ret{arg=Arg}; +prefer_xregs_terminator(#cg_switch{arg=Arg0}=I, Copies, St) -> + Arg = do_prefer_xreg(Arg0, Copies, St), + I#cg_switch{arg=Arg}. + +prefer_xregs_prune(#cg_set{anno=#{clobbers:=true}}, _, _) -> + #{}; +prefer_xregs_prune(#cg_set{dst=Dst}, Copies, St) -> + DstReg = beam_arg(Dst, St), + F = fun(_, Alias) -> + beam_arg(Alias, St) =/= DstReg + end, + maps:filter(F, Copies). + +%% prefer_xregs_call(Instruction, Copies, St) -> Instruction. +%% Given a 'call' or 'make_fun' instruction, minimize the number +%% of 'move' instructions to set up the argument registers. +%% Prefer using X registers over Y registers, unless that will +%% result in more 'move' instructions. + +prefer_xregs_call(#cg_set{args=[_]}=I, _Copies, _St) -> + I; +prefer_xregs_call(#cg_set{args=[F|Args0]}=I, Copies, St) -> + case Args0 of + [A0] -> + %% Only one argument. Always prefer the X register + %% if available. + A = do_prefer_xreg(A0, Copies, St), + I#cg_set{args=[F,A]}; + [_|_] -> + %% Two or more arguments. Try rewriting arguments in + %% two ways and see which way produces the least + %% number of 'move' instructions. + Args1 = prefer_xregs_call_1(Args0, Copies, 0, St), + Args2 = [do_prefer_xreg(A, Copies, St) || A <- Args0], + case {count_moves(Args1, St),count_moves(Args2, St)} of + {N1,N2} when N1 < N2 -> + %% There will be fewer 'move' instructions if + %% we keep using Y registers. + I#cg_set{args=[F|Args1]}; + {_,_} -> + %% Always use the values in X registers. + I#cg_set{args=[F|Args2]} + end + end. + +count_moves(Args, St) -> + length(setup_args(beam_args(Args, St))). + +prefer_xregs_call_1([#b_var{}=A|As], Copies, X, St) -> + case {beam_arg(A, St),Copies} of + {{y,_},#{A:=Other}} -> + case beam_arg(Other, St) of + {x,X} -> + %% This value is already in the correct X register. + %% It is always benefical to use the X register variable. + [Other|prefer_xregs_call_1(As, Copies, X+1, St)]; + _ -> + %% This value is another X register. Keep using + %% the Y register variable. + [A|prefer_xregs_call_1(As, Copies, X+1, St)] + end; + {_,_} -> + %% The value is not available in an X register. + [A|prefer_xregs_call_1(As, Copies, X+1, St)] + end; +prefer_xregs_call_1([A|As], Copies, X, St) -> + [A|prefer_xregs_call_1(As, Copies, X+1, St)]; +prefer_xregs_call_1([], _, _, _) -> []. + +do_prefer_xreg(#b_var{}=A, Copies, St) -> + case {beam_arg(A, St),Copies} of + {{y,_},#{A:=Copy}} -> + Copy; + {_,_} -> + A + end; +do_prefer_xreg(A, _, _) -> A. + +merge_copies(Copies0, Copies1) when map_size(Copies0) =< map_size(Copies1) -> + maps:filter(fun(K, V) -> + case Copies1 of + #{K:=V} -> true; + #{} -> false + end + end, Copies0); +merge_copies(Copies0, Copies1) -> + merge_copies(Copies1, Copies0). + + +%%% +%%% Add annotations for the number of live registers. +%%% + +liveness(Linear, #cg{regs=Regs}) -> + liveness(reverse(Linear), #{}, Regs, []). + +liveness([{L,#cg_blk{is=Is0,last=Last0}=Blk0}|Bs], LiveMap0, Regs, Acc) -> + Successors = liveness_successors(Last0), + Live0 = ordsets:union([liveness_get(S, LiveMap0) || S <- Successors]), + Live1 = liveness_terminator(Last0, Live0), + {Is,Live} = liveness_is(reverse(Is0), Regs, Live1, []), + LiveMap = LiveMap0#{L=>Live}, + Blk = Blk0#cg_blk{is=Is}, + liveness(Bs, LiveMap, Regs, [{L,Blk}|Acc]); +liveness([], _LiveMap, _Regs, Acc) -> Acc. + +liveness_get(S, LiveMap) -> + case LiveMap of + #{S:=Live} -> Live; + #{} -> [] + end. + +liveness_successors(Terminator) -> + successors(Terminator) -- [?BADARG_BLOCK]. + +liveness_is([#cg_alloc{}=I0|Is], Regs, Live, Acc) -> + I = I0#cg_alloc{live=num_live(Live, Regs)}, + liveness_is(Is, Regs, Live, [I|Acc]); +liveness_is([#cg_set{dst=Dst,args=Args}=I0|Is], Regs, Live0, Acc) -> + Live1 = liveness_clobber(I0, Live0, Regs), + I1 = liveness_yregs_anno(I0, Live1, Regs), + Live2 = liveness_args(Args, Live1), + Live = ordsets:del_element(Dst, Live2), + I = liveness_anno(I1, Live, Regs), + liveness_is(Is, Regs, Live, [I|Acc]); +liveness_is([], _, Live, Acc) -> + {Acc,Live}. + +liveness_terminator(#cg_br{bool=Arg}, Live) -> + liveness_terminator_1(Arg, Live); +liveness_terminator(#cg_switch{arg=Arg}, Live) -> + liveness_terminator_1(Arg, Live); +liveness_terminator(#cg_ret{arg=Arg}, Live) -> + liveness_terminator_1(Arg, Live). + +liveness_terminator_1(#b_var{}=V, Live) -> + ordsets:add_element(V, Live); +liveness_terminator_1(#b_literal{}, Live) -> + Live; +liveness_terminator_1(Reg, Live) -> + _ = verify_beam_register(Reg), + ordsets:add_element(Reg, Live). + +liveness_args([#b_var{}=V|As], Live) -> + liveness_args(As, ordsets:add_element(V, Live)); +liveness_args([#b_remote{mod=Mod,name=Name}|As], Live) -> + liveness_args([Mod,Name|As], Live); +liveness_args([A|As], Live) -> + case is_beam_register(A) of + true -> + liveness_args(As, ordsets:add_element(A, Live)); + false -> + liveness_args(As, Live) + end; +liveness_args([], Live) -> Live. + +liveness_anno(#cg_set{op=Op}=I, Live, Regs) -> + case need_live_anno(Op) of + true -> + NumLive = num_live(Live, Regs), + Anno = (I#cg_set.anno)#{live=>NumLive}, + I#cg_set{anno=Anno}; + false -> + I + end. + +liveness_yregs_anno(#cg_set{op=Op,dst=Dst}=I, Live0, Regs) -> + case need_live_anno(Op) of + true -> + Live = ordsets:del_element(Dst, Live0), + LiveYregs = [V || V <- Live, is_yreg(V, Regs)], + Anno = (I#cg_set.anno)#{live_yregs=>LiveYregs}, + I#cg_set{anno=Anno}; + false -> + I + end. + +liveness_clobber(#cg_set{anno=Anno}, Live, Regs) -> + case Anno of + #{clobbers:=true} -> + [R || R <- Live, is_yreg(R, Regs)]; + _ -> + Live + end. + +is_yreg(R, Regs) -> + case Regs of + #{R:={y,_}} -> true; + #{} -> false + end. + +num_live(Live, Regs) -> + Rs = ordsets:from_list([get_register(V, Regs) || V <- Live]), + num_live_1(Rs, 0). + +num_live_1([{x,X}|T], X) -> + num_live_1(T, X+1); +num_live_1([{x,_}|_]=T, X) -> + %% error({hole,{x,X},expected,Next}); + num_live_1(T, X+1); +num_live_1([{y,_}|_], X) -> + X; +num_live_1([{z,_}|_], X) -> + X; +num_live_1([{fr,_}|T], X) -> + num_live_1(T, X); +num_live_1([], X) -> + X. + +get_live(#cg_set{anno=#{live:=Live}}) -> + Live. + +%% need_live_anno(Operation) -> true|false. +%% Return 'true' if the instruction needs a 'live' annotation with +%% the number live X registers, or 'false' otherwise. + +need_live_anno(Op) -> + case Op of + {bif,_} -> true; + bs_get -> true; + bs_init -> true; + bs_get_position -> true; + bs_get_tail -> true; + bs_start_match -> true; + bs_skip -> true; + call -> true; + put_map -> true; + _ -> false + end. + +%%% +%%% Add the following annotations for Y registers: +%%% +%%% def_yregs An ordset with variables that refer to live Y registers. +%%% That is, Y registers that that have been killed +%%% are not included. This annotation is added to all +%%% instructions that require Y registers to be initialized. +%%% +%%% kill_yregs This annotation is added to call instructions. It is +%%% an ordset containing variables referring to Y registers +%%% that will no longer be used after the call instruction. +%%% + +defined(Linear, #cg{regs=Regs}) -> + def(Linear, #{}, Regs). + +def([{L,#cg_blk{is=Is0,last=Last}=Blk0}|Bs], DefMap0, Regs) -> + Def0 = def_get(L, DefMap0), + {Is,Def} = def_is(Is0, Regs, Def0, []), + Successors = successors(Last), + DefMap = def_successors(Successors, Def, DefMap0), + Blk = Blk0#cg_blk{is=Is}, + [{L,Blk}|def(Bs, DefMap, Regs)]; +def([], _, _) -> []. + +def_get(L, DefMap) -> + case DefMap of + #{L:=Def} -> Def; + #{} -> [] + end. + +def_is([#cg_alloc{anno=Anno0}=I0|Is], Regs, Def, Acc) -> + I = I0#cg_alloc{anno=Anno0#{def_yregs=>Def}}, + def_is(Is, Regs, Def, [I|Acc]); +def_is([#cg_set{op=kill_try_tag,args=[#b_var{}=Tag]}=I|Is], Regs, Def0, Acc) -> + Def = ordsets:del_element(Tag, Def0), + def_is(Is, Regs, Def, [I|Acc]); +def_is([#cg_set{op=catch_end,args=[#b_var{}=Tag|_]}=I|Is], Regs, Def0, Acc) -> + Def = ordsets:del_element(Tag, Def0), + def_is(Is, Regs, Def, [I|Acc]); +def_is([#cg_set{anno=Anno0,op=call,dst=Dst}=I0|Is], + Regs, Def0, Acc) -> + #{live_yregs:=LiveYregVars} = Anno0, + LiveRegs = gb_sets:from_list([maps:get(V, Regs) || V <- LiveYregVars]), + Kill0 = ordsets:subtract(Def0, LiveYregVars), + + %% Kill0 is the set of variables that have just died. However, the registers + %% used for killed variables may have been reused, so we must check that the + %% registers to be killed are not used by other variables. + Kill = [K || K <- Kill0, not gb_sets:is_element(maps:get(K, Regs), LiveRegs)], + Anno = Anno0#{def_yregs=>Def0,kill_yregs=>Kill}, + I = I0#cg_set{anno=Anno}, + Def1 = ordsets:subtract(Def0, Kill), + Def = def_add_yreg(Dst, Def1, Regs), + def_is(Is, Regs, Def, [I|Acc]); +def_is([#cg_set{anno=Anno0,op={bif,Bif},dst=Dst,args=Args}=I0|Is], + Regs, Def0, Acc) -> + Arity = length(Args), + I = case is_gc_bif(Bif, Args) orelse not erl_bifs:is_safe(erlang, Bif, Arity) of + true -> + I0#cg_set{anno=Anno0#{def_yregs=>Def0}}; + false -> + I0 + end, + Def = def_add_yreg(Dst, Def0, Regs), + def_is(Is, Regs, Def, [I|Acc]); +def_is([#cg_set{anno=Anno0,dst=Dst}=I0|Is], Regs, Def0, Acc) -> + I = case need_y_init(I0) of + true -> + I0#cg_set{anno=Anno0#{def_yregs=>Def0}}; + false -> + I0 + end, + Def = def_add_yreg(Dst, Def0, Regs), + def_is(Is, Regs, Def, [I|Acc]); +def_is([], _, Def, Acc) -> + {reverse(Acc),Def}. + +def_add_yreg(Dst, Def, Regs) -> + case is_yreg(Dst, Regs) of + true -> ordsets:add_element(Dst, Def); + false -> Def + end. + +def_successors([S|Ss], Def0, DefMap) -> + case DefMap of + #{S:=Def1} -> + Def = ordsets:intersection(Def0, Def1), + def_successors(Ss, Def0, DefMap#{S:=Def}); + #{} -> + def_successors(Ss, Def0, DefMap#{S=>Def0}) + end; +def_successors([], _, DefMap) -> DefMap. + +%% need_y_init(#cg_set{}) -> true|false. +%% Return true if this instructions needs initialized Y registers +%% (because the instruction may do a GC or cause an exception +%% so that the stack will be scanned), or false otherwise. + +need_y_init(#cg_set{anno=#{clobbers:=Clobbers}}) -> Clobbers; +need_y_init(#cg_set{op=bs_get}) -> true; +need_y_init(#cg_set{op=bs_get_position}) -> true; +need_y_init(#cg_set{op=bs_get_tail}) -> true; +need_y_init(#cg_set{op=bs_init}) -> true; +need_y_init(#cg_set{op=bs_skip,args=[#b_literal{val=Type}|_]}) -> + case Type of + utf8 -> true; + utf16 -> true; + utf32 -> true; + _ -> false + end; +need_y_init(#cg_set{op=bs_start_match}) -> true; +need_y_init(#cg_set{op=put_map}) -> true; +need_y_init(#cg_set{}) -> false. + +%% opt_allocate([{BlockLabel,Block}], #st{}) -> [BeamInstruction]. +%% Update the def_yregs field of each #cg_alloc{} that allocates +%% a stack frame. #cg_alloc.def_yregs will list all Y registers +%% that will be initialized by the subsequent code (thus, the +%% listed Y registers don't require init/1 instructions). + +opt_allocate(Linear, #cg{regs=Regs}) -> + opt_allocate_1(Linear, Regs). + +opt_allocate_1([{L,#cg_blk{is=[#cg_alloc{stack=Stk}=I0|Is]}=Blk0}|Bs]=Bs0, Regs) + when is_integer(Stk) -> + %% Collect the variables that are initialized by copy + %% instruction in this block. + case ordsets:from_list(opt_allocate_defs(Is, Regs)) of + Yregs when length(Yregs) =:= Stk -> + %% Those copy instructions are sufficient to fully + %% initialize the stack frame. + I = I0#cg_alloc{def_yregs=Yregs}, + [{L,Blk0#cg_blk{is=[I|Is]}}|opt_allocate_1(Bs, Regs)]; + Yregs0 -> + %% Determine a conservative approximation of the Y + %% registers that are guaranteed to be initialized by all + %% successors of this block, and to it add the variables + %% initialized by copy instructions in this block. + Yregs1 = opt_alloc_def(Bs0, gb_sets:singleton(L), []), + Yregs = ordsets:union(Yregs0, Yregs1), + I = I0#cg_alloc{def_yregs=Yregs}, + [{L,Blk0#cg_blk{is=[I|Is]}}|opt_allocate_1(Bs, Regs)] + end; +opt_allocate_1([B|Bs], Regs) -> + [B|opt_allocate_1(Bs, Regs)]; +opt_allocate_1([], _) -> []. + +opt_allocate_defs([#cg_set{op=copy,dst=Dst}|Is], Regs) -> + case is_yreg(Dst, Regs) of + true -> [Dst|opt_allocate_defs(Is, Regs)]; + false -> [] + end; +opt_allocate_defs(_, _Regs) -> []. + +opt_alloc_def([{L,#cg_blk{is=Is,last=Last}}|Bs], Ws0, Def0) -> + case gb_sets:is_member(L, Ws0) of + false -> + opt_alloc_def(Bs, Ws0, Def0); + true -> + case opt_allocate_is(Is) of + none -> + Succ = successors(Last), + Ws = gb_sets:union(Ws0, gb_sets:from_list(Succ)), + opt_alloc_def(Bs, Ws, Def0); + Def1 when is_list(Def1) -> + Def = [Def1|Def0], + opt_alloc_def(Bs, Ws0, Def) + end + end; +opt_alloc_def([], _, Def) -> + ordsets:intersection(Def). + +opt_allocate_is([#cg_set{anno=Anno}|Is]) -> + case Anno of + #{def_yregs:=Yregs} -> + Yregs; + #{} -> + opt_allocate_is(Is) + end; +opt_allocate_is([#cg_alloc{anno=#{def_yregs:=Yregs},stack=none}|_]) -> + Yregs; +opt_allocate_is([#cg_alloc{}|Is]) -> + opt_allocate_is(Is); +opt_allocate_is([]) -> none. + +%%% +%%% Here follows the main code generation functions. +%%% + +%% cg_linear([{BlockLabel,Block}]) -> [BeamInstruction]. +%% Generate BEAM instructions. + +cg_linear([{L,#cg_blk{anno=#{recv_set:=L}=Anno0}=B0}|Bs], St0) -> + Anno = maps:remove(recv_set, Anno0), + B = B0#cg_blk{anno=Anno}, + {Is,St1} = cg_linear([{L,B}|Bs], St0), + {Fail,St} = use_block_label(L, St1), + {[{recv_set,Fail}|Is],St}; +cg_linear([{L,#cg_blk{is=Is0,last=Last}}|Bs], St0) -> + Next = next_block(Bs), + St1 = new_block_label(L, St0), + {Is1,St2} = cg_block(Is0, Last, Next, St1), + {Is2,St} = cg_linear(Bs, St2), + {def_block_label(L, St)++Is1++Is2,St}; +cg_linear([], St) -> {[],St}. + +cg_block([#cg_set{op=recv_next}], #cg_br{succ=Lr0}, _Next, St0) -> + {Lr,St} = use_block_label(Lr0, St0), + {[{loop_rec_end,Lr}],St}; +cg_block([#cg_set{op=wait}], #cg_br{succ=Lr0}, _Next, St0) -> + {Lr,St} = use_block_label(Lr0, St0), + {[{wait,Lr}],St}; +cg_block(Is0, Last, Next, St0) -> + case Last of + #cg_br{succ=Next,fail=Next} -> + cg_block(Is0, none, St0); + #cg_br{succ=Same,fail=Same} -> + {Fail,St1} = use_block_label(Same, St0), + {Is,St} = cg_block(Is0, none, St1), + {Is++[jump(Fail)],St}; + #cg_br{bool=Bool,succ=Next,fail=Fail0} -> + {Fail,St1} = use_block_label(Fail0, St0), + {Is,St} = cg_block(Is0, {Bool,Fail}, St1), + {Is,St}; + #cg_br{bool=Bool,succ=Succ0,fail=Fail0} -> + {[Succ,Fail],St1} = use_block_labels([Succ0,Fail0], St0), + {Is,St} = cg_block(Is0, {Bool,Fail}, St1), + {Is++[jump(Succ)],St}; + #cg_ret{arg=Src0,dealloc=N} -> + Src = beam_arg(Src0, St0), + cg_block(Is0, {return,Src,N}, St0); + #cg_switch{} -> + cg_switch(Is0, Last, St0) + end. + +cg_switch(Is0, Last, St0) -> + #cg_switch{arg=Src0,fail=Fail0,list=List0} = Last, + Src = beam_arg(Src0, St0), + {Fail1,St1} = use_block_label(Fail0, St0), + Fail = ensure_label(Fail1, St1), + {List1,St2} = + flatmapfoldl(fun({V,L}, S0) -> + {Lbl,S} = use_block_label(L, S0), + {[beam_arg(V, S),Lbl],S} + end, St1, List0), + {Is1,St} = cg_block(Is0, none, St2), + case reverse(Is1) of + [{bif,tuple_size,_,[Tuple],{z,_}=Src}|More] -> + List = map(fun({integer,Arity}) -> Arity; + ({f,_}=F) -> F + end, List1), + Is = reverse(More, [{select_tuple_arity,Tuple,Fail,{list,List}}]), + {Is,St}; + _ -> + SelectVal = {select_val,Src,Fail,{list,List1}}, + {Is1 ++ [SelectVal],St} + end. + +jump({f,_}=Fail) -> + {jump,Fail}; +jump({catch_tag,Fail}) -> + {jump,Fail}. + +bif_fail({f,_}=Fail) -> Fail; +bif_fail({catch_tag,_}) -> {f,0}. + +next_block([]) -> none; +next_block([{Next,_}|_]) -> Next. + +ensure_label(Fail0, #cg{ultimate_fail=Lbl}) -> + case bif_fail(Fail0) of + {f,0} -> {f,Lbl}; + {f,_}=Fail -> Fail + end. + +cg_block([#cg_set{anno=#{recv_mark:=L}=Anno0}=I0|T], Context, St0) -> + Anno = maps:remove(recv_mark, Anno0), + I = I0#cg_set{anno=Anno}, + {Is,St1} = cg_block([I|T], Context, St0), + {Fail,St} = use_block_label(L, St1), + {[{recv_mark,Fail}|Is],St}; +cg_block([#cg_set{op=new_try_tag,dst=Tag,args=Args}], {Tag,Fail0}, St) -> + {catch_tag,Fail} = Fail0, + [Reg,{atom,Kind}] = beam_args([Tag|Args], St), + {[{Kind,Reg,Fail}],St}; +cg_block([#cg_set{anno=Anno,op={bif,Name},dst=Dst0,args=Args0}=I, + #cg_set{op=succeeded,dst=Bool}], {Bool,Fail0}, St) -> + [Dst|Args] = beam_args([Dst0|Args0], St), + Line0 = call_line(body, {extfunc,erlang,Name,length(Args)}, Anno), + Fail = bif_fail(Fail0), + Line = case Fail of + {f,0} -> Line0; + {f,_} -> [] + end, + case is_gc_bif(Name, Args) of + true -> + Live = get_live(I), + Kill = kill_yregs(Anno, St), + {Kill++Line++[{gc_bif,Name,Fail,Live,Args,Dst}],St}; + false -> + {Line++[{bif,Name,Fail,Args,Dst}],St} + end; +cg_block([#cg_set{op={bif,tuple_size},dst=Arity0,args=[Tuple0]}, + #cg_set{op={bif,'=:='},dst=Bool,args=[Arity0,#b_literal{val=Ar}]}=Eq], + {Bool,Fail}=Context, St0) -> + Tuple = beam_arg(Tuple0, St0), + case beam_arg(Arity0, St0) of + {z,_} -> + %% The size will only be used once. Combine to a test_arity instruction. + Test = {test,test_arity,ensure_label(Fail, St0),[Tuple,Ar]}, + {[Test],St0}; + Arity -> + %% The size will be used more than once. Must do an explicit + %% BIF call followed by the '==' test. + TupleSize = {bif,tuple_size,{f,0},[Tuple],Arity}, + {Is,St} = cg_block([Eq], Context, St0), + {[TupleSize|Is],St} + end; +cg_block([#cg_set{op={bif,Name},dst=Dst0,args=Args0}]=Is0, {Dst0,Fail}, St0) -> + [Dst|Args] = beam_args([Dst0|Args0], St0), + case Dst of + {z,_} -> + %% The result of the BIF call will only be used once. Convert to + %% a test instruction. + {Test,St1} = bif_to_test(Name, Args, ensure_label(Fail, St0), St0), + {Test,St1}; + _ -> + %% Must explicitly call the BIF since the result will be used + %% more than once. + {Is1,St1} = cg_block(Is0, none, St0), + {Is2,St} = cg_block([], {Dst0,Fail}, St1), + {Is1++Is2,St} + end; +cg_block([#cg_set{anno=Anno,op={bif,Name},dst=Dst0,args=Args0}=I|T], + Context, St0) -> + [Dst|Args] = beam_args([Dst0|Args0], St0), + {Is0,St} = cg_block(T, Context, St0), + case is_gc_bif(Name, Args) of + true -> + Line = call_line(body, {extfunc,erlang,Name,length(Args)}, Anno), + Live = get_live(I), + Kill = kill_yregs(Anno, St), + Is = Kill++Line++[{gc_bif,Name,{f,0},Live,Args,Dst}|Is0], + {Is,St}; + false -> + Is = [{bif,Name,{f,0},Args,Dst}|Is0], + {Is,St} + end; +cg_block([#cg_set{op=bs_init,dst=Dst0,args=Args0,anno=Anno}=I, + #cg_set{op=succeeded,dst=Bool}], {Bool,Fail0}, St) -> + Fail = bif_fail(Fail0), + Line = line(Anno), + Alloc = map_get(alloc, Anno), + [#b_literal{val=Kind}|Args1] = Args0, + case Kind of + new -> + [Dst,Size,{integer,Unit}] = beam_args([Dst0|Args1], St), + Live = get_live(I), + {[Line|cg_bs_init(Dst, Size, Alloc, Unit, Live, Fail)],St}; + private_append -> + [Dst,Src,Bits,{integer,Unit}] = beam_args([Dst0|Args1], St), + Flags = {field_flags,[]}, + Is = [Line,{bs_private_append,Fail,Bits,Unit,Src,Flags,Dst}], + {Is,St}; + append -> + [Dst,Src,Bits,{integer,Unit}] = beam_args([Dst0|Args1], St), + Flags = {field_flags,[]}, + Live = get_live(I), + Is = [Line,{bs_append,Fail,Bits,Alloc,Live,Unit,Src,Flags,Dst}], + {Is,St} + end; +cg_block([#cg_set{anno=Anno,op=bs_start_match,dst=Ctx0,args=[Bin0]}=I, + #cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) -> + [Dst,Bin1] = beam_args([Ctx0,Bin0], St), + {Bin,Pre} = force_reg(Bin1, Dst), + Live = get_live(I), + %% num_slots is only set when using the old instructions. + case maps:find(num_slots, Anno) of + {ok, Slots} -> + Is = Pre ++ [{test,bs_start_match2,Fail,Live,[Bin,Slots],Dst}], + {Is,St}; + error -> + Is = Pre ++ [{test,bs_start_match3,Fail,Live,[Bin],Dst}], + {Is,St} + end; +cg_block([#cg_set{op=bs_get}=Set, + #cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) -> + {cg_bs_get(Fail, Set, St),St}; +cg_block([#cg_set{op=bs_match_string,args=[CtxVar,#b_literal{val=String}]}, + #cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) -> + CtxReg = beam_arg(CtxVar, St), + Is = [{test,bs_match_string,Fail,[CtxReg,String]}], + {Is,St}; +cg_block([#cg_set{dst=Dst0,op=landingpad,args=Args0}|T], Context, St0) -> + [Dst,{atom,Kind},Tag] = beam_args([Dst0|Args0], St0), + case Kind of + 'catch' -> + cg_catch(Dst, T, Context, St0); + 'try' -> + cg_try(Dst, Tag, T, Context, St0) + end; +cg_block([#cg_set{op=kill_try_tag,args=Args0}|Is], Context, St0) -> + [Reg] = beam_args(Args0, St0), + {Is0,St} = cg_block(Is, Context, St0), + {[{try_end,Reg}|Is0],St}; +cg_block([#cg_set{op=catch_end,dst=Dst0,args=Args0}|Is], Context, St0) -> + [Dst,Reg,{x,0}] = beam_args([Dst0|Args0], St0), + {Is0,St} = cg_block(Is, Context, St0), + {[{catch_end,Reg}|copy({x,0}, Dst)++Is0],St}; +cg_block([#cg_set{op=call}=I, + #cg_set{op=succeeded,dst=Bool}], {Bool,_Fail}, St) -> + %% A call in try/catch block. + cg_block([I], none, St); +cg_block([#cg_set{op=Op,dst=Dst0,args=Args0}=I, + #cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) -> + [Dst|Args] = beam_args([Dst0|Args0], St), + {cg_test(Op, bif_fail(Fail), Args, Dst, I),St}; +cg_block([#cg_set{op=bs_put,dst=Bool,args=Args0}], {Bool,Fail}, St) -> + Args = beam_args(Args0, St), + {cg_bs_put(bif_fail(Fail), Args),St}; +cg_block([#cg_set{op=bs_test_tail,dst=Bool,args=Args0}], {Bool,Fail}, St) -> + [Ctx,{integer,Bits}] = beam_args(Args0, St), + {[{test,bs_test_tail2,bif_fail(Fail),[Ctx,Bits]}],St}; +cg_block([#cg_set{op={float,checkerror},dst=Bool}], {Bool,Fail}, St) -> + {[{fcheckerror,bif_fail(Fail)}],St}; +cg_block([#cg_set{op=is_tagged_tuple,dst=Bool,args=Args0}], {Bool,Fail}, St) -> + [Src,{integer,Arity},Tag] = beam_args(Args0, St), + {[{test,is_tagged_tuple,ensure_label(Fail, St),[Src,Arity,Tag]}],St}; +cg_block([#cg_set{op=is_nonempty_list,dst=Bool,args=Args0}], {Bool,Fail}, St) -> + Args = beam_args(Args0, St), + {[{test,is_nonempty_list,ensure_label(Fail, St),Args}],St}; +cg_block([#cg_set{op=has_map_field,dst=Bool,args=Args0}], {Bool,Fail}, St) -> + [Src,Key] = beam_args(Args0, St), + {[{test,has_map_fields,Fail,Src,{list,[Key]}}],St}; +cg_block([#cg_set{op=call}=Call], {_Bool,_Fail}=Context, St0) -> + {Is0,St1} = cg_call(Call, body, none, St0), + {Is1,St} = cg_block([], Context, St1), + {Is0++Is1,St}; +cg_block([#cg_set{op=call,dst=Dst0}=Call], Context, St) -> + Dst = beam_arg(Dst0, St), + case Context of + {return,Dst,_} -> + cg_call(Call, tail, Context, St); + _ -> + cg_call(Call, body, Context, St) + end; +cg_block([#cg_set{op=call}=Call|T], Context, St0) -> + {Is0,St1} = cg_call(Call, body, none, St0), + {Is1,St} = cg_block(T, Context, St1), + {Is0++Is1,St}; +cg_block([#cg_set{op=make_fun,dst=Dst0,args=[Local|Args0]}|T], + Context, St0) -> + #b_local{name=#b_literal{val=Func},arity=Arity} = Local, + [Dst|Args] = beam_args([Dst0|Args0], St0), + {FuncLbl,St1} = local_func_label(Func, Arity, St0), + Is0 = setup_args(Args) ++ + [{make_fun2,{f,FuncLbl},0,0,length(Args)}|copy({x,0}, Dst)], + {Is1,St} = cg_block(T, Context, St1), + {Is0++Is1,St}; +cg_block([#cg_set{op=copy}|_]=T0, Context, St0) -> + {Is0,T} = cg_copy(T0, St0), + {Is1,St} = cg_block(T, Context, St0), + Is = Is0 ++ Is1, + case is_call(T) of + {yes,Arity} -> + {opt_call_moves(Is, Arity),St}; + no -> + {Is,St} + end; +cg_block([#cg_set{op=Op,dst=Dst0,args=Args0}=Set], none, St) -> + [Dst|Args] = beam_args([Dst0|Args0], St), + Is = cg_instr(Op, Args, Dst, Set), + {Is,St}; +cg_block([#cg_set{op=Op,dst=Dst0,args=Args0}=Set|T], Context, St0) -> + [Dst|Args] = beam_args([Dst0|Args0], St0), + Is0 = cg_instr(Op, Args, Dst, Set), + {Is1,St} = cg_block(T, Context, St0), + {Is0++Is1,St}; +cg_block([#cg_alloc{}=Alloc|T], Context, St0) -> + Is0 = cg_alloc(Alloc, St0), + {Is1,St} = cg_block(T, Context, St0), + {Is0++Is1,St}; +cg_block([], {return,Arg,none}, St) -> + Is = copy(Arg, {x,0}) ++ [return], + {Is,St}; +cg_block([], {return,Arg,N}, St) -> + Is = copy(Arg, {x,0}) ++ [{deallocate,N},return], + {Is,St}; +cg_block([], none, St) -> + {[],St}; +cg_block([], {Bool0,Fail}, St) -> + [Bool] = beam_args([Bool0], St), + {[{test,is_eq_exact,Fail,[Bool,{atom,true}]}],St}. + +cg_copy(T0, St) -> + {Copies,T} = splitwith(fun(#cg_set{op=copy}) -> true; + (_) -> false + end, T0), + Moves0 = cg_copy_1(Copies, St), + Moves1 = [Move || {move,Src,Dst}=Move <- Moves0, Src =/= Dst], + Scratch = {x,1022}, + Moves = order_moves(Moves1, Scratch), + {Moves,T}. + +cg_copy_1([#cg_set{dst=Dst0,args=Args}|T], St) -> + [Dst,Src] = beam_args([Dst0|Args], St), + Copies = cg_copy_1(T, St), + case keymember(Dst, 3, Copies) of + true -> + %% Will be overwritten. Don't generate a move instruction. + Copies; + false -> + [{move,Src,Dst}|Copies] + end; +cg_copy_1([], _St) -> []. + +-define(IS_LITERAL(Val), (Val =:= nil orelse + element(1, Val) =:= integer orelse + element(1, Val) =:= float orelse + element(1, Val) =:= atom orelse + element(1, Val) =:= literal)). + +bif_to_test('or', [V1,V2], {f,Lbl}=Fail, St0) when Lbl =/= 0 -> + {SuccLabel,St} = new_label(St0), + {[{test,is_eq_exact,{f,SuccLabel},[V1,{atom,false}]}, + {test,is_eq_exact,Fail,[V2,{atom,true}]}, + {label,SuccLabel}],St}; +bif_to_test(Op, Args, Fail, St) -> + {bif_to_test(Op, Args, Fail),St}. + +bif_to_test('and', [V1,V2], Fail) -> + [{test,is_eq_exact,Fail,[V1,{atom,true}]}, + {test,is_eq_exact,Fail,[V2,{atom,true}]}]; +bif_to_test('not', [Var], Fail) -> + [{test,is_eq_exact,Fail,[Var,{atom,false}]}]; +bif_to_test(Name, Args, Fail) -> + [bif_to_test_1(Name, Args, Fail)]. + +bif_to_test_1(is_atom, [_]=Ops, Fail) -> + {test,is_atom,Fail,Ops}; +bif_to_test_1(is_boolean, [_]=Ops, Fail) -> + {test,is_boolean,Fail,Ops}; +bif_to_test_1(is_binary, [_]=Ops, Fail) -> + {test,is_binary,Fail,Ops}; +bif_to_test_1(is_bitstring,[_]=Ops, Fail) -> + {test,is_bitstr,Fail,Ops}; +bif_to_test_1(is_float, [_]=Ops, Fail) -> + {test,is_float,Fail,Ops}; +bif_to_test_1(is_function, [_]=Ops, Fail) -> + {test,is_function,Fail,Ops}; +bif_to_test_1(is_function, [_,_]=Ops, Fail) -> + {test,is_function2,Fail,Ops}; +bif_to_test_1(is_integer, [_]=Ops, Fail) -> + {test,is_integer,Fail,Ops}; +bif_to_test_1(is_list, [_]=Ops, Fail) -> + {test,is_list,Fail,Ops}; +bif_to_test_1(is_map, [_]=Ops, Fail) -> + {test,is_map,Fail,Ops}; +bif_to_test_1(is_number, [_]=Ops, Fail) -> + {test,is_number,Fail,Ops}; +bif_to_test_1(is_pid, [_]=Ops, Fail) -> + {test,is_pid,Fail,Ops}; +bif_to_test_1(is_port, [_]=Ops, Fail) -> + {test,is_port,Fail,Ops}; +bif_to_test_1(is_reference, [_]=Ops, Fail) -> + {test,is_reference,Fail,Ops}; +bif_to_test_1(is_tuple, [_]=Ops, Fail) -> + {test,is_tuple,Fail,Ops}; +bif_to_test_1('=<', [A,B], Fail) -> + {test,is_ge,Fail,[B,A]}; +bif_to_test_1('>', [A,B], Fail) -> + {test,is_lt,Fail,[B,A]}; +bif_to_test_1('<', [_,_]=Ops, Fail) -> + {test,is_lt,Fail,Ops}; +bif_to_test_1('>=', [_,_]=Ops, Fail) -> + {test,is_ge,Fail,Ops}; +bif_to_test_1('==', [C,A], Fail) when ?IS_LITERAL(C) -> + {test,is_eq,Fail,[A,C]}; +bif_to_test_1('==', [_,_]=Ops, Fail) -> + {test,is_eq,Fail,Ops}; +bif_to_test_1('/=', [C,A], Fail) when ?IS_LITERAL(C) -> + {test,is_ne,Fail,[A,C]}; +bif_to_test_1('/=', [_,_]=Ops, Fail) -> + {test,is_ne,Fail,Ops}; +bif_to_test_1('=:=', [C,A], Fail) when ?IS_LITERAL(C) -> + {test,is_eq_exact,Fail,[A,C]}; +bif_to_test_1('=:=', [_,_]=Ops, Fail) -> + {test,is_eq_exact,Fail,Ops}; +bif_to_test_1('=/=', [C,A], Fail) when ?IS_LITERAL(C) -> + {test,is_ne_exact,Fail,[A,C]}; +bif_to_test_1('=/=', [_,_]=Ops, Fail) -> + {test,is_ne_exact,Fail,Ops}. + +opt_call_moves(Is0, Arity) -> + {Moves0,Is} = splitwith(fun({move,_,_}) -> true; + ({kill,_}) -> true; + (_) -> false + end, Is0), + Moves = opt_call_moves_1(Moves0, Arity), + Moves ++ Is. + +opt_call_moves_1([{move,Src,{x,_}=Tmp}=M1|[{kill,_}|_]=Is], Arity) -> + %% There could be a {move,Tmp,{x,0}} instruction after the + %% kill/1 instructions (moved to there by opt_move_to_x0/1). + case splitwith(fun({kill,_}) -> true; + (_) -> false + end, Is) of + {Kills,[{move,{x,_}=Tmp,{x,0}}=M2]} -> + %% The two move/2 instructions (M1 and M2) can be combined + %% to one. The question is, though, is it safe to place + %% them after the kill/1 instructions? + case is_killed(Src, Kills, Arity) of + true -> + %% Src (a Y register) is killed by one of the + %% kill/1 instructions. Thus M1 and M2 + %% must be placed before the kill/1 instructions + %% (essentially undoing what opt_move_to_x0/1 + %% did, which turned out to be a pessimization + %% in this case). + opt_call_moves_1([M1,M2|Kills], Arity); + false -> + %% Src is not killed by any of the kill/1 + %% instructions. Thus it is safe to place + %% M1 and M2 after the kill/1 instructions. + opt_call_moves_1(Kills++[M1,M2], Arity) + end; + {_,_} -> + [M1|Is] + end; +opt_call_moves_1([{move,Src,{x,_}=Tmp}=M1,{move,Tmp,Dst}=M2|Is], Arity) -> + case is_killed(Tmp, Is, Arity) of + true -> + %% The X register Tmp is never used again. We can collapse + %% the two move instruction into one. + [{move,Src,Dst}|opt_call_moves_1(Is, Arity)]; + false -> + [M1|opt_call_moves_1([M2|Is], Arity)] + end; +opt_call_moves_1([M|Ms], Arity) -> + [M|opt_call_moves_1(Ms, Arity)]; +opt_call_moves_1([], _Arity) -> []. + +is_killed(Y, [{kill,Y}|_], _) -> + true; +is_killed(R, [{kill,_}|Is], Arity) -> + is_killed(R, Is, Arity); +is_killed(R, [{move,R,_}|_], _) -> + false; +is_killed(R, [{move,_,R}|_], _) -> + true; +is_killed(R, [{move,_,_}|Is], Arity) -> + is_killed(R, Is, Arity); +is_killed({x,X}, [], Arity) -> + X >= Arity; +is_killed({y,_}, [], _) -> + false. + +cg_alloc(#cg_alloc{stack=none,words=#need{h=0,f=0}}, _St) -> + []; +cg_alloc(#cg_alloc{stack=none,words=Need,live=Live}, _St) -> + [{test_heap,alloc(Need),Live}]; +cg_alloc(#cg_alloc{stack=Stk,words=Need,live=Live,def_yregs=DefYregs}, + #cg{regs=Regs}) when is_integer(Stk) -> + Alloc = alloc(Need), + All = [{y,Y} || Y <- lists:seq(0, Stk-1)], + Def = ordsets:from_list([maps:get(V, Regs) || V <- DefYregs]), + NeedInit = ordsets:subtract(All, Def), + NoZero = length(Def)*2 > Stk, + I = case {NoZero,Alloc} of + {true,0} -> {allocate,Stk,Live}; + {true,_} -> {allocate_heap,Stk,Alloc,Live}; + {false,0} -> {allocate_zero,Stk,Live}; + {false,_} -> {allocate_heap_zero,Stk,Alloc,Live} + end, + [I|case NoZero of + true -> [{init,Y} || Y <- NeedInit]; + false -> [] + end]. + +alloc(#need{h=Words,f=0}) -> + Words; +alloc(#need{h=Words,f=Floats}) -> + {alloc,[{words,Words},{floats,Floats}]}. + +is_call([#cg_set{op=call,args=[#b_var{}|Args]}|_]) -> + {yes,1+length(Args)}; +is_call([#cg_set{op=call,args=[_|Args]}|_]) -> + {yes,length(Args)}; +is_call([#cg_set{op=make_fun,args=[_|Args]}|_]) -> + {yes,length(Args)}; +is_call(_) -> + no. + +cg_call(#cg_set{anno=Anno,op=call,dst=Dst0,args=[#b_local{}=Func0|Args0]}, + Where, Context, St0) -> + [Dst|Args] = beam_args([Dst0|Args0], St0), + #b_local{name=Name0,arity=Arity} = Func0, + {atom,Name} = beam_arg(Name0, St0), + {FuncLbl,St} = local_func_label(Name, Arity, St0), + Line = call_line(Where, local, Anno), + Call = build_call(call, Arity, {f,FuncLbl}, Context, Dst), + Is = setup_args(Args, Anno, Context, St) ++ Line ++ Call, + case Anno of + #{ result_type := Info } -> + {Is ++ [{'%', {type_info, Dst, Info}}], St}; + #{} -> + {Is, St} + end; +cg_call(#cg_set{anno=Anno0,op=call,dst=Dst0,args=[#b_remote{}=Func0|Args0]}, + Where, Context, St) -> + [Dst|Args] = beam_args([Dst0|Args0], St), + #b_remote{mod=Mod0,name=Name0,arity=Arity} = Func0, + case {beam_arg(Mod0, St),beam_arg(Name0, St)} of + {{atom,Mod},{atom,Name}} -> + Func = {extfunc,Mod,Name,Arity}, + Line = call_line(Where, Func, Anno0), + Call = build_call(call_ext, Arity, Func, Context, Dst), + Anno = case erl_bifs:is_exit_bif(Mod, Name, Arity) of + true -> + %% There is no need to kill Y registers + %% before calling an exit BIF. + maps:remove(kill_yregs, Anno0); + false -> + Anno0 + end, + Is = setup_args(Args, Anno, Context, St) ++ Line ++ Call, + {Is,St}; + {Mod,Name} -> + Apply = build_apply(Arity, Context, Dst), + Is = setup_args(Args++[Mod,Name], Anno0, Context, St) ++ + [line(Anno0)] ++ Apply, + {Is,St} + end; +cg_call(#cg_set{anno=Anno,op=call,dst=Dst0,args=Args0}, + Where, Context, St) -> + [Dst,Func|Args] = beam_args([Dst0|Args0], St), + Line = call_line(Where, Func, Anno), + Arity = length(Args), + Call = build_call(call_fun, Arity, Func, Context, Dst), + Is = setup_args(Args++[Func], Anno, Context, St) ++ Line ++ Call, + {Is,St}. + +build_call(call_fun, Arity, _Func, none, Dst) -> + [{call_fun,Arity}|copy({x,0}, Dst)]; +build_call(call_fun, Arity, _Func, {return,Dst,N}, Dst) when is_integer(N) -> + [{call_fun,Arity},{deallocate,N},return]; +build_call(call_fun, Arity, _Func, {return,Val,N}, _Dst) when is_integer(N) -> + [{call_fun,Arity},{move,Val,{x,0}},{deallocate,N},return]; +build_call(call_ext, 2, {extfunc,erlang,'!',2}, none, Dst) -> + [send|copy({x,0}, Dst)]; +build_call(call_ext, 2, {extfunc,erlang,'!',2}, {return,Dst,N}, Dst) + when is_integer(N) -> + [send,{deallocate,N},return]; +build_call(Prefix, Arity, Func, {return,Dst,none}, Dst) -> + I = case Prefix of + call -> call_only; + call_ext -> call_ext_only + end, + [{I,Arity,Func}]; +build_call(call_ext, Arity, {extfunc,Mod,Name,Arity}=Func, {return,_,none}, _Dst) -> + true = erl_bifs:is_exit_bif(Mod, Name, Arity), %Assertion. + [{call_ext_only,Arity,Func}]; +build_call(Prefix, Arity, Func, {return,Dst,N}, Dst) when is_integer(N) -> + I = case Prefix of + call -> call_last; + call_ext -> call_ext_last + end, + [{I,Arity,Func,N}]; +build_call(I, Arity, Func, {return,Val,N}, _Dst) when is_integer(N) -> + [{I,Arity,Func}|copy(Val, {x,0})++[{deallocate,N},return]]; +build_call(I, Arity, Func, none, Dst) -> + [{I,Arity,Func}|copy({x,0}, Dst)]. + +build_apply(Arity, {return,Dst,N}, Dst) when is_integer(N) -> + [{apply_last,Arity,N}]; +build_apply(Arity, {return,Val,N}, _Dst) when is_integer(N) -> + [{apply,Arity}|copy(Val, {x,0})++[{deallocate,N},return]]; +build_apply(Arity, none, Dst) -> + [{apply,Arity}|copy({x,0}, Dst)]. + +cg_instr(put_map, [{atom,assoc},SrcMap|Ss], Dst, Set) -> + Live = get_live(Set), + [{put_map_assoc,{f,0},SrcMap,Dst,Live,{list,Ss}}]; +cg_instr(bs_get_tail, [Src], Dst, Set) -> + Live = get_live(Set), + [{bs_get_tail,Src,Dst,Live}]; +cg_instr(bs_get_position, [Ctx], Dst, Set) -> + Live = get_live(Set), + [{bs_get_position,Ctx,Dst,Live}]; +cg_instr(Op, Args, Dst, _Set) -> + cg_instr(Op, Args, Dst). + +cg_instr(bs_init_writable, Args, Dst) -> + setup_args(Args) ++ [bs_init_writable|copy({x,0}, Dst)]; +cg_instr(bs_restore, [Ctx,Slot], _Dst) -> + case Slot of + {integer,N} -> + [{bs_restore2,Ctx,N}]; + {atom,start} -> + [{bs_restore2,Ctx,Slot}] + end; +cg_instr(bs_save, [Ctx,Slot], _Dst) -> + {integer,N} = Slot, + [{bs_save2,Ctx,N}]; +cg_instr(bs_set_position, [Ctx,Pos], _Dst) -> + [{bs_set_position,Ctx,Pos}]; +cg_instr(build_stacktrace, Args, Dst) -> + setup_args(Args) ++ [build_stacktrace|copy({x,0}, Dst)]; +cg_instr(set_tuple_element=Op, [New,Tuple,{integer,Index}], _Dst) -> + [{Op,New,Tuple,Index}]; +cg_instr({float,clearerror}, [], _Dst) -> + [fclearerror]; +cg_instr({float,get}, [Src], Dst) -> + [{fmove,Src,Dst}]; +cg_instr({float,put}, [Src], Dst) -> + [{fmove,Src,Dst}]; +cg_instr(get_hd=Op, [Src], Dst) -> + [{Op,Src,Dst}]; +cg_instr(get_tl=Op, [Src], Dst) -> + [{Op,Src,Dst}]; +cg_instr(get_tuple_element=Op, [Src,{integer,N}], Dst) -> + [{Op,Src,N,Dst}]; +cg_instr(put_list=Op, [Hd,Tl], Dst) -> + [{Op,Hd,Tl,Dst}]; +cg_instr(put_tuple, Elements, Dst) -> + [{put_tuple2,Dst,{list,Elements}}]; +cg_instr(put_tuple_arity, [{integer,Arity}], Dst) -> + [{put_tuple,Arity,Dst}]; +cg_instr(put_tuple_elements, Elements, _Dst) -> + [{put,E} || E <- Elements]; +cg_instr(raw_raise, Args, Dst) -> + setup_args(Args) ++ [raw_raise|copy({x,0}, Dst)]; +cg_instr(remove_message, [], _Dst) -> + [remove_message]; +cg_instr(resume, [A,B], _Dst) -> + [{bif,raise,{f,0},[A,B],{x,0}}]; +cg_instr(timeout, [], _Dst) -> + [timeout]. + +cg_test(bs_add=Op, Fail, [Src1,Src2,{integer,Unit}], Dst, _I) -> + [{Op,Fail,[Src1,Src2,Unit],Dst}]; +cg_test(bs_skip, Fail, Args, _Dst, I) -> + cg_bs_skip(Fail, Args, I); +cg_test(bs_utf8_size=Op, Fail, [Src], Dst, _I) -> + [{Op,Fail,Src,Dst}]; +cg_test(bs_utf16_size=Op, Fail, [Src], Dst, _I) -> + [{Op,Fail,Src,Dst}]; +cg_test({float,convert}, Fail, [Src], Dst, _I) -> + {f,0} = Fail, %Assertion. + [{fconv,Src,Dst}]; +cg_test({float,Op0}, Fail, Args, Dst, #cg_set{anno=Anno}) -> + Op = case Op0 of + '+' -> fadd; + '-' when length(Args) =:= 2 -> fsub; + '-' -> fnegate; + '*' -> fmul; + '/' -> fdiv + end, + [line(Anno),{bif,Op,Fail,Args,Dst}]; +cg_test(get_map_element, Fail, [Map,Key], Dst, _I) -> + [{get_map_elements,Fail,Map,{list,[Key,Dst]}}]; +cg_test(peek_message, Fail, [], Dst, _I) -> + [{loop_rec,Fail,{x,0}}|copy({x,0}, Dst)]; +cg_test(put_map, Fail, [{atom,exact},SrcMap|Ss], Dst, Set) -> + Live = get_live(Set), + [{put_map_exact,Fail,SrcMap,Dst,Live,{list,Ss}}]; +cg_test(wait_timeout, Fail, [Timeout], _Dst, _) -> + case Timeout of + {atom,infinity} -> + [{wait,Fail}]; + _ -> + [{wait_timeout,Fail,Timeout}] + end. + +cg_bs_get(Fail, #cg_set{dst=Dst0,args=[#b_literal{val=Type}|Ss0]}=Set, St) -> + Op = case Type of + integer -> bs_get_integer2; + float -> bs_get_float2; + binary -> bs_get_binary2; + utf8 -> bs_get_utf8; + utf16 -> bs_get_utf16; + utf32 -> bs_get_utf32 + end, + [Dst|Ss1] = beam_args([Dst0|Ss0], St), + Ss = case Ss1 of + [Ctx,{literal,Flags},Size,{integer,Unit}] -> + %% Plain integer/float/binary. + [Ctx,Size,Unit,field_flags(Flags, Set)]; + [Ctx,{literal,Flags}] -> + %% Utf8/16/32. + [Ctx,field_flags(Flags, Set)] + end, + Live = get_live(Set), + [{test,Op,Fail,Live,Ss,Dst}]. + +cg_bs_skip(Fail, [{atom,Type}|Ss0], Set) -> + Op = case Type of + utf8 -> bs_skip_utf8; + utf16 -> bs_skip_utf16; + utf32 -> bs_skip_utf32; + _ -> bs_skip_bits2 + end, + Live = get_live(Set), + Ss = case Ss0 of + [Ctx,{literal,Flags},Size,{integer,Unit}] -> + %% Plain integer/float/binary. + [Ctx,Size,Unit,field_flags(Flags, Set)]; + [Ctx,{literal,Flags}] -> + %% Utf8/16/32. + [Ctx,Live,field_flags(Flags, Set)] + end, + case {Type,Ss} of + {binary,[_,{atom,all},1,_]} -> + []; + {binary,[R,{atom,all},U,_]} -> + [{test,bs_test_unit,Fail,[R,U]}]; + {_,_} -> + [{test,Op,Fail,Ss}] + end. + +field_flags(Flags, #cg_set{anno=#{location:={File,Line}}}) -> + {field_flags,[{anno,[Line,{file,File}]}|Flags]}; +field_flags(Flags, _) -> + {field_flags,Flags}. + +cg_bs_put(Fail, [{atom,Type},{literal,Flags}|Args]) -> + Op = case Type of + integer -> bs_put_integer; + float -> bs_put_float; + binary -> bs_put_binary; + utf8 -> bs_put_utf8; + utf16 -> bs_put_utf16; + utf32 -> bs_put_utf32 + end, + case Args of + [Src,Size,{integer,Unit}] -> + [{Op,Fail,Size,Unit,{field_flags,Flags},Src}]; + [Src] -> + [{Op,Fail,{field_flags,Flags},Src}] + end. + +cg_bs_init(Dst, Size0, Alloc, Unit, Live, Fail) -> + Op = case Unit of + 1 -> bs_init_bits; + 8 -> bs_init2 + end, + Size = cg_bs_init_size(Size0), + [{Op,Fail,Size,Alloc,Live,{field_flags,[]},Dst}]. + +cg_bs_init_size({x,_}=R) -> R; +cg_bs_init_size({y,_}=R) -> R; +cg_bs_init_size({integer,Int}) -> Int. + +cg_catch(Agg, T0, Context, St0) -> + {Moves,T1} = cg_extract(T0, Agg, St0), + {T,St} = cg_block(T1, Context, St0), + {Moves++T,St}. + +cg_try(Agg, Tag, T0, Context, St0) -> + {Moves0,T1} = cg_extract(T0, Agg, St0), + Moves = order_moves(Moves0, {x,3}), + [#cg_set{op=kill_try_tag}|T2] = T1, + {T,St} = cg_block(T2, Context, St0), + {[{try_case,Tag}|Moves++T],St}. + +cg_extract([#cg_set{op=extract,dst=Dst0,args=Args0}|Is0], Agg, St) -> + [Dst,Agg,{integer,X}] = beam_args([Dst0|Args0], St), + {Ds,Is} = cg_extract(Is0, Agg, St), + case keymember(Dst, 3, Ds) of + true -> + %% This destination will be overwritten. + {Ds,Is}; + false -> + {copy({x,X}, Dst)++Ds,Is} + end; +cg_extract(Is, _, _) -> + {[],Is}. + +copy(Src, Src) -> []; +copy(Src, Dst) -> [{move,Src,Dst}]. + +force_reg({literal,_}=Lit, Reg) -> + {Reg,[{move,Lit,Reg}]}; +force_reg({integer,_}=Lit, Reg) -> + {Reg,[{move,Lit,Reg}]}; +force_reg({atom,_}=Lit, Reg) -> + {Reg,[{move,Lit,Reg}]}; +force_reg({float,_}=Lit, Reg) -> + {Reg,[{move,Lit,Reg}]}; +force_reg(nil=Lit, Reg) -> + {Reg,[{move,Lit,Reg}]}; +force_reg({Kind,_}=R, _) when Kind =:= x; Kind =:= y -> + {R,[]}. + +%% successors(Terminator) -> [Successor]. +%% Return an ordset of all successors for the given terminator. + +successors(#cg_br{succ=Succ,fail=Fail}) -> + ordsets:from_list([Succ,Fail]); +successors(#cg_switch{fail=Fail,list=List}) -> + ordsets:from_list([Fail|[Lbl || {_,Lbl} <- List]]); +successors(#cg_ret{}) -> []. + +%% linearize(Blocks) -> [{BlockLabel,#cg_blk{}}]. +%% Linearize the intermediate representation of the code. Also +%% translate blocks from the SSA records to internal record types +%% used only in this module. + +linearize(Blocks) -> + Linear = beam_ssa:linearize(Blocks), + linearize_1(Linear, Blocks). + +linearize_1([{?BADARG_BLOCK,_}|Ls], Blocks) -> + linearize_1(Ls, Blocks); +linearize_1([{L,Block0}|Ls], Blocks) -> + Block = translate_block(L, Block0, Blocks), + [{L,Block}|linearize_1(Ls, Blocks)]; +linearize_1([], _Blocks) -> []. + +%% translate_block(BlockLabel, #b_blk{}, Blocks) -> #cg_blk{}. +%% Translate a block to the internal records used in this module. +%% Also eliminate phi nodes, replacing them with 'copy' instructions +%% in the predecessor blocks. + +translate_block(L, #b_blk{anno=Anno,is=Is0,last=Last0}, Blocks) -> + Last = translate_terminator(Last0), + PhiCopies = translate_phis(L, Last, Blocks), + Is1 = translate_is(Is0, PhiCopies), + Is = case Anno of + #{frame_size:=Size} -> + Alloc = #cg_alloc{stack=Size}, + [Alloc|Is1]; + #{} -> Is1 + end, + #cg_blk{anno=Anno,is=Is,last=Last}. + +translate_is([#b_set{op=phi}|Is], Tail) -> + translate_is(Is, Tail); +translate_is([#b_set{anno=Anno0,op=Op,dst=Dst,args=Args}=I|Is], Tail) -> + Anno = case beam_ssa:clobbers_xregs(I) of + true -> Anno0#{clobbers=>true}; + false -> Anno0 + end, + [#cg_set{anno=Anno,op=Op,dst=Dst,args=Args}|translate_is(Is, Tail)]; +translate_is([], Tail) -> Tail. + +translate_terminator(#b_ret{anno=Anno,arg=Arg}) -> + Dealloc = case Anno of + #{deallocate:=N} -> N; + #{} -> none + end, + #cg_ret{arg=Arg,dealloc=Dealloc}; +translate_terminator(#b_br{bool=#b_literal{val=true},succ=Succ}) -> + #cg_br{bool=#b_literal{val=true},succ=Succ,fail=Succ}; +translate_terminator(#b_br{bool=#b_literal{val=false},fail=Fail}) -> + #cg_br{bool=#b_literal{val=true},succ=Fail,fail=Fail}; +translate_terminator(#b_br{bool=Bool,succ=Succ,fail=Fail}) -> + #cg_br{bool=Bool,succ=Succ,fail=Fail}; +translate_terminator(#b_switch{arg=Bool,fail=Fail,list=List}) -> + #cg_switch{arg=Bool,fail=Fail,list=List}. + +translate_phis(L, #cg_br{succ=Target,fail=Target}, Blocks) -> + #b_blk{is=Is} = maps:get(Target, Blocks), + Phis = takewhile(fun(#b_set{op=phi}) -> true; + (#b_set{}) -> false + end, Is), + phi_copies(Phis, L); +translate_phis(_, _, _) -> []. + +phi_copies([#b_set{dst=Dst,args=PhiArgs}|Sets], L) -> + CopyArgs = [V || {V,Target} <- PhiArgs, Target =:= L], + [#cg_set{op=copy,dst=Dst,args=CopyArgs}|phi_copies(Sets, L)]; +phi_copies([], _) -> []. + +%% opt_move_to_x0([Instruction]) -> [Instruction]. +%% Simple peep-hole optimization to move a {move,Any,{x,0}} past +%% any kill up to the next call instruction. (To give the loader +%% an opportunity to combine the 'move' and the 'call' instructions.) + +opt_move_to_x0(Moves) -> + opt_move_to_x0(Moves, []). + +opt_move_to_x0([{move,_,{x,0}}=I|Is0], Acc0) -> + case move_past_kill(Is0, I, Acc0) of + impossible -> opt_move_to_x0(Is0, [I|Acc0]); + {Is,Acc} -> opt_move_to_x0(Is, Acc) + end; +opt_move_to_x0([I|Is], Acc) -> + opt_move_to_x0(Is, [I|Acc]); +opt_move_to_x0([], Acc) -> reverse(Acc). + +move_past_kill([{kill,Src}|_], {move,Src,_}, _) -> + impossible; +move_past_kill([{kill,_}=I|Is], Move, Acc) -> + move_past_kill(Is, Move, [I|Acc]); +move_past_kill(Is, Move, Acc) -> + {Is,[Move|Acc]}. + +%% setup_args(Args, Anno, Context) -> [Instruction]. +%% setup_args(Args) -> [Instruction]. +%% Set up X registers for a call. + +setup_args(Args, Anno, none, St) -> + case {setup_args(Args),kill_yregs(Anno, St)} of + {Moves,[]} -> + Moves; + {Moves,Kills} -> + opt_move_to_x0(Moves ++ Kills) + end; +setup_args(Args, _, _, _) -> + setup_args(Args). + +setup_args([]) -> + []; +setup_args([_|_]=Args) -> + Moves = gen_moves(Args, 0, []), + Scratch = {x,1+last(sort([length(Args)-1|[X || {x,X} <- Args]]))}, + order_moves(Moves, Scratch). + +%% kill_yregs(Anno, #cg{}) -> [{kill,{y,Y}}]. +%% Kill Y registers that will not be used again. + +kill_yregs(#{kill_yregs:=Kill}, #cg{regs=Regs}) -> + ordsets:from_list([{kill,maps:get(V, Regs)} || V <- Kill]); +kill_yregs(#{}, #cg{}) -> []. + +%% gen_moves(As, I, Acc) +%% Generate the basic move instruction to move the arguments +%% to their proper registers. The list will be sorted on +%% destinations. (I.e. the move to {x,0} will be first -- +%% see the comment to order_moves/2.) + +gen_moves([A|As], I, Acc) -> + gen_moves(As, I+1, copy(A, {x,I}) ++ Acc); +gen_moves([], _, Acc) -> + keysort(3, Acc). + +%% order_moves([Move], ScratchReg) -> [Move] +%% Orders move instruction so that source registers are not +%% destroyed before they are used. If there are cycles +%% (such as {move,{x,0},{x,1}}, {move,{x,1},{x,1}}), +%% the scratch register is used to break up the cycle. +%% If possible, the first move of the input list is placed +%% last in the result list (to make the move to {x,0} occur +%% just before the call to allow the Beam loader to coalesce +%% the instructions). + +order_moves(Ms, Scr) -> order_moves(Ms, Scr, []). + +order_moves([{move,_,_}=M|Ms0], ScrReg, Acc0) -> + {Chain,Ms} = collect_chain(Ms0, [M], ScrReg), + Acc = reverse(Chain, Acc0), + order_moves(Ms, ScrReg, Acc); +order_moves([], _, Acc) -> Acc. + +collect_chain(Ms, Path, ScrReg) -> + collect_chain(Ms, Path, [], ScrReg). + +collect_chain([{move,Src,Same}=M|Ms0], [{move,Same,_}|_]=Path, Others, ScrReg) -> + case keymember(Src, 3, Path) of + false -> + collect_chain(reverse(Others, Ms0), [M|Path], [], ScrReg); + true -> + %% There is a cycle, which we must break up. + {break_up_cycle(M, Path, ScrReg),reverse(Others, Ms0)} + end; +collect_chain([M|Ms], Path, Others, ScrReg) -> + collect_chain(Ms, Path, [M|Others], ScrReg); +collect_chain([], Path, Others, _) -> + {Path,Others}. + +break_up_cycle({move,Src,_}=M, Path, ScrReg) -> + [{move,ScrReg,Src},M|break_up_cycle1(Src, Path, ScrReg)]. + +break_up_cycle1(Dst, [{move,Src,Dst}|Path], ScrReg) -> + [{move,Src,ScrReg}|Path]; +break_up_cycle1(Dst, [M|Path], LastMove) -> + [M|break_up_cycle1(Dst, Path, LastMove)]. + +%%% +%%% General utility functions. +%%% + +verify_beam_register({x,_}=Reg) -> Reg. + +is_beam_register({x,_}) -> true; +is_beam_register(_) -> false. + +get_register(V, Regs) -> + case is_beam_register(V) of + true -> V; + false -> maps:get(V, Regs) + end. + +beam_args(As, St) -> + [beam_arg(A, St) || A <- As]. + +beam_arg(#b_var{}=Name, #cg{regs=Regs}) -> + maps:get(Name, Regs); +beam_arg(#b_literal{val=Val}, _) -> + if + is_atom(Val) -> {atom,Val}; + is_float(Val) -> {float,Val}; + is_integer(Val) -> {integer,Val}; + Val =:= [] -> nil; + true -> {literal,Val} + end; +beam_arg(Reg, _) -> + verify_beam_register(Reg). + +new_block_label(L, St0) -> + {_Lbl,St} = label_for_block(L, St0), + St. + +def_block_label(L, #cg{labels=Labels,used_labels=Used}) -> + Lbl = maps:get(L, Labels), + case gb_sets:is_member(Lbl, Used) of + false -> []; + true -> [{label,Lbl}] + end. + +use_block_labels(Ls, St) -> + mapfoldl(fun use_block_label/2, St, Ls). + +use_block_label(L, #cg{used_labels=Used,catches=Catches}=St0) -> + {Lbl,St} = label_for_block(L, St0), + case gb_sets:is_member(L, Catches) of + true -> + {{catch_tag,{f,Lbl}}, + St#cg{used_labels=gb_sets:add(Lbl, Used)}}; + false -> + {{f,Lbl},St#cg{used_labels=gb_sets:add(Lbl, Used)}} + end. + +label_for_block(L, #cg{labels=Labels0}=St0) -> + case Labels0 of + #{L:=Lbl} -> + {Lbl,St0}; + #{} -> + {Lbl,St} = new_label(St0), + Labels = Labels0#{L=>Lbl}, + {Lbl,St#cg{labels=Labels}} + end. + +%% local_func_label(Name, Arity, State) -> {Label,State'} +%% local_func_label({Name,Arity}, State) -> {Label,State'} +%% Get the function entry label for a local function. + +local_func_label(Name, Arity, St) -> + local_func_label({Name,Arity}, St). + +local_func_label(Key, #cg{functable=Map}=St0) -> + case Map of + #{Key := Label} -> + {Label,St0}; + _ -> + {Label,St} = new_label(St0), + {Label,St#cg{functable=Map#{Key => Label}}} + end. + +%% is_gc_bif(Name, Args) -> true|false. +%% Determines whether the BIF Name/Arity might do a GC. + +-spec is_gc_bif(atom(), [beam_ssa:value()]) -> boolean(). + +is_gc_bif(hd, [_]) -> false; +is_gc_bif(tl, [_]) -> false; +is_gc_bif(self, []) -> false; +is_gc_bif(node, []) -> false; +is_gc_bif(node, [_]) -> false; +is_gc_bif(element, [_,_]) -> false; +is_gc_bif(get, [_]) -> false; +is_gc_bif(is_map_key, [_,_]) -> false; +is_gc_bif(map_get, [_,_]) -> false; +is_gc_bif(tuple_size, [_]) -> false; +is_gc_bif(Bif, Args) -> + Arity = length(Args), + not (erl_internal:bool_op(Bif, Arity) orelse + erl_internal:new_type_test(Bif, Arity) orelse + erl_internal:comp_op(Bif, Arity)). + +%% new_label(St) -> {L,St}. + +new_label(#cg{lcount=Next}=St) -> + {Next,St#cg{lcount=Next+1}}. + +%% call_line(tail|body, Func, Anno) -> [] | [{line,...}]. +%% Produce a line instruction if it will be needed by the +%% call to Func. + +call_line(_Context, {extfunc,Mod,Name,Arity}, Anno) -> + case erl_bifs:is_safe(Mod, Name, Arity) of + false -> + %% The call could be to a BIF. + %% We'll need a line instruction in case the + %% BIF call fails. + [line(Anno)]; + true -> + %% Call to a safe BIF. Since it cannot fail, + %% we don't need any line instruction here. + [] + end; +call_line(body, _, Anno) -> + [line(Anno)]; +call_line(tail, local, _) -> + %% Tail-recursive call to a local function. A line + %% instruction will not be useful. + []; +call_line(tail, _, Anno) -> + %% Call to a fun. + [line(Anno)]. + +%% line(Le) -> {line,[] | {location,File,Line}} +%% Create a line instruction, containing information about +%% the current filename and line number. A line information +%% instruction should be placed before any operation that could +%% cause an exception. + +line(#{location:={File,Line}}) -> + {line,[{location,File,Line}]}; +line(#{}) -> + {line,[]}. + +flatmapfoldl(F, Accu0, [Hd|Tail]) -> + {R,Accu1} = F(Hd, Accu0), + {Rs,Accu2} = flatmapfoldl(F, Accu1, Tail), + {R++Rs,Accu2}; +flatmapfoldl(_, Accu, []) -> {[],Accu}. diff --git a/lib/compiler/src/beam_ssa_dead.erl b/lib/compiler/src/beam_ssa_dead.erl new file mode 100644 index 0000000000..bb43a550ae --- /dev/null +++ b/lib/compiler/src/beam_ssa_dead.erl @@ -0,0 +1,1076 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% Dead code is code that is executed but has no effect. This +%% optimization pass either removes dead code or jumps around it, +%% potentially making it unreachable so that it can be dropped +%% the next time beam_ssa:linearize/1 is called. +%% + +-module(beam_ssa_dead). +-export([opt/1]). + +-include("beam_ssa.hrl"). +-import(lists, [append/1,keymember/3,last/1,member/2, + takewhile/2,reverse/1]). + +-type used_vars() :: #{beam_ssa:label():=ordsets:ordset(beam_ssa:var_name())}. + +-type basic_type_test() :: atom() | {'is_tagged_tuple',pos_integer(),atom()}. +-type type_test() :: basic_type_test() | {'not',basic_type_test()}. +-type op_name() :: atom(). +-type basic_rel_op() :: {op_name(),beam_ssa:b_var(),beam_ssa:value()} | + {basic_type_test(),beam_ssa:value()}. +-type rel_op() :: {op_name(),beam_ssa:b_var(),beam_ssa:value()} | + {type_test(),beam_ssa:value()}. + +-record(st, + {bs :: beam_ssa:block_map(), + us :: used_vars(), + skippable :: #{beam_ssa:label():='true'}, + rel_op=none :: 'none' | rel_op(), + target=any :: 'any' | 'one_way' | beam_ssa:label() + }). + +-spec opt([{Label0,Block0}]) -> [{Label,Block}] when + Label0 :: beam_ssa:label(), + Block0 :: beam_ssa:b_blk(), + Label :: beam_ssa:label(), + Block :: beam_ssa:b_blk(). + +opt(Linear) -> + {Used,Skippable} = used_vars(Linear), + Blocks0 = maps:from_list(Linear), + St0 = #st{bs=Blocks0,us=Used,skippable=Skippable}, + St = shortcut_opt(St0), + #st{bs=Blocks} = combine_eqs(St#st{us=#{}}), + beam_ssa:linearize(Blocks). + +%%% +%%% Shortcut br/switch targets. +%%% +%%% A br/switch may branch to another br/switch that in turn always +%%% branches to another target. Rewrite br/switch to refer to the +%%% ultimate targets directly. That will save execution time, but +%%% could also reduce the size of the code if some of the original +%%% targets become unreachable and be deleted. +%%% +%%% When rewriting branches, we must be careful not to skip instructions +%%% that have side effects or that bind variables that will be used +%%% at the new target. +%%% +%%% We must also avoid branching to phi nodes. The reason is +%%% twofold. First, we might create a critical edge which is strictly +%%% forbidden. Second, there will be a branch from a block that is not +%%% listed in the list of predecessors in the phi node. Those +%%% limitations could probably be overcome, but it is not clear how +%%% much that would improve the code. +%%% + +shortcut_opt(#st{bs=Blocks}=St) -> + %% Processing the blocks in reverse post order seems to give more + %% opportunities for optimizations compared to post order. (Based on + %% running scripts/diffable with both PO and RPO and looking at + %% the diff.) + %% + %% Unfortunately, processing the blocks in reverse post order + %% potentially makes the time complexity quadratic or even cubic if + %% the ordset of unset variables grows large, instead of + %% linear for post order processing. We try to still get reasonable + %% compilation times by optimizations that will keep the constant + %% factor as low as possible, and we try to avoid the cubic time + %% complexity by trying to keep the set of unset variables as small + %% as possible. + + Ls = beam_ssa:rpo(Blocks), + shortcut_opt(Ls, #{}, St). + +shortcut_opt([L|Ls], Bs, #st{bs=Blocks0}=St) -> + #b_blk{is=Is,last=Last0} = Blk0 = get_block(L, St), + case shortcut_terminator(Last0, Is, L, Bs, St) of + Last0 -> + %% No change. No need to update the block. + shortcut_opt(Ls, Bs, St); + Last -> + %% The terminator was simplified in some way. + %% Update the block. + Blk = Blk0#b_blk{last=Last}, + Blocks = Blocks0#{L=>Blk}, + shortcut_opt(Ls, Bs, St#st{bs=Blocks}) + end; +shortcut_opt([], _, St) -> St. + +shortcut_terminator(#b_br{bool=#b_literal{val=true},succ=Succ0}, + _Is, From, Bs, St0) -> + St = St0#st{rel_op=none}, + shortcut(Succ0, From, Bs, St); +shortcut_terminator(#b_br{bool=#b_var{}=Bool,succ=Succ0,fail=Fail0}=Br, + Is, From, Bs, St0) -> + St = St0#st{target=one_way}, + RelOp = get_rel_op(Bool, Is), + SuccBs = bind_var(Bool, #b_literal{val=true}, Bs), + BrSucc = shortcut(Succ0, From, SuccBs, St#st{rel_op=RelOp}), + FailBs = bind_var(Bool, #b_literal{val=false}, Bs), + BrFail = shortcut(Fail0, From, FailBs, St#st{rel_op=invert_op(RelOp)}), + case {BrSucc,BrFail} of + {#b_br{bool=#b_literal{val=true},succ=Succ}, + #b_br{bool=#b_literal{val=true},succ=Fail}} + when Succ =/= Succ0; Fail =/= Fail0 -> + %% One or both of the targets were cut short. + beam_ssa:normalize(Br#b_br{succ=Succ,fail=Fail}); + {_,_} -> + %% No change. + Br + end; +shortcut_terminator(#b_switch{arg=Bool,list=List0}=Sw, _Is, From, Bs, St) -> + List = shortcut_switch(List0, Bool, From, Bs, St), + beam_ssa:normalize(Sw#b_switch{list=List}); +shortcut_terminator(Last, _Is, _Bs, _From, _St) -> + Last. + +shortcut_switch([{Lit,L0}|T], Bool, From, Bs, St0) -> + RelOp = {'=:=',Bool,Lit}, + St = St0#st{rel_op=RelOp}, + #b_br{bool=#b_literal{val=true},succ=L} = + shortcut(L0, From, bind_var(Bool, Lit, Bs), St#st{target=one_way}), + [{Lit,L}|shortcut_switch(T, Bool, From, Bs, St0)]; +shortcut_switch([], _, _, _, _) -> []. + +shortcut(L, From, Bs, St) -> + shortcut_1(L, From, Bs, ordsets:new(), St). + +shortcut_1(L, From, Bs0, UnsetVars0, St) -> + case shortcut_2(L, From, Bs0, UnsetVars0, St) of + none -> + %% No more shortcuts found. Package up the previous + %% label in an unconditional branch. + #b_br{bool=#b_literal{val=true},succ=L,fail=L}; + {#b_br{bool=#b_var{}}=Br,_,_} -> + %% This is a two-way branch. We can't do any better. + Br; + {#b_br{bool=#b_literal{val=true},succ=Succ},Bs,UnsetVars} -> + %% This is a safe `br`, but try to find a better one. + shortcut_1(Succ, L, Bs, UnsetVars, St) + end. + +%% Try to shortcut this block, branching to a successor. +shortcut_2(L, From, Bs0, UnsetVars0, St) -> + #b_blk{is=Is,last=Last} = get_block(L, St), + case eval_is(Is, From, Bs0, St) of + none -> + %% It is not safe to avoid this block because it + %% has instructions with potential side effects. + none; + Bs -> + %% The instructions in the block (if any) don't + %% have any side effects and can be skipped. + %% Evaluate the terminator. + case eval_terminator(Last, Bs, St) of + none -> + %% The terminator is not suitable (could be + %% because it is a switch that can't be simplified + %% or it is a ret instruction). + none; + #b_br{}=Br -> + %% We have a potentially suitable br. + %% Now update the set of variables that will never + %% be set if this block will be skipped. + case update_unset_vars(L, Is, Br, UnsetVars0, St) of + unsafe -> + %% It is unsafe to use this br, + %% because it refers to a variable defined + %% in this block. + shortcut_unsafe_br(Br, L, Bs, UnsetVars0, St); + UnsetVars -> + %% Continue checking whether this br is + %% suitable. + shortcut_test_br(Br, L, Bs, UnsetVars, St) + end + end + end. + +shortcut_test_br(Br, From, Bs, UnsetVars, St) -> + case is_br_safe(UnsetVars, Br, St) of + false -> + shortcut_unsafe_br(Br, From, Bs, UnsetVars, St); + true -> + shortcut_safe_br(Br, From, Bs, UnsetVars, St) + end. + +shortcut_unsafe_br(Br, From, Bs, UnsetVars, #st{target=Target}=St) -> + %% Branching using this `br` is unsafe, either because it + %% is an unconditional branch to a phi node, or because + %% one or more of the variables that are not set will be + %% used. Try to follow branches of this `br`, to find a + %% safe `br`. + case Br of + #b_br{bool=#b_literal{val=true},succ=L} -> + case Target of + L -> + %% We have reached the forced target, and it + %% is unsafe. Give up. + none; + _ -> + %% Try following this branch to see whether it + %% leads to a safe `br`. + shortcut_2(L, From, Bs, UnsetVars, St) + end; + #b_br{bool=#b_var{},succ=Succ,fail=Fail} -> + case {Succ,Fail} of + {L,Target} -> + %% The failure label is the forced target. + %% Try following the success label to see + %% whether it also ultimately ends up at the + %% forced target. + shortcut_2(L, From, Bs, UnsetVars, St); + {Target,L} -> + %% The success label is the forced target. + %% Try following the failure label to see + %% whether it also ultimately ends up at the + %% forced target. + shortcut_2(L, From, Bs, UnsetVars, St); + {_,_} -> + case Target of + any -> + %% This two-way branch is unsafe. Try + %% reducing it to a one-way branch. + shortcut_two_way(Br, From, Bs, UnsetVars, St); + one_way -> + %% This two-way branch is unsafe. Try + %% reducing it to a one-way branch. + shortcut_two_way(Br, From, Bs, UnsetVars, St); + _ when is_integer(Target) -> + %% This two-way branch is unsafe, and + %% there already is a forced target. + %% Give up. + none + end + end + end. + +shortcut_safe_br(Br, From, Bs, UnsetVars, #st{target=Target}=St) -> + %% This `br` instruction is safe. It does not branch to a phi + %% node, and all variables that will be used are guaranteed to be + %% defined. + case Br of + #b_br{bool=#b_literal{val=true},succ=L} -> + %% This is a one-way branch. + case Target of + any -> + %% No forced target. Success! + {Br,Bs,UnsetVars}; + one_way -> + %% The target must be a one-way branch, which this + %% `br` is. Success! + {Br,Bs,UnsetVars}; + L when is_integer(Target) -> + %% The forced target is L. Success! + {Br,Bs,UnsetVars}; + _ when is_integer(Target) -> + %% Wrong forced target. Try following this branch + %% to see if it ultimately ends up at the forced + %% target. + shortcut_2(L, From, Bs, UnsetVars, St) + end; + #b_br{bool=#b_var{}} -> + %% This is a two-way branch. + if + Target =:= any; Target =:= one_way -> + %% No specific forced target. Try to reduce the + %% two-way branch to an one-way branch. + case shortcut_two_way(Br, From, Bs, UnsetVars, St) of + none when Target =:= any -> + %% This `br` can't be reduced to a one-way + %% branch. Return the `br` as-is. + {Br,Bs,UnsetVars}; + none when Target =:= one_way -> + %% This `br` can't be reduced to a one-way + %% branch. The caller wants a one-way + %% branch. Give up. + none; + {_,_,_}=Res -> + %% This `br` was successfully reduced to a + %% one-way branch. + Res + end; + is_integer(Target) -> + %% There is a forced target, which can't + %% be reached because this `br` is a two-way + %% branch. Give up. + none + end + end. + +update_unset_vars(L, Is, Br, UnsetVars, #st{skippable=Skippable}) -> + case is_map_key(L, Skippable) of + true -> + %% None of the variables used in this block are used in + %% the successors. Thus, there is no need to add the + %% variables to the set of unset variables. + case Br of + #b_br{bool=#b_var{}=Bool} -> + case keymember(Bool, #b_set.dst, Is) of + true -> + %% Bool is a variable defined in this + %% block. Using the br instruction from + %% this block (and skipping the body of + %% the block) is unsafe. + unsafe; + false -> + %% Bool is either a variable not defined + %% in this block or a literal. Adding it + %% to the UnsetVars set would not change + %% the outcome of the tests in + %% is_br_safe/2. + UnsetVars + end; + #b_br{} -> + UnsetVars + end; + false -> + %% Some variables defined in this block are used by + %% successors. We must update the set of unset variables. + SetInThisBlock = [V || #b_set{dst=V} <- Is], + ordsets:union(UnsetVars, ordsets:from_list(SetInThisBlock)) + end. + +shortcut_two_way(#b_br{succ=Succ,fail=Fail}, From, Bs0, UnsetVars0, St0) -> + case shortcut_2(Succ, From, Bs0, UnsetVars0, St0#st{target=Fail}) of + {#b_br{bool=#b_literal{},succ=Fail},_,_}=Res -> + Res; + none -> + St = St0#st{target=Succ}, + case shortcut_2(Fail, From, Bs0, UnsetVars0, St) of + {#b_br{bool=#b_literal{},succ=Succ},_,_}=Res -> + Res; + none -> + none + end + end. + +get_block(L, St) -> + #st{bs=#{L:=Blk}} = St, + Blk. + +is_br_safe(UnsetVars, Br, #st{us=Us}=St) -> + %% Check that none of the unset variables will be used. + case Br of + #b_br{bool=#b_var{}=V,succ=Succ,fail=Fail} -> + #{Succ:=Used0,Fail:=Used1} = Us, + + %% A two-way branch never branches to a phi node, so there + %% is no need to check for phi nodes here. + not member(V, UnsetVars) andalso + ordsets:is_disjoint(Used0, UnsetVars) andalso + ordsets:is_disjoint(Used1, UnsetVars); + #b_br{succ=Same,fail=Same} -> + %% An unconditional branch must not jump to + %% a phi node. + not is_forbidden(Same, St) andalso + ordsets:is_disjoint(map_get(Same, Us), UnsetVars) + end. + +is_forbidden(L, St) -> + case get_block(L, St) of + #b_blk{is=[#b_set{op=phi}|_]} -> true; + #b_blk{is=[#b_set{op=peek_message}|_]} -> true; + #b_blk{} -> false + end. + + +%% Evaluate the instructions in the block. +%% Return the updated bindings, or 'none' if there is +%% any instruction with potential side effects. + +eval_is([#b_set{op=phi,dst=Dst,args=Args}|Is], From, Bs0, St) -> + Val = get_phi_arg(Args, From), + Bs = bind_var(Dst, Val, Bs0), + eval_is(Is, From, Bs, St); +eval_is([#b_set{op={bif,_},dst=Dst}=I0|Is], From, Bs, St) -> + I = sub(I0, Bs), + case eval_bif(I, St) of + #b_literal{}=Val -> + eval_is(Is, From, bind_var(Dst, Val, Bs), St); + none -> + eval_is(Is, From, Bs, St) + end; +eval_is([#b_set{op=Op,dst=Dst}=I|Is], From, Bs, St) + when Op =:= is_tagged_tuple; Op =:= is_nonempty_list -> + #b_set{args=Args} = sub(I, Bs), + case eval_rel_op(Op, Args, St) of + #b_literal{}=Val -> + eval_is(Is, From, bind_var(Dst, Val, Bs), St); + none -> + eval_is(Is, From, Bs, St) + end; +eval_is([#b_set{}=I|Is], From, Bs, St) -> + case beam_ssa:no_side_effect(I) of + true -> + %% This instruction has no side effects. It can + %% safely be omitted. + eval_is(Is, From, Bs, St); + false -> + %% This instruction may have some side effect. + %% It is not safe to avoid this instruction. + none + end; +eval_is([], _From, Bs, _St) -> Bs. + +get_phi_arg([{Val,From}|_], From) -> Val; +get_phi_arg([_|As], From) -> get_phi_arg(As, From). + +eval_terminator(#b_br{bool=#b_var{}=Bool}=Br, Bs, _St) -> + Val = get_value(Bool, Bs), + beam_ssa:normalize(Br#b_br{bool=Val}); +eval_terminator(#b_br{bool=#b_literal{}}=Br, _Bs, _St) -> + beam_ssa:normalize(Br); +eval_terminator(#b_switch{arg=Arg,fail=Fail,list=List}=Sw, Bs, St) -> + case get_value(Arg, Bs) of + #b_literal{}=Val -> + %% Literal argument. Simplify to a `br`. + beam_ssa:normalize(Sw#b_switch{arg=Val}); + #b_var{} -> + %% Try optimizing the switch. + case eval_switch(List, Arg, St, Fail) of + none -> + none; + To when is_integer(To) -> + %% Either one of the values in the switch + %% matched a previous value in a '=:=' test, or + %% none of the values matched a previous test. + #b_br{bool=#b_literal{val=true},succ=To,fail=To} + end + end; +eval_terminator(#b_ret{}, _Bs, _St) -> + none. + +eval_switch(List, Arg, #st{rel_op={_,Arg,_}=PrevOp}, Fail) -> + %% There is a previous relational operator testing the same variable. + %% Optimization may be possible. + eval_switch_1(List, Arg, PrevOp, Fail); +eval_switch(_, _, _, _) -> + %% There is either no previous relational operator, or it tests + %% a different variable. Nothing to optimize. + none. + +eval_switch_1([{Lit,Lbl}|T], Arg, PrevOp, Fail) -> + RelOp = {'=:=',Arg,Lit}, + case will_succeed(PrevOp, RelOp) of + yes -> + %% Success. This branch will always be taken. + Lbl; + no -> + %% This branch will never be taken. + eval_switch_1(T, Arg, PrevOp, Fail); + maybe -> + %% This label could be reached. + eval_switch_1(T, Arg, PrevOp, none) + end; +eval_switch_1([], _Arg, _PrevOp, Fail) -> + %% Fail is now either the failure label or 'none'. + Fail. + +bind_var(Var, Val0, Bs) -> + Val = get_value(Val0, Bs), + Bs#{Var=>Val}. + +get_value(#b_var{}=Var, Bs) -> + case Bs of + #{Var:=Val} -> get_value(Val, Bs); + #{} -> Var + end; +get_value(#b_literal{}=Lit, _Bs) -> Lit. + +eval_bif(#b_set{op={bif,Bif},args=Args}, St) -> + Arity = length(Args), + case erl_bifs:is_pure(erlang, Bif, Arity) of + false -> + none; + true -> + case get_lit_args(Args) of + none -> + %% Not literal arguments. Try to evaluate + %% it based on a previous relational operator. + eval_rel_op({bif,Bif}, Args, St); + LitArgs -> + try apply(erlang, Bif, LitArgs) of + Val -> #b_literal{val=Val} + catch + error:_ -> none + end + end + end. + +get_lit_args([#b_literal{val=Lit1}]) -> + [Lit1]; +get_lit_args([#b_literal{val=Lit1}, + #b_literal{val=Lit2}]) -> + [Lit1,Lit2]; +get_lit_args([#b_literal{val=Lit1}, + #b_literal{val=Lit2}, + #b_literal{val=Lit3}]) -> + [Lit1,Lit2,Lit3]; +get_lit_args(_) -> none. + +%%% +%%% Handling of relational operators. +%%% + +get_rel_op(Bool, [_|_]=Is) -> + case last(Is) of + #b_set{op=Op,dst=Bool,args=Args} -> + normalize_op(Op, Args); + #b_set{} -> + none + end; +get_rel_op(_, []) -> none. + +%% normalize_op(Instruction) -> {Normalized,FailLabel} | error +%% Normalized = {Operator,Variable,Variable|Literal} | +%% {TypeTest,Variable} +%% Operation = '<' | '=<' | '=:=' | '=/=' | '>=' | '>' +%% TypeTest = is_atom | is_integer ... +%% Variable = #b_var{} +%% Literal = #b_literal{} +%% +%% Normalize a relational operator to facilitate further +%% comparisons between operators. Always make the register +%% operand the first operand. If there are two registers, +%% order the registers in lexical order. +%% +%% For example, this instruction: +%% +%% #b_set{op={bif,=<},args=[#b_literal{}, #b_var{}} +%% +%% will be normalized to: +%% +%% {'=<',#b_var{},#b_literal{}} + +-spec normalize_op(Op, Args) -> NormalizedOp | 'none' when + Op :: beam_ssa:op(), + Args :: [beam_ssa:value()], + NormalizedOp :: basic_rel_op(). + +normalize_op(is_tagged_tuple, [Arg,#b_literal{val=Size},#b_literal{val=Tag}]) + when is_integer(Size), is_atom(Tag) -> + {{is_tagged_tuple,Size,Tag},Arg}; +normalize_op(is_nonempty_list, [Arg]) -> + {is_nonempty_list,Arg}; +normalize_op({bif,Bif}, [Arg]) -> + case erl_internal:new_type_test(Bif, 1) of + true -> {Bif,Arg}; + false -> none + end; +normalize_op({bif,Bif}, [_,_]=Args) -> + case erl_internal:comp_op(Bif, 2) of + true -> + normalize_op_1(Bif, Args); + false -> + none + end; +normalize_op(_, _) -> none. + +normalize_op_1(Bif, Args) -> + case Args of + [#b_literal{}=Arg1,#b_var{}=Arg2] -> + {turn_op(Bif),Arg2,Arg1}; + [#b_var{}=Arg1,#b_literal{}=Arg2] -> + {Bif,Arg1,Arg2}; + [#b_var{}=A,#b_var{}=B] -> + if A < B -> {Bif,A,B}; + true -> {turn_op(Bif),B,A} + end; + [#b_literal{},#b_literal{}] -> + none + end. + +-spec invert_op(basic_rel_op() | 'none') -> rel_op() | 'none'. + +invert_op({Op,Arg1,Arg2}) -> + {invert_op_1(Op),Arg1,Arg2}; +invert_op({TypeTest,Arg}) -> + {{'not',TypeTest},Arg}; +invert_op(none) -> none. + +invert_op_1('>=') -> '<'; +invert_op_1('<') -> '>='; +invert_op_1('=<') -> '>'; +invert_op_1('>') -> '=<'; +invert_op_1('=:=') -> '=/='; +invert_op_1('=/=') -> '=:='; +invert_op_1('==') -> '/='; +invert_op_1('/=') -> '=='. + +turn_op('<') -> '>'; +turn_op('=<') -> '>='; +turn_op('>') -> '<'; +turn_op('>=') -> '=<'; +turn_op('=:='=Op) -> Op; +turn_op('=/='=Op) -> Op; +turn_op('=='=Op) -> Op; +turn_op('/='=Op) -> Op. + +eval_rel_op(_Bif, _Args, #st{rel_op=none}) -> + none; +eval_rel_op(Bif, Args, #st{rel_op=Prev}) -> + case normalize_op(Bif, Args) of + none -> + none; + RelOp -> + case will_succeed(Prev, RelOp) of + yes -> #b_literal{val=true}; + no -> #b_literal{val=false}; + maybe -> none + end + end. + +%% will_succeed(PrevCondition, Condition) -> yes | no | maybe +%% PrevCondition is a condition known to be true. This function +%% will tell whether Condition will succeed. + +will_succeed({_Op,_Var,_Value}=Same, {_Op,_Var,_Value}=Same) -> + %% Repeated test. + yes; +will_succeed({Op1,Var,#b_literal{val=A}}, {Op2,Var,#b_literal{val=B}}) -> + will_succeed_1(Op1, A, Op2, B); +will_succeed({Op1,Var,#b_var{}=A}, {Op2,Var,#b_var{}=B}) -> + will_succeed_vars(Op1, A, Op2, B); +will_succeed({'=:=',Var,#b_literal{val=A}}, {TypeTest,Var}) -> + eval_type_test(TypeTest, A); +will_succeed({_,_}=Same, {_,_}=Same) -> + %% Repeated type test. + yes; +will_succeed({Test1,Var}, {Test2,Var}) -> + will_succeed_test(Test1, Test2); +will_succeed({_,_}, {_,_}) -> + maybe; +will_succeed({_,_}, {_,_,_}) -> + maybe; +will_succeed({_,_,_}, {_,_}) -> + maybe; +will_succeed({_,_,_}, {_,_,_}) -> + maybe. + +will_succeed_test({'not',Test1}, Test2) -> + case Test1 =:= Test2 of + true -> no; + false -> maybe + end; +will_succeed_test(is_tuple, {is_tagged_tuple,_,_}) -> + maybe; +will_succeed_test({is_tagged_tuple,_,_}, is_tuple) -> + yes; +will_succeed_test(is_list, is_nonempty_list) -> + maybe; +will_succeed_test(is_nonempty_list, is_list) -> + yes; +will_succeed_test(T1, T2) -> + case is_numeric_test(T1) andalso is_numeric_test(T2) of + true -> maybe; + false -> no + end. + +will_succeed_1('=:=', A, '<', B) -> + if + B =< A -> no; + true -> yes + end; +will_succeed_1('=:=', A, '=<', B) -> + if + B < A -> no; + true -> yes + end; +will_succeed_1('=:=', A, '=:=', B) when A =/= B -> + no; +will_succeed_1('=:=', A, '=/=', B) -> + if + A =:= B -> no; + true -> yes + end; +will_succeed_1('=:=', A, '>=', B) -> + if + B > A -> no; + true -> yes + end; +will_succeed_1('=:=', A, '>', B) -> + if + B >= A -> no; + true -> yes + end; + +will_succeed_1('=/=', A, '=:=', B) when A =:= B -> no; + +will_succeed_1('<', A, '=:=', B) when B >= A -> no; +will_succeed_1('<', A, '=/=', B) when B >= A -> yes; +will_succeed_1('<', A, '<', B) when B >= A -> yes; +will_succeed_1('<', A, '=<', B) when B > A -> yes; +will_succeed_1('<', A, '>=', B) when B > A -> no; +will_succeed_1('<', A, '>', B) when B >= A -> no; + +will_succeed_1('=<', A, '=:=', B) when B > A -> no; +will_succeed_1('=<', A, '=/=', B) when B > A -> yes; +will_succeed_1('=<', A, '<', B) when B > A -> yes; +will_succeed_1('=<', A, '=<', B) when B >= A -> yes; +will_succeed_1('=<', A, '>=', B) when B > A -> no; +will_succeed_1('=<', A, '>', B) when B >= A -> no; + +will_succeed_1('>=', A, '=:=', B) when B < A -> no; +will_succeed_1('>=', A, '=/=', B) when B < A -> yes; +will_succeed_1('>=', A, '<', B) when B =< A -> no; +will_succeed_1('>=', A, '=<', B) when B < A -> no; +will_succeed_1('>=', A, '>=', B) when B =< A -> yes; +will_succeed_1('>=', A, '>', B) when B < A -> yes; + +will_succeed_1('>', A, '=:=', B) when B =< A -> no; +will_succeed_1('>', A, '=/=', B) when B =< A -> yes; +will_succeed_1('>', A, '<', B) when B =< A -> no; +will_succeed_1('>', A, '=<', B) when B < A -> no; +will_succeed_1('>', A, '>=', B) when B =< A -> yes; +will_succeed_1('>', A, '>', B) when B < A -> yes; + +will_succeed_1('==', A, '==', B) -> + if + A == B -> yes; + true -> no + end; +will_succeed_1('==', A, '/=', B) -> + if + A == B -> no; + true -> yes + end; +will_succeed_1('/=', A, '/=', B) when A == B -> yes; +will_succeed_1('/=', A, '==', B) when A == B -> no; + +will_succeed_1(_, _, _, _) -> maybe. + +will_succeed_vars('=/=', Val, '=:=', Val) -> no; +will_succeed_vars('=:=', Val, '=/=', Val) -> no; +will_succeed_vars('=:=', Val, '>=', Val) -> yes; +will_succeed_vars('=:=', Val, '=<', Val) -> yes; + +will_succeed_vars('/=', Val1, '==', Val2) when Val1 == Val2 -> no; +will_succeed_vars('==', Val1, '/=', Val2) when Val1 == Val2 -> no; + +will_succeed_vars(_, _, _, _) -> maybe. + +is_numeric_test(is_float) -> true; +is_numeric_test(is_integer) -> true; +is_numeric_test(is_number) -> true; +is_numeric_test(_) -> false. + +eval_type_test(Test, Arg) -> + case eval_type_test_1(Test, Arg) of + true -> yes; + false -> no + end. + +eval_type_test_1(is_nonempty_list, Arg) -> + case Arg of + [_|_] -> true; + _ -> false + end; +eval_type_test_1({is_tagged_tuple,Sz,Tag}, Arg) -> + if + tuple_size(Arg) =:= Sz, element(1, Arg) =:= Tag -> + true; + true -> + false + end; +eval_type_test_1(Test, Arg) -> + erlang:Test(Arg). + +%%% +%%% Combine bif:'=:=' and switch instructions +%%% to switch instructions. +%%% +%%% Consider this code: +%%% +%%% 0: +%%% @ssa_bool = bif:'=:=' Var, literal 1 +%%% br @ssa_bool, label 2, label 3 +%%% +%%% 2: +%%% ret literal a +%%% +%%% 3: +%%% @ssa_bool:7 = bif:'=:=' Var, literal 2 +%%% br @ssa_bool:7, label 4, label 999 +%%% +%%% 4: +%%% ret literal b +%%% +%%% 999: +%%% . +%%% . +%%% . +%%% +%%% The two bif:'=:=' instructions can be combined +%%% to a switch: +%%% +%%% 0: +%%% switch Var, label 999, [ { literal 1, label 2 }, +%%% { literal 2, label 3 } ] +%%% +%%% 2: +%%% ret literal a +%%% +%%% 4: +%%% ret literal b +%%% +%%% 999: +%%% . +%%% . +%%% . +%%% + +combine_eqs(#st{bs=Blocks}=St) -> + Ls = reverse(beam_ssa:rpo(Blocks)), + combine_eqs_1(Ls, St). + +combine_eqs_1([L|Ls], #st{bs=Blocks0}=St0) -> + case comb_get_sw(L, St0) of + none -> + combine_eqs_1(Ls, St0); + {_,Arg,_,Fail0,List0} -> + case comb_get_sw(Fail0, St0) of + {true,Arg,Fail1,Fail,List1} -> + %% Another switch/br with the same arguments was + %% found. Try combining them. + case combine_lists(Fail1, List0, List1, Blocks0) of + none -> + %% Different types of literals in the lists, + %% or the success cases in the first switch + %% could branch to the second switch + %% (increasing code size and repeating tests). + combine_eqs_1(Ls, St0); + List -> + %% Everything OK! Combine the lists. + Sw0 = #b_switch{arg=Arg,fail=Fail,list=List}, + Sw = beam_ssa:normalize(Sw0), + Blk0 = map_get(L, Blocks0), + Blk = Blk0#b_blk{last=Sw}, + Blocks = Blocks0#{L:=Blk}, + St = St0#st{bs=Blocks}, + combine_eqs_1(Ls, St) + end; + {true,_OtherArg,_,_,_} -> + %% The other switch/br uses a different Arg. + combine_eqs_1(Ls, St0); + {false,_,_,_,_} -> + %% Not safe: Bindings of variables that will be used + %% or execution of instructions with potential + %% side effects will be skipped. + combine_eqs_1(Ls, St0); + none -> + %% No switch/br at this label. + combine_eqs_1(Ls, St0) + end + end; +combine_eqs_1([], St) -> St. + +comb_get_sw(L, Blocks) -> + comb_get_sw(L, true, Blocks). + +comb_get_sw(L, Safe0, #st{bs=Blocks,skippable=Skippable}) -> + #b_blk{is=Is,last=Last} = map_get(L, Blocks), + Safe1 = Safe0 andalso is_map_key(L, Skippable), + case Last of + #b_ret{} -> + none; + #b_br{bool=#b_var{}=Bool,succ=Succ,fail=Fail} -> + case comb_is(Is, Bool, Safe1) of + {none,_} -> + none; + {#b_set{op={bif,'=:='},args=[#b_var{}=Arg,#b_literal{}=Lit]},Safe} -> + {Safe,Arg,L,Fail,[{Lit,Succ}]}; + {#b_set{},_} -> + none + end; + #b_br{} -> + none; + #b_switch{arg=#b_var{}=Arg,fail=Fail,list=List} -> + {none,Safe} = comb_is(Is, none, Safe1), + {Safe,Arg,L,Fail,List} + end. + +comb_is([#b_set{dst=#b_var{}=Bool}=I], Bool, Safe) -> + {I,Safe}; +comb_is([#b_set{}=I|Is], Bool, Safe0) -> + Safe = Safe0 andalso beam_ssa:no_side_effect(I), + comb_is(Is, Bool, Safe); +comb_is([], _Bool, Safe) -> + {none,Safe}. + +%% combine_list(Fail, List1, List2, Blocks) -> List|none. +%% Try to combine two switch lists, returning the combined +%% list or 'none' if not possible. +%% +%% The values in the two lists must be all of the same type. +%% +%% The code reached from the labels in the first list must +%% not reach the failure label (if they do, tests could +%% be repeated). +%% + +combine_lists(Fail, L1, L2, Blocks) -> + Ls = beam_ssa:rpo([Lbl || {_,Lbl} <- L1], Blocks), + case member(Fail, Ls) of + true -> + %% One or more of labels in the first list + %% could reach the failure label. That + %% means that the second switch/br instruction + %% will be retained, increasing code size and + %% potentially also execution time. + none; + false -> + %% The combined switch will replace both original + %% br/switch instructions, leading to a reduction in code + %% size and potentially also in execution time. + combine_lists_1(L1, L2) + end. + +combine_lists_1(List0, List1) -> + case are_lists_compatible(List0, List1) of + true -> + First = maps:from_list(List0), + List0 ++ [{Val,Lbl} || {Val,Lbl} <- List1, + not is_map_key(Val, First)]; + false -> + none + end. + +are_lists_compatible([{#b_literal{val=Val1},_}|_], + [{#b_literal{val=Val2},_}|_]) -> + case lit_type(Val1) of + none -> false; + Type -> Type =:= lit_type(Val2) + end. + +lit_type(Val) -> + if + is_atom(Val) -> atom; + is_float(Val) -> float; + is_integer(Val) -> integer; + true -> none + end. + +%%% +%%% Calculate used variables for each block. +%%% + +used_vars(Linear) -> + used_vars(reverse(Linear), #{}, #{}). + +used_vars([{L,#b_blk{is=Is}=Blk}|Bs], UsedVars0, Skip0) -> + %% Calculate the variables used by each block and its + %% successors. This information is used by + %% shortcut_opt/1. + + Successors = beam_ssa:successors(Blk), + Used0 = used_vars_succ(Successors, L, UsedVars0, []), + Used = used_vars_blk(Blk, Used0), + UsedVars = used_vars_phis(Is, L, Used, UsedVars0), + + %% combine_eqs/1 needs different variable usage information than + %% shortcut_opt/1. The Skip map will have an entry for each block + %% that can be skipped (does not bind any variable used in + %% successor). This information is also useful for speeding up + %% shortcut_opt/1. + + Defined0 = [Def || #b_set{dst=Def} <- Is], + Defined = ordsets:from_list(Defined0), + MaySkip = ordsets:is_disjoint(Defined, Used0), + case MaySkip of + true -> + Skip = Skip0#{L=>true}, + used_vars(Bs, UsedVars, Skip); + false -> + used_vars(Bs, UsedVars, Skip0) + end; +used_vars([], UsedVars, Skip) -> + {UsedVars,Skip}. + +used_vars_succ([S|Ss], L, LiveMap, Live0) -> + Key = {S,L}, + case LiveMap of + #{Key:=Live} -> + %% The successor has a phi node, and the value for + %% this block in the phi node is a variable. + used_vars_succ(Ss, L, LiveMap, ordsets:union(Live, Live0)); + #{S:=Live} -> + %% No phi node in the successor, or the value for + %% this block in the phi node is a literal. + used_vars_succ(Ss, L, LiveMap, ordsets:union(Live, Live0)); + #{} -> + %% A peek_message block which has not been processed yet. + used_vars_succ(Ss, L, LiveMap, Live0) + end; +used_vars_succ([], _, _, Acc) -> Acc. + +used_vars_phis(Is, L, Live0, UsedVars0) -> + UsedVars = UsedVars0#{L=>Live0}, + Phis = takewhile(fun(#b_set{op=Op}) -> Op =:= phi end, Is), + case Phis of + [] -> + UsedVars; + [_|_] -> + PhiArgs = append([Args || #b_set{args=Args} <- Phis]), + case [{P,V} || {#b_var{}=V,P} <- PhiArgs] of + [_|_]=PhiVars -> + PhiLive0 = rel2fam(PhiVars), + PhiLive = [{{L,P},ordsets:union(ordsets:from_list(Vs), Live0)} || + {P,Vs} <- PhiLive0], + maps:merge(UsedVars, maps:from_list(PhiLive)); + [] -> + %% There were only literals in the phi node(s). + UsedVars + end + end. + +used_vars_blk(#b_blk{is=Is,last=Last}, Used0) -> + Used = ordsets:union(Used0, beam_ssa:used(Last)), + used_vars_is(reverse(Is), Used). + +used_vars_is([#b_set{op=phi}|Is], Used) -> + used_vars_is(Is, Used); +used_vars_is([#b_set{dst=Dst}=I|Is], Used0) -> + Used1 = ordsets:union(Used0, beam_ssa:used(I)), + Used = ordsets:del_element(Dst, Used1), + used_vars_is(Is, Used); +used_vars_is([], Used) -> + Used. + +%%% +%%% Common utilities. +%%% + +sub(#b_set{args=Args}=I, Sub) -> + I#b_set{args=[sub_arg(A, Sub) || A <- Args]}. + +sub_arg(#b_var{}=Old, Sub) -> + case Sub of + #{Old:=New} -> New; + #{} -> Old + end; +sub_arg(Old, _Sub) -> Old. + +rel2fam(S0) -> + S1 = sofs:relation(S0), + S = sofs:rel2fam(S1), + sofs:to_external(S). diff --git a/lib/compiler/src/beam_ssa_funs.erl b/lib/compiler/src/beam_ssa_funs.erl new file mode 100644 index 0000000000..e77c00fa89 --- /dev/null +++ b/lib/compiler/src/beam_ssa_funs.erl @@ -0,0 +1,149 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%%% +%%% If a fun is defined locally and only used for calls, it can be replaced +%%% with direct calls to the relevant function. This greatly speeds up "named +%%% functions" (which rely on make_fun to recreate themselves) and macros that +%%% wrap their body in a fun. +%%% + +-module(beam_ssa_funs). + +-export([module/2]). + +-include("beam_ssa.hrl"). + +-import(lists, [foldl/3]). + +-spec module(Module, Options) -> Result when + Module :: beam_ssa:b_module(), + Options :: [compile:option()], + Result :: {ok, beam_ssa:b_module()}. + +module(#b_module{body=Fs0}=Module, _Opts) -> + Trampolines = foldl(fun find_trampolines/2, #{}, Fs0), + Fs = [lfo(F, Trampolines) || F <- Fs0], + {ok, Module#b_module{body=Fs}}. + +%% If a function does absolutely nothing beyond calling another function with +%% the same arguments in the same order, we can shave off a call by short- +%% circuiting it. +find_trampolines(#b_function{args=Args,bs=Blocks}=F, Trampolines) -> + case map_get(0, Blocks) of + #b_blk{is=[#b_set{op=call, + args=[#b_local{}=Actual | Args], + dst=Dst}], + last=#b_ret{arg=Dst}} -> + {_, Name, Arity} = beam_ssa:get_anno(func_info, F), + Trampoline = #b_local{name=#b_literal{val=Name},arity=Arity}, + Trampolines#{Trampoline => Actual}; + _ -> + Trampolines + end. + +lfo(#b_function{bs=Blocks0}=F, Trampolines) -> + Linear0 = beam_ssa:linearize(Blocks0), + Linear = lfo_optimize(Linear0, lfo_analyze(Linear0, #{}), Trampolines), + F#b_function{bs=maps:from_list(Linear)}. + +%% Gather a map of the locally defined funs that are only used for calls. +lfo_analyze([{_L,#b_blk{is=Is,last=Last}}|Bs], LFuns0) -> + LFuns = lfo_analyze_last(Last, lfo_analyze_is(Is, LFuns0)), + lfo_analyze(Bs, LFuns); +lfo_analyze([], LFuns) -> + LFuns. + +lfo_analyze_is([#b_set{op=make_fun, + dst=Dst, + args=[#b_local{} | FreeVars]}=Def | Is], + LFuns0) -> + LFuns = maps:put(Dst, Def, maps:without(FreeVars, LFuns0)), + lfo_analyze_is(Is, LFuns); +lfo_analyze_is([#b_set{op=call, + args=[Fun | CallArgs]} | Is], + LFuns) when is_map_key(Fun, LFuns) -> + #b_set{args=[#b_local{arity=Arity} | FreeVars]} = map_get(Fun, LFuns), + case length(CallArgs) + length(FreeVars) of + Arity -> + lfo_analyze_is(Is, maps:without(CallArgs, LFuns)); + _ -> + %% This will `badarity` at runtime, and it's easier to disable the + %% optimization than to simulate it. + lfo_analyze_is(Is, maps:without([Fun | CallArgs], LFuns)) + end; +lfo_analyze_is([#b_set{args=Args} | Is], LFuns) when map_size(LFuns) =/= 0 -> + %% We disqualify funs that are used outside calls because this forces them + %% to be created anyway, and the slight performance gain from direct calls + %% is not enough to offset the potential increase in stack frame size (the + %% free variables need to be kept alive until the call). + %% + %% This is also a kludge to make HiPE work, as the latter will generate + %% code with the assumption that the functions referenced in a make_fun + %% will only be used by funs, which will not be the case if we mix it with + %% direct calls. See cerl_cconv.erl for details. + %% + %% Future optimizations like delaying fun creation until use may require us + %% to copy affected functions so that HiPE gets its own to play with (until + %% HiPE is fixed anyway). + lfo_analyze_is(Is, maps:without(Args, LFuns)); +lfo_analyze_is([_ | Is], LFuns) -> + lfo_analyze_is(Is, LFuns); +lfo_analyze_is([], LFuns) -> + LFuns. + +lfo_analyze_last(#b_switch{arg=Arg}, LFuns) -> + maps:remove(Arg, LFuns); +lfo_analyze_last(#b_ret{arg=Arg}, LFuns) -> + maps:remove(Arg, LFuns); +lfo_analyze_last(_, LFuns) -> + LFuns. + +%% Replace all calls of suitable funs with a direct call to their +%% implementation. Liveness optimization will get rid of the make_fun +%% instruction. +lfo_optimize(Linear, LFuns, _Trampolines) when map_size(LFuns) =:= 0 -> + Linear; +lfo_optimize(Linear, LFuns, Trampolines) -> + lfo_optimize_1(Linear, LFuns, Trampolines). + +lfo_optimize_1([{L,#b_blk{is=Is0}=Blk}|Bs], LFuns, Trampolines) -> + Is = lfo_optimize_is(Is0, LFuns, Trampolines), + [{L,Blk#b_blk{is=Is}} | lfo_optimize_1(Bs, LFuns, Trampolines)]; +lfo_optimize_1([], _LFuns, _Trampolines) -> + []. + +lfo_optimize_is([#b_set{op=call, + args=[Fun | CallArgs]}=Call0 | Is], + LFuns, Trampolines) when is_map_key(Fun, LFuns) -> + #b_set{args=[Local | FreeVars]} = map_get(Fun, LFuns), + Args = [lfo_short_circuit(Local, Trampolines) | CallArgs ++ FreeVars], + Call = beam_ssa:add_anno(local_fun_opt, Fun, Call0#b_set{args=Args}), + [Call | lfo_optimize_is(Is, LFuns, Trampolines)]; +lfo_optimize_is([I | Is], LFuns, Trampolines) -> + [I | lfo_optimize_is(Is, LFuns, Trampolines)]; +lfo_optimize_is([], _LFuns, _Trampolines) -> + []. + +lfo_short_circuit(Call, Trampolines) -> + case maps:find(Call, Trampolines) of + {ok, Other} -> lfo_short_circuit(Other, Trampolines); + error -> Call + end. diff --git a/lib/compiler/src/beam_ssa_lint.erl b/lib/compiler/src/beam_ssa_lint.erl new file mode 100644 index 0000000000..a003607dab --- /dev/null +++ b/lib/compiler/src/beam_ssa_lint.erl @@ -0,0 +1,349 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% Purpose: Internal consistency checks for the beam_ssa format. + +-module(beam_ssa_lint). + +-export([module/2, format_error/1]). + +-import(lists, [append/1, foldl/3, foreach/2]). + +-include("beam_ssa.hrl"). + +-spec module(#b_module{}, [compile:option()]) -> + {'ok',#b_module{}} | {'error',list()}. +module(#b_module{body=Fs,name=Name}=Mod0, _Options) -> + Es0 = append([validate_function(F) || F <- Fs]), + case [{?MODULE,E} || E <- Es0] of + [] -> + {ok, Mod0}; + [_|_]=Es -> + {error,[{atom_to_list(Name), Es}]} + end. + +-spec format_error(term()) -> iolist(). +format_error({{_M,F,A},{redefined_variable, Name, Old, I}}) -> + io_lib:format("~p/~p: Variable ~ts (~ts) redefined by ~ts", + [F, A, format_var(Name), format_instr(Old), format_instr(I)]); +format_error({{_M,F,A},{missing_phi_paths, Paths, I}}) -> + io_lib:format("~p/~p: Phi node ~ts doesn't define a value for these " + "branches: ~w", + [F, A, format_instr(I), Paths]); +format_error({{_M,F,A},{garbage_phi_paths, Paths, I}}) -> + io_lib:format("~p/~p: Phi node ~ts defines a value for these unreachable " + "or non-existent branches: ~w", + [F, A, format_instr(I), Paths]); +format_error({{_M,F,A},{unknown_phi_variable, Name, {From, _To}, I}}) -> + io_lib:format("~p/~p: Variable ~ts used in phi node ~ts is undefined on " + "branch ~w", + [F, A, format_var(Name), format_instr(I), From]); +format_error({{_M,F,A},{unknown_block, Label, I}}) -> + io_lib:format("~p/~p: Unknown block ~p referenced in ~ts", + [F, A, Label, I]); +format_error({{_M,F,A},{unknown_variable, Name, I}}) -> + io_lib:format("~p/~p: Unbound variable ~ts used in ~ts", + [F, A, format_var(Name), format_instr(I)]); +format_error({{_M,F,A},{phi_inside_block, Name, Id}}) -> + io_lib:format("~p/~p: Phi node defining ~ts is not at start of block ~p", + [F, A, format_var(Name), Id]); +format_error({{_M,F,A},{undefined_label_in_phi, Label, I}}) -> + io_lib:format("~p/~p: Unknown block label ~p in phi node ~ts", + [F, A, Label, format_instr(I)]). + +format_instr(I) -> + [$',beam_ssa_pp:format_instr(I),$']. + +format_var(V) -> + beam_ssa_pp:format_var(#b_var{name=V}). + +validate_function(F) -> + try + validate_variables(F), + [] + catch + throw:Reason -> + #{func_info:=MFA} = F#b_function.anno, + [{MFA,Reason}]; + Class:Error:Stack -> + io:fwrite("Function: ~p\n", [F#b_function.anno]), + erlang:raise(Class, Error, Stack) + end. + +-type defined_vars() :: gb_sets:set(beam_ssa:var_name()). + +-record(vvars, + {blocks :: #{ beam_ssa:label() => beam_ssa:b_blk() }, + branch_def_vars :: #{ + %% Describes the variable state at the time of this exact branch (phi + %% node validation). + {From :: beam_ssa:label(), To :: beam_ssa:label()} => defined_vars(), + %% Describes the variable state common to all branches leading to this + %% label (un/redefined variable validation). + beam_ssa:label() => defined_vars() }, + defined_vars :: defined_vars()}). + +-spec validate_variables(beam_ssa:b_function()) -> ok. +validate_variables(#b_function{ args = Args, bs = Blocks }) -> + %% Prefill the mapping with function arguments. + ArgNames = vvars_get_varnames(Args), + DefVars = gb_sets:from_list(ArgNames), + Entry = 0, + + State = #vvars{blocks = Blocks, + branch_def_vars = #{ Entry => DefVars }, + defined_vars = DefVars}, + ok = vvars_assert_unique(Blocks, ArgNames), + vvars_phi_nodes(vvars_block(Entry, State)). + +%% Checks the uniqueness of all variables across all blocks. +-spec vvars_assert_unique(Blocks, [beam_ssa:var_name()]) -> ok when + Blocks :: #{ beam_ssa:label() => beam_ssa:b_blk() }. +vvars_assert_unique(Blocks, Args) -> + BlockIs = [Is || #b_blk{is=Is} <- maps:values(Blocks)], + Defined0 = maps:from_list([{V,argument} || V <- Args]), + _ = foldl(fun(Is, Defined) -> + vvars_assert_unique_1(Is, Defined) + end, Defined0, BlockIs), + ok. + +-spec vvars_assert_unique_1(Is, Defined) -> ok when + Is :: list(beam_ssa:b_set()), + Defined :: #{ beam_ssa:var_name() => beam_ssa:b_set() }. +vvars_assert_unique_1([#b_set{dst=#b_var{name=DstName}}=I|Is], Defined) -> + case Defined of + #{DstName:=Old} -> throw({redefined_variable, DstName, Old, I}); + _ -> vvars_assert_unique_1(Is, Defined#{DstName=>I}) + end; +vvars_assert_unique_1([], Defined) -> + Defined. + +-spec vvars_phi_nodes(State :: #vvars{}) -> ok. +vvars_phi_nodes(#vvars{ blocks = Blocks }=State) -> + _ = [vvars_phi_nodes_1(Is, Id, State) || + {Id, #b_blk{ is = Is }} <- maps:to_list(Blocks)], + ok. + +-spec vvars_phi_nodes_1(Is, Id, State) -> ok when + Is :: list(beam_ssa:b_set()), + Id :: beam_ssa:label(), + State :: #vvars{}. +vvars_phi_nodes_1([#b_set{ op = phi, args = Phis }=I | Is], Id, State) -> + ok = vvars_assert_phi_paths(Phis, I, Id, State), + ok = vvars_assert_phi_vars(Phis, I, Id, State), + vvars_phi_nodes_1(Is, Id, State); +vvars_phi_nodes_1([_ | Is], Id, _State) -> + case [Dst || #b_set{op=phi,dst=#b_var{name=Dst}} <- Is] of + [Name|_] -> + throw({phi_inside_block, Name, Id}); + [] -> + ok + end; +vvars_phi_nodes_1([], _Id, _State) -> + ok. + +%% Checks whether all paths leading to this phi node are represented, and that +%% it doesn't reference any non-existent paths. +-spec vvars_assert_phi_paths(Phis, I, Id, State) -> ok when + Phis :: list({beam_ssa:argument(), beam_ssa:label()}), + Id :: beam_ssa:label(), + I :: beam_ssa:b_set(), + State :: #vvars{}. +vvars_assert_phi_paths(Phis, I, Id, State) -> + BranchKeys = maps:keys(State#vvars.branch_def_vars), + RequiredPaths = ordsets:from_list([From || {From, To} <- BranchKeys, To =:= Id]), + ProvidedPaths = ordsets:from_list([From || {_Value, From} <- Phis]), + case ordsets:subtract(RequiredPaths, ProvidedPaths) of + [_|_]=MissingPaths -> throw({missing_phi_paths, MissingPaths, I}); + [] -> ok + end. + %% %% The following test is sometimes useful to find missing optimizations. + %% %% It is commented out, though, because it can be triggered by + %% %% by weird but legal code. + %% case ordsets:subtract(ProvidedPaths, RequiredPaths) of + %% [_|_]=GarbagePaths -> throw({garbage_phi_paths, GarbagePaths, I}); + %% [] -> ok + %% end. + +%% Checks whether all variables used in this phi node are defined in the branch +%% they arrived on. +-spec vvars_assert_phi_vars(Phis, I, Id, State) -> ok when + Phis :: list({beam_ssa:argument(), beam_ssa:label()}), + Id :: beam_ssa:label(), + I :: beam_ssa:b_set(), + State :: #vvars{}. +vvars_assert_phi_vars(Phis, I, Id, #vvars{blocks=Blocks, + branch_def_vars=BranchDefVars}) -> + Vars = [{Var, From} || {#b_var{}=Var, From} <- Phis], + foreach(fun({#b_var{name=VarName}, From}) -> + BranchKey = {From, Id}, + case BranchDefVars of + #{BranchKey:=DefVars} -> + case gb_sets:is_member(VarName, DefVars) of + true -> ok; + false -> throw({unknown_variable, VarName, I}) + end; + #{} -> + throw({unknown_phi_variable, VarName, BranchKey, I}) + end + end, Vars), + Labels = [From || {#b_literal{},From} <- Phis], + foreach(fun(Label) -> + case Blocks of + #{Label:=_} -> + ok; + #{} -> + throw({undefined_label_in_phi, Label, I}) + end + end, Labels). + +-spec vvars_block(Id, State) -> #vvars{} when + Id :: beam_ssa:label(), + State :: #vvars{}. +vvars_block(Id, State0) -> + #{ Id := #b_blk{ is = Is, last = Terminator} } = State0#vvars.blocks, + #{ Id := DefVars } = State0#vvars.branch_def_vars, + State = State0#vvars{ defined_vars = DefVars }, + vvars_terminator(Terminator, Id, vvars_block_1(Is, State)). + +-spec vvars_block_1(Blocks, State) -> #vvars{} when + Blocks :: list(beam_ssa:b_blk()), + State :: #vvars{}. +vvars_block_1([], State) -> + State; +vvars_block_1([#b_set{ dst = #b_var{ name = DstName }, op = phi } | Is], State0) -> + %% We don't check phi node arguments at this point since we may not have + %% visited their definition yet. They'll be handled later on in + %% vvars_phi_nodes/1 after all blocks are processed. + vvars_block_1(Is, vvars_save_var(DstName, State0)); +vvars_block_1([#b_set{ dst = #b_var{ name = DstName }, args = Args }=I | Is], State0) -> + ok = vvars_assert_args(Args, I, State0), + vvars_block_1(Is, vvars_save_var(DstName, State0)). + +-spec vvars_terminator(Terminator, From, State) -> #vvars{} when + Terminator :: beam_ssa:terminator(), + From :: beam_ssa:label(), + State :: #vvars{}. +vvars_terminator(#b_ret{ arg = Arg }=I, _From, State) -> + ok = vvars_assert_args([Arg], I, State), + State; +vvars_terminator(#b_switch{arg=Arg,fail=Fail,list=Switch}=I, From, State) -> + ok = vvars_assert_args([Arg], I, State), + ok = vvars_assert_args([A || {A,_Lbl} <- Switch], I, State), + Labels = [Fail | [Lbl || {_Arg, Lbl} <- Switch]], + ok = vvars_assert_labels(Labels, I, State), + vvars_terminator_1(Labels, From, State); +vvars_terminator(#b_br{bool=#b_literal{val=true},succ=Succ}=I, From, State) -> + Labels = [Succ], + ok = vvars_assert_labels(Labels, I, State), + vvars_terminator_1(Labels, From, State); +vvars_terminator(#b_br{bool=#b_literal{val=false},fail=Fail}=I, From, State) -> + Labels = [Fail], + ok = vvars_assert_labels(Labels, I, State), + vvars_terminator_1(Labels, From, State); +vvars_terminator(#b_br{ bool = Arg, succ = Succ, fail = Fail }=I, From, State) -> + ok = vvars_assert_args([Arg], I, State), + Labels = [Fail, Succ], + ok = vvars_assert_labels(Labels, I, State), + vvars_terminator_1(Labels, From, State). + +-spec vvars_terminator_1(Labels, From, State) -> #vvars{} when + Labels :: list(beam_ssa:label()), + From :: beam_ssa:label(), + State :: #vvars{}. +vvars_terminator_1(Labels0, From, State0) -> + %% Filter out all branches that have already been taken. This should result + %% in either all of Labels0 or an empty list. + Labels = [To || To <- Labels0, + not maps:is_key({From, To}, State0#vvars.branch_def_vars)], + true = Labels =:= Labels0 orelse Labels =:= [], %Assertion + State1 = foldl(fun(To, State) -> + vvars_save_branch(From, To, State) + end, State0, Labels), + foldl(fun(To, State) -> + vvars_block(To, State) + end, State1, Labels). + +%% Gets all variable names in args, ignoring literals etc +-spec vvars_get_varnames(Args) -> list(beam_ssa:var_name()) when + Args :: list(beam_ssa:argument()). +vvars_get_varnames(Args) -> + [Name || #b_var{ name = Name } <- Args]. + +%% Checks that all variables in Args are defined in all paths leading to the +%% current State. +-spec vvars_assert_args(Args, I, State) -> ok when + Args :: list(beam_ssa:argument()), + I :: beam_ssa:terminator() | beam_ssa:b_set(), + State :: #vvars{}. +vvars_assert_args(Args, I, #vvars{defined_vars=DefVars}=State) -> + foreach(fun(#b_remote{mod=Mod,name=Name}) -> + vvars_assert_args([Mod,Name], I, State); + (#b_var{name=Name}) -> + case gb_sets:is_member(Name, DefVars) of + true -> ok; + false -> throw({unknown_variable,Name,I}) + end; + (_) -> ok + end, Args). + +%% Checks that all given labels are defined in State. +-spec vvars_assert_labels(Labels, I, State) -> ok when + Labels :: list(beam_ssa:label()), + I :: beam_ssa:terminator(), + State :: #vvars{}. +vvars_assert_labels(Labels, I, #vvars{blocks=Blocks}) -> + foreach(fun(Label) -> + case maps:is_key(Label, Blocks) of + false -> throw({unknown_block, Label, I}); + true -> ok + end + end, Labels). + +-spec vvars_save_branch(From, To, State) -> #vvars{} when + From :: beam_ssa:label(), + To :: beam_ssa:label(), + State :: #vvars{}. +vvars_save_branch(From, To, State) -> + DefVars = State#vvars.defined_vars, + Branches0 = State#vvars.branch_def_vars, + case Branches0 of + #{ To := LblDefVars } -> + MergedVars = vvars_merge_branches(DefVars, LblDefVars), + + Branches = Branches0#{ To => MergedVars, {From, To} => DefVars }, + State#vvars { branch_def_vars = Branches }; + _ -> + Branches = Branches0#{ To => DefVars, {From, To} => DefVars }, + State#vvars { branch_def_vars = Branches } + end. + +-spec vvars_merge_branches(New, Existing) -> defined_vars() when + New :: defined_vars(), + Existing :: defined_vars(). +vvars_merge_branches(New, Existing) -> + gb_sets:intersection(New, Existing). + +-spec vvars_save_var(VarName, State) -> #vvars{} when + VarName :: beam_ssa:var_name(), + State :: #vvars{}. +vvars_save_var(VarName, State0) -> + %% vvars_assert_unique guarantees that variables are never set twice. + DefVars = gb_sets:insert(VarName, State0#vvars.defined_vars), + State0#vvars{ defined_vars = DefVars }. diff --git a/lib/compiler/src/beam_ssa_opt.erl b/lib/compiler/src/beam_ssa_opt.erl new file mode 100644 index 0000000000..90c0d3cf16 --- /dev/null +++ b/lib/compiler/src/beam_ssa_opt.erl @@ -0,0 +1,2279 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%%% +%%% This is a collection of various optimizations that don't need a separate +%%% pass by themselves and/or are mutually beneficial to other passes. +%%% +%%% The optimizations are applied in "phases," each with a list of sub-passes +%%% to run. These sub-passes are applied on all functions in a module before +%%% moving on to the next phase, which lets us gather module-level information +%%% in one phase and then apply it in the next without having to risk working +%%% with incomplete information. +%%% +%%% Each sub-pass operates on a #st{} record and a func_info_db(), where the +%%% former is just a #b_function{} whose blocks can be represented either in +%%% linear or map form, and the latter is a map with information about all +%%% functions in the module (see beam_ssa_opt.hrl for more details). +%%% + +-module(beam_ssa_opt). +-export([module/2]). + +-include("beam_ssa_opt.hrl"). + +-import(lists, [all/2,append/1,duplicate/2,foldl/3,keyfind/3,member/2, + reverse/1,reverse/2, + splitwith/2,sort/1,takewhile/2,unzip/1]). + +-define(DEFAULT_REPETITIONS, 2). + +-spec module(beam_ssa:b_module(), [compile:option()]) -> + {'ok',beam_ssa:b_module()}. + +-record(st, {ssa :: [{beam_ssa:label(),beam_ssa:b_blk()}] | + beam_ssa:block_map(), + args :: [beam_ssa:b_var()], + cnt :: beam_ssa:label(), + anno :: beam_ssa:anno()}). +-type st_map() :: #{ func_id() => #st{} }. + +module(Module, Opts) -> + FuncDb0 = case proplists:get_value(no_module_opt, Opts, false) of + false -> build_func_db(Module); + true -> #{} + end, + + %% Passes that perform module-level optimizations are often aided by + %% optimizing callers before callees and vice versa, so we optimize all + %% functions in call order, flipping it as required. + StMap0 = build_st_map(Module), + Order = get_call_order_po(StMap0, FuncDb0), + + Phases = + [{Order, prologue_passes(Opts)}] ++ + repeat(Opts, repeated_passes(Opts), Order) ++ + [{Order, epilogue_passes(Opts)}], + + {StMap, _FuncDb} = foldl(fun({FuncIds, Ps}, {StMap, FuncDb}) -> + phase(FuncIds, Ps, StMap, FuncDb) + end, {StMap0, FuncDb0}, Phases), + + {ok, finish(Module, StMap)}. + +phase([FuncId | Ids], Ps, StMap, FuncDb0) -> + try compile:run_sub_passes(Ps, {map_get(FuncId, StMap), FuncDb0}) of + {St, FuncDb} -> + phase(Ids, Ps, StMap#{ FuncId => St }, FuncDb) + catch + Class:Error:Stack -> + #b_local{name=#b_literal{val=Name},arity=Arity} = FuncId, + io:fwrite("Function: ~w/~w\n", [Name,Arity]), + erlang:raise(Class, Error, Stack) + end; +phase([], _Ps, StMap, FuncDb) -> + {StMap, FuncDb}. + +%% Repeats the given passes, alternating the order between runs to make the +%% type pass more efficient. +repeat(Opts, Ps, OrderA) -> + Repeat = proplists:get_value(ssa_opt_repeat, Opts, ?DEFAULT_REPETITIONS), + OrderB = reverse(OrderA), + repeat_1(Repeat, Ps, OrderA, OrderB). + +repeat_1(0, _Opts, _OrderA, _OrderB) -> + []; +repeat_1(N, Ps, OrderA, OrderB) when N > 0, N rem 2 =:= 0 -> + [{OrderA, Ps} | repeat_1(N - 1, Ps, OrderA, OrderB)]; +repeat_1(N, Ps, OrderA, OrderB) when N > 0, N rem 2 =:= 1 -> + [{OrderB, Ps} | repeat_1(N - 1, Ps, OrderA, OrderB)]. + +%% + +get_func_id(F) -> + {_Mod, Name, Arity} = beam_ssa:get_anno(func_info, F), + #b_local{name=#b_literal{val=Name}, arity=Arity}. + +-spec build_st_map(#b_module{}) -> st_map(). +build_st_map(#b_module{body=Fs}) -> + build_st_map_1(Fs, #{}). + +build_st_map_1([F | Fs], Map) -> + #b_function{anno=Anno,args=Args,cnt=Counter,bs=Bs} = F, + St = #st{anno=Anno,args=Args,cnt=Counter,ssa=Bs}, + build_st_map_1(Fs, Map#{ get_func_id(F) => St }); +build_st_map_1([], Map) -> + Map. + +-spec finish(#b_module{}, st_map()) -> #b_module{}. +finish(#b_module{body=Fs0}=Module, StMap) -> + Module#b_module{body=finish_1(Fs0, StMap)}. + +finish_1([F0 | Fs], StMap) -> + #st{anno=Anno,cnt=Counter,ssa=Blocks} = map_get(get_func_id(F0), StMap), + F = F0#b_function{anno=Anno,bs=Blocks,cnt=Counter}, + [F | finish_1(Fs, StMap)]; +finish_1([], _StMap) -> + []. + +%% + +-define(PASS(N), {N,fun N/1}). + +prologue_passes(Opts) -> + Ps = [?PASS(ssa_opt_split_blocks), + ?PASS(ssa_opt_coalesce_phis), + ?PASS(ssa_opt_tail_phis), + ?PASS(ssa_opt_element), + ?PASS(ssa_opt_linearize), + ?PASS(ssa_opt_tuple_size), + ?PASS(ssa_opt_record), + ?PASS(ssa_opt_cse), %Helps the first type pass. + ?PASS(ssa_opt_type_start)], + passes_1(Ps, Opts). + +%% These passes all benefit from each other (in roughly this order), so they +%% are repeated as required. +repeated_passes(Opts) -> + Ps = [?PASS(ssa_opt_live), + ?PASS(ssa_opt_bs_puts), + ?PASS(ssa_opt_dead), + ?PASS(ssa_opt_cse), + ?PASS(ssa_opt_tail_phis), + ?PASS(ssa_opt_type_continue)], %Must run after ssa_opt_dead to + %clean up phi nodes. + passes_1(Ps, Opts). + +epilogue_passes(Opts) -> + Ps = [?PASS(ssa_opt_type_finish), + ?PASS(ssa_opt_float), + ?PASS(ssa_opt_sw), + + %% Run live one more time to clean up after the float and sw + %% passes. + ?PASS(ssa_opt_live), + ?PASS(ssa_opt_bsm), + ?PASS(ssa_opt_bsm_units), + ?PASS(ssa_opt_bsm_shortcut), + ?PASS(ssa_opt_blockify), + ?PASS(ssa_opt_sink), + ?PASS(ssa_opt_merge_blocks), + ?PASS(ssa_opt_get_tuple_element), + ?PASS(ssa_opt_trim_unreachable)], + passes_1(Ps, Opts). + +passes_1(Ps, Opts0) -> + Negations = [{list_to_atom("no_"++atom_to_list(N)),N} || + {N,_} <- Ps], + Opts = proplists:substitute_negations(Negations, Opts0), + [case proplists:get_value(Name, Opts, true) of + true -> + P; + false -> + {NoName,Name} = keyfind(Name, 2, Negations), + {NoName,fun(S) -> S end} + end || {Name,_}=P <- Ps]. + +%% Builds a function information map with basic information about incoming and +%% outgoing local calls, as well as whether the function is exported. +-spec build_func_db(#b_module{}) -> func_info_db(). +build_func_db(#b_module{body=Fs,exports=Exports}) -> + try + fdb_1(Fs, gb_sets:from_list(Exports), #{}) + catch + %% All module-level optimizations are invalid when a NIF can override a + %% function, so we have to bail out. + throw:load_nif -> #{} + end. + +fdb_1([#b_function{ args=Args,bs=Bs }=F | Fs], Exports, FuncDb0) -> + Id = get_func_id(F), + + #b_local{name=#b_literal{val=Name}, arity=Arity} = Id, + Exported = gb_sets:is_element({Name, Arity}, Exports), + ArgTypes = duplicate(length(Args), #{}), + + FuncDb1 = case FuncDb0 of + %% We may have an entry already if someone's called us. + #{ Id := Info } -> + FuncDb0#{ Id := Info#func_info{ exported=Exported, + arg_types=ArgTypes }}; + #{} -> + FuncDb0#{ Id => #func_info{ exported=Exported, + arg_types=ArgTypes }} + end, + + FuncDb = beam_ssa:fold_rpo(fun(_L, #b_blk{is=Is}, FuncDb) -> + fdb_is(Is, Id, FuncDb) + end, FuncDb1, Bs), + + fdb_1(Fs, Exports, FuncDb); +fdb_1([], _Exports, FuncDb) -> + FuncDb. + +fdb_is([#b_set{op=call, + args=[#b_local{}=Callee | _]} | Is], + Caller, FuncDb) -> + fdb_is(Is, Caller, fdb_update(Caller, Callee, FuncDb)); +fdb_is([#b_set{op=call, + args=[#b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=load_nif}}, + _Path, _LoadInfo]} | _Is], _Caller, _FuncDb) -> + throw(load_nif); +fdb_is([_ | Is], Caller, FuncDb) -> + fdb_is(Is, Caller, FuncDb); +fdb_is([], _Caller, FuncDb) -> + FuncDb. + +fdb_update(Caller, Callee, FuncDb) -> + CallerVertex = maps:get(Caller, FuncDb, #func_info{}), + CalleeVertex = maps:get(Callee, FuncDb, #func_info{}), + + Calls = ordsets:add_element(Callee, CallerVertex#func_info.out), + CalledBy = ordsets:add_element(Caller, CalleeVertex#func_info.in), + + FuncDb#{ Caller => CallerVertex#func_info{out=Calls}, + Callee => CalleeVertex#func_info{in=CalledBy} }. + +%% Returns the post-order of all local calls in this module. That is, +%% called functions will be ordered before the functions calling them. +%% +%% Functions where module-level optimization is disabled are added last in +%% arbitrary order. + +get_call_order_po(StMap, FuncDb) -> + Order = gco_po(FuncDb), + Order ++ maps:fold(fun(K, _V, Acc) -> + case is_map_key(K, FuncDb) of + false -> [K | Acc]; + true -> Acc + end + end, [], StMap). + +gco_po(FuncDb) -> + All = sort(maps:keys(FuncDb)), + {RPO,_} = gco_rpo(All, FuncDb, cerl_sets:new(), []), + reverse(RPO). + +gco_rpo([Id|Ids], FuncDb, Seen0, Acc0) -> + case cerl_sets:is_element(Id, Seen0) of + true -> + gco_rpo(Ids, FuncDb, Seen0, Acc0); + false -> + #func_info{out=Successors} = map_get(Id, FuncDb), + Seen1 = cerl_sets:add_element(Id, Seen0), + {Acc,Seen} = gco_rpo(Successors, FuncDb, Seen1, Acc0), + gco_rpo(Ids, FuncDb, Seen, [Id|Acc]) + end; +gco_rpo([], _, Seen, Acc) -> + {Acc,Seen}. + +%%% +%%% Trivial sub passes. +%%% + +ssa_opt_dead({#st{ssa=Linear}=St, FuncDb}) -> + {St#st{ssa=beam_ssa_dead:opt(Linear)}, FuncDb}. + +ssa_opt_linearize({#st{ssa=Blocks}=St, FuncDb}) -> + {St#st{ssa=beam_ssa:linearize(Blocks)}, FuncDb}. + +ssa_opt_type_start({#st{ssa=Linear0,args=Args,anno=Anno}=St0, FuncDb0}) -> + {Linear, FuncDb} = beam_ssa_type:opt_start(Linear0, Args, Anno, FuncDb0), + {St0#st{ssa=Linear}, FuncDb}. + +ssa_opt_type_continue({#st{ssa=Linear0,args=Args,anno=Anno}=St0, FuncDb0}) -> + {Linear, FuncDb} = beam_ssa_type:opt_continue(Linear0, Args, Anno, FuncDb0), + {St0#st{ssa=Linear}, FuncDb}. + +ssa_opt_type_finish({#st{args=Args,anno=Anno0}=St0, FuncDb0}) -> + {Anno, FuncDb} = beam_ssa_type:opt_finish(Args, Anno0, FuncDb0), + {St0#st{anno=Anno}, FuncDb}. + +ssa_opt_blockify({#st{ssa=Linear}=St, FuncDb}) -> + {St#st{ssa=maps:from_list(Linear)}, FuncDb}. + +ssa_opt_trim_unreachable({#st{ssa=Blocks}=St, FuncDb}) -> + {St#st{ssa=beam_ssa:trim_unreachable(Blocks)}, FuncDb}. + +%%% +%%% Split blocks before certain instructions to enable more optimizations. +%%% +%%% Splitting before element/2 enables the optimization that swaps +%%% element/2 instructions. +%%% +%%% Splitting before call and make_fun instructions gives more opportunities +%%% for sinking get_tuple_element instructions. +%%% + +ssa_opt_split_blocks({#st{ssa=Blocks0,cnt=Count0}=St, FuncDb}) -> + P = fun(#b_set{op={bif,element}}) -> true; + (#b_set{op=call}) -> true; + (#b_set{op=make_fun}) -> true; + (_) -> false + end, + {Blocks,Count} = beam_ssa:split_blocks(P, Blocks0, Count0), + {St#st{ssa=Blocks,cnt=Count}, FuncDb}. + +%%% +%%% Coalesce phi nodes. +%%% +%%% Nested cases can led to code such as this: +%%% +%%% 10: +%%% _1 = phi {literal value1, label 8}, {Var, label 9} +%%% br 11 +%%% +%%% 11: +%%% _2 = phi {_1, label 10}, {literal false, label 3} +%%% +%%% The phi nodes can be coalesced like this: +%%% +%%% 11: +%%% _2 = phi {literal value1, label 8}, {Var, label 9}, {literal false, label 3} +%%% +%%% Coalescing can help other optimizations, and can in some cases reduce register +%%% shuffling (if the phi variables for two phi nodes happens to be allocated to +%%% different registers). +%%% + +ssa_opt_coalesce_phis({#st{ssa=Blocks0}=St, FuncDb}) -> + Ls = beam_ssa:rpo(Blocks0), + Blocks = c_phis_1(Ls, Blocks0), + {St#st{ssa=Blocks}, FuncDb}. + +c_phis_1([L|Ls], Blocks0) -> + case map_get(L, Blocks0) of + #b_blk{is=[#b_set{op=phi}|_]}=Blk -> + Blocks = c_phis_2(L, Blk, Blocks0), + c_phis_1(Ls, Blocks); + #b_blk{} -> + c_phis_1(Ls, Blocks0) + end; +c_phis_1([], Blocks) -> Blocks. + +c_phis_2(L, #b_blk{is=Is0}=Blk0, Blocks0) -> + case c_phis_args(Is0, Blocks0) of + none -> + Blocks0; + {_,_,Preds}=Info -> + Is = c_rewrite_phis(Is0, Info), + Blk = Blk0#b_blk{is=Is}, + Blocks = Blocks0#{L:=Blk}, + c_fix_branches(Preds, L, Blocks) + end. + +c_phis_args([#b_set{op=phi,args=Args0}|Is], Blocks) -> + case c_phis_args_1(Args0, Blocks) of + none -> + c_phis_args(Is, Blocks); + Res -> + Res + end; +c_phis_args(_, _Blocks) -> none. + +c_phis_args_1([{Var,Pred}|As], Blocks) -> + case c_get_pred_vars(Var, Pred, Blocks) of + none -> + c_phis_args_1(As, Blocks); + Result -> + Result + end; +c_phis_args_1([], _Blocks) -> none. + +c_get_pred_vars(Var, Pred, Blocks) -> + case map_get(Pred, Blocks) of + #b_blk{is=[#b_set{op=phi,dst=Var,args=Args}]} -> + {Var,Pred,Args}; + #b_blk{} -> + none + end. + +c_rewrite_phis([#b_set{op=phi,args=Args0}=I|Is], Info) -> + Args = c_rewrite_phi(Args0, Info), + [I#b_set{args=Args}|c_rewrite_phis(Is, Info)]; +c_rewrite_phis(Is, _Info) -> Is. + +c_rewrite_phi([{Var,Pred}|As], {Var,Pred,Values}) -> + Values ++ As; +c_rewrite_phi([{Value,Pred}|As], {_,Pred,Values}) -> + [{Value,P} || {_,P} <- Values] ++ As; +c_rewrite_phi([A|As], Info) -> + [A|c_rewrite_phi(As, Info)]; +c_rewrite_phi([], _Info) -> []. + +c_fix_branches([{_,Pred}|As], L, Blocks0) -> + #b_blk{last=Last0} = Blk0 = map_get(Pred, Blocks0), + #b_br{bool=#b_literal{val=true}} = Last0, %Assertion. + Last = Last0#b_br{bool=#b_literal{val=true},succ=L,fail=L}, + Blk = Blk0#b_blk{last=Last}, + Blocks = Blocks0#{Pred:=Blk}, + c_fix_branches(As, L, Blocks); +c_fix_branches([], _, Blocks) -> Blocks. + +%%% +%%% Eliminate phi nodes in the tail of a function. +%%% +%%% Try to eliminate short blocks that starts with a phi node +%%% and end in a return. For example: +%%% +%%% Result = phi { Res1, 4 }, { literal true, 5 } +%%% Ret = put_tuple literal ok, Result +%%% ret Ret +%%% +%%% The code in this block can be inserted at the end blocks 4 and +%%% 5. Thus, the following code can be inserted into block 4: +%%% +%%% Ret:1 = put_tuple literal ok, Res1 +%%% ret Ret:1 +%%% +%%% And the following code into block 5: +%%% +%%% Ret:2 = put_tuple literal ok, literal true +%%% ret Ret:2 +%%% +%%% Which can be further simplified to: +%%% +%%% ret literal {ok, true} +%%% +%%% This transformation may lead to more code improvements: +%%% +%%% - Stack trimming +%%% - Fewer test_heap instructions +%%% - Smaller stack frames +%%% + +ssa_opt_tail_phis({#st{ssa=SSA0,cnt=Count0}=St, FuncDb}) -> + {SSA,Count} = opt_tail_phis(SSA0, Count0), + {St#st{ssa=SSA,cnt=Count}, FuncDb}. + +opt_tail_phis(Blocks, Count) when is_map(Blocks) -> + opt_tail_phis(maps:values(Blocks), Blocks, Count); +opt_tail_phis(Linear0, Count0) when is_list(Linear0) -> + Blocks0 = maps:from_list(Linear0), + {Blocks,Count} = opt_tail_phis(Blocks0, Count0), + {beam_ssa:linearize(Blocks),Count}. + +opt_tail_phis([#b_blk{is=Is0,last=Last}|Bs], Blocks0, Count0) -> + case {Is0,Last} of + {[#b_set{op=phi,args=[_,_|_]}|_],#b_ret{arg=#b_var{}}=Ret} -> + {Phis,Is} = splitwith(fun(#b_set{op=Op}) -> Op =:= phi end, Is0), + case suitable_tail_ops(Is) of + true -> + {Blocks,Count} = opt_tail_phi(Phis, Is, Ret, + Blocks0, Count0), + opt_tail_phis(Bs, Blocks, Count); + false -> + opt_tail_phis(Bs, Blocks0, Count0) + end; + {_,_} -> + opt_tail_phis(Bs, Blocks0, Count0) + end; +opt_tail_phis([], Blocks, Count) -> + {Blocks,Count}. + +opt_tail_phi(Phis0, Is, Ret, Blocks0, Count0) -> + Phis = rel2fam(reduce_phis(Phis0)), + {Blocks,Count,Cost} = + foldl(fun(PhiArg, Acc) -> + opt_tail_phi_arg(PhiArg, Is, Ret, Acc) + end, {Blocks0,Count0,0}, Phis), + MaxCost = length(Phis) * 3 + 2, + if + Cost =< MaxCost -> + %% The transformation would cause at most a slight + %% increase in code size if no more optimizations + %% can be applied. + {Blocks,Count}; + true -> + %% The code size would be increased too much. + {Blocks0,Count0} + end. + +reduce_phis([#b_set{dst=PhiDst,args=PhiArgs}|Is]) -> + [{L,{PhiDst,Val}} || {Val,L} <- PhiArgs] ++ reduce_phis(Is); +reduce_phis([]) -> []. + +opt_tail_phi_arg({PredL,Sub0}, Is0, Ret0, {Blocks0,Count0,Cost0}) -> + Blk0 = map_get(PredL, Blocks0), + #b_blk{is=IsPrefix,last=#b_br{succ=Next,fail=Next}} = Blk0, + case is_exit_bif(IsPrefix) of + false -> + Sub1 = maps:from_list(Sub0), + {Is1,Count,Sub} = new_names(Is0, Sub1, Count0, []), + Is2 = [sub(I, Sub) || I <- Is1], + Cost = build_cost(Is2, Cost0), + Is = IsPrefix ++ Is2, + Ret = sub(Ret0, Sub), + Blk = Blk0#b_blk{is=Is,last=Ret}, + Blocks = Blocks0#{PredL:=Blk}, + {Blocks,Count,Cost}; + true -> + %% The block ends in a call to a function that + %% will cause an exception. + {Blocks0,Count0,Cost0+3} + end. + +is_exit_bif([#b_set{op=call, + args=[#b_remote{mod=#b_literal{val=Mod}, + name=#b_literal{val=Name}}|Args]}]) -> + erl_bifs:is_exit_bif(Mod, Name, length(Args)); +is_exit_bif(_) -> false. + +new_names([#b_set{dst=Dst}=I|Is], Sub0, Count0, Acc) -> + {NewDst,Count} = new_var(Dst, Count0), + Sub = Sub0#{Dst=>NewDst}, + new_names(Is, Sub, Count, [I#b_set{dst=NewDst}|Acc]); +new_names([], Sub, Count, Acc) -> + {reverse(Acc),Count,Sub}. + +suitable_tail_ops(Is) -> + all(fun(#b_set{op=Op}) -> + is_suitable_tail_op(Op) + end, Is). + +is_suitable_tail_op({bif,_}) -> true; +is_suitable_tail_op(put_list) -> true; +is_suitable_tail_op(put_tuple) -> true; +is_suitable_tail_op(_) -> false. + +build_cost([#b_set{op=put_list,args=Args}|Is], Cost) -> + case are_all_literals(Args) of + true -> + build_cost(Is, Cost); + false -> + build_cost(Is, Cost + 1) + end; +build_cost([#b_set{op=put_tuple,args=Args}|Is], Cost) -> + case are_all_literals(Args) of + true -> + build_cost(Is, Cost); + false -> + build_cost(Is, Cost + length(Args) + 1) + end; +build_cost([#b_set{op={bif,_},args=Args}|Is], Cost) -> + case are_all_literals(Args) of + true -> + build_cost(Is, Cost); + false -> + build_cost(Is, Cost + 1) + end; +build_cost([], Cost) -> Cost. + +are_all_literals(Args) -> + all(fun(#b_literal{}) -> true; + (_) -> false + end, Args). + +%%% +%%% Order element/2 calls. +%%% +%%% Order an unbroken chain of element/2 calls for the same tuple +%%% with the same failure label so that the highest element is +%%% retrieved first. That will allow the other element/2 calls to +%%% be replaced with get_tuple_element/3 instructions. +%%% + +ssa_opt_element({#st{ssa=Blocks}=St, FuncDb}) -> + %% Collect the information about element instructions in this + %% function. + GetEls = collect_element_calls(beam_ssa:linearize(Blocks)), + + %% Collect the element instructions into chains. The + %% element calls in each chain are ordered in reverse + %% execution order. + Chains = collect_chains(GetEls, []), + + %% For each chain, swap the first element call with the + %% element call with the highest index. + {St#st{ssa=swap_element_calls(Chains, Blocks)}, FuncDb}. + +collect_element_calls([{L,#b_blk{is=Is0,last=Last}}|Bs]) -> + case {Is0,Last} of + {[#b_set{op={bif,element},dst=Element, + args=[#b_literal{val=N},#b_var{}=Tuple]}, + #b_set{op=succeeded,dst=Bool,args=[Element]}], + #b_br{bool=Bool,succ=Succ,fail=Fail}} -> + Info = {L,Succ,{Tuple,Fail},N}, + [Info|collect_element_calls(Bs)]; + {_,_} -> + collect_element_calls(Bs) + end; +collect_element_calls([]) -> []. + +collect_chains([{This,_,V,_}=El|Els], [{_,This,V,_}|_]=Chain) -> + %% Add to the previous chain. + collect_chains(Els, [El|Chain]); +collect_chains([El|Els], [_,_|_]=Chain) -> + %% Save the previous chain and start a new chain. + [Chain|collect_chains(Els, [El])]; +collect_chains([El|Els], _Chain) -> + %% The previous chain is too short; discard it and start a new. + collect_chains(Els, [El]); +collect_chains([], [_,_|_]=Chain) -> + %% Save the last chain. + [Chain]; +collect_chains([], _) -> []. + +swap_element_calls([[{L,_,_,N}|_]=Chain|Chains], Blocks0) -> + Blocks = swap_element_calls_1(Chain, {N,L}, Blocks0), + swap_element_calls(Chains, Blocks); +swap_element_calls([], Blocks) -> Blocks. + +swap_element_calls_1([{L1,_,_,N1}], {N2,L2}, Blocks) when N2 > N1 -> + %% We have reached the end of the chain, and the first + %% element instrution to be executed. Its index is lower + %% than the maximum index found while traversing the chain, + %% so we will need to swap the instructions. + #{L1:=Blk1,L2:=Blk2} = Blocks, + [#b_set{dst=Dst1}=GetEl1,Succ1] = Blk1#b_blk.is, + [#b_set{dst=Dst2}=GetEl2,Succ2] = Blk2#b_blk.is, + Is1 = [GetEl2,Succ1#b_set{args=[Dst2]}], + Is2 = [GetEl1,Succ2#b_set{args=[Dst1]}], + Blocks#{L1:=Blk1#b_blk{is=Is1},L2:=Blk2#b_blk{is=Is2}}; +swap_element_calls_1([{L,_,_,N1}|Els], {N2,_}, Blocks) when N1 > N2 -> + swap_element_calls_1(Els, {N2,L}, Blocks); +swap_element_calls_1([_|Els], Highest, Blocks) -> + swap_element_calls_1(Els, Highest, Blocks); +swap_element_calls_1([], _, Blocks) -> + %% Nothing to do. The element call with highest index + %% is already the first one to be executed. + Blocks. + +%%% +%%% Record optimization. +%%% +%%% Replace tuple matching with an is_tagged_tuple instruction +%%% when applicable. +%%% + +ssa_opt_record({#st{ssa=Linear}=St, FuncDb}) -> + Blocks = maps:from_list(Linear), + {St#st{ssa=record_opt(Linear, Blocks)}, FuncDb}. + +record_opt([{L,#b_blk{is=Is0,last=Last}=Blk0}|Bs], Blocks) -> + Is = record_opt_is(Is0, Last, Blocks), + Blk = Blk0#b_blk{is=Is}, + [{L,Blk}|record_opt(Bs, Blocks)]; +record_opt([], _Blocks) -> []. + +record_opt_is([#b_set{op={bif,is_tuple},dst=Bool,args=[Tuple]}=Set], + Last, Blocks) -> + case is_tagged_tuple(Tuple, Bool, Last, Blocks) of + {yes,Size,Tag} -> + Args = [Tuple,Size,Tag], + [Set#b_set{op=is_tagged_tuple,args=Args}]; + no -> + [Set] + end; +record_opt_is([I|Is]=Is0, #b_br{bool=Bool}=Last, Blocks) -> + case is_tagged_tuple_1(Is0, Last, Blocks) of + {yes,_Fail,Tuple,Arity,Tag} -> + Args = [Tuple,Arity,Tag], + [I#b_set{op=is_tagged_tuple,dst=Bool,args=Args}]; + no -> + [I|record_opt_is(Is, Last, Blocks)] + end; +record_opt_is([I|Is], Last, Blocks) -> + [I|record_opt_is(Is, Last, Blocks)]; +record_opt_is([], _Last, _Blocks) -> []. + +is_tagged_tuple(#b_var{}=Tuple, Bool, + #b_br{bool=Bool,succ=Succ,fail=Fail}, + Blocks) -> + #b_blk{is=Is,last=Last} = map_get(Succ, Blocks), + case is_tagged_tuple_1(Is, Last, Blocks) of + {yes,Fail,Tuple,Arity,Tag} -> + {yes,Arity,Tag}; + _ -> + no + end; +is_tagged_tuple(_, _, _, _) -> no. + +is_tagged_tuple_1(Is, Last, Blocks) -> + case {Is,Last} of + {[#b_set{op={bif,tuple_size},dst=ArityVar, + args=[#b_var{}=Tuple]}, + #b_set{op={bif,'=:='}, + dst=Bool, + args=[ArityVar, #b_literal{val=ArityVal}=Arity]}], + #b_br{bool=Bool,succ=Succ,fail=Fail}} + when is_integer(ArityVal) -> + SuccBlk = map_get(Succ, Blocks), + case is_tagged_tuple_2(SuccBlk, Tuple, Fail) of + no -> + no; + {yes,Tag} -> + {yes,Fail,Tuple,Arity,Tag} + end; + _ -> + no + end. + +is_tagged_tuple_2(#b_blk{is=Is, + last=#b_br{bool=#b_var{}=Bool,fail=Fail}}, + Tuple, Fail) -> + is_tagged_tuple_3(Is, Bool, Tuple); +is_tagged_tuple_2(#b_blk{}, _, _) -> no. + +is_tagged_tuple_3([#b_set{op=get_tuple_element, + dst=TagVar, + args=[#b_var{}=Tuple,#b_literal{val=0}]}|Is], + Bool, Tuple) -> + is_tagged_tuple_4(Is, Bool, TagVar); +is_tagged_tuple_3([_|Is], Bool, Tuple) -> + is_tagged_tuple_3(Is, Bool, Tuple); +is_tagged_tuple_3([], _, _) -> no. + +is_tagged_tuple_4([#b_set{op={bif,'=:='},dst=Bool, + args=[#b_var{}=TagVar, + #b_literal{val=TagVal}=Tag]}], + Bool, TagVar) when is_atom(TagVal) -> + {yes,Tag}; +is_tagged_tuple_4([_|Is], Bool, TagVar) -> + is_tagged_tuple_4(Is, Bool, TagVar); +is_tagged_tuple_4([], _, _) -> no. + +%%% +%%% Common subexpression elimination (CSE). +%%% +%%% Eliminate repeated evaluation of identical expressions. To avoid +%%% increasing the size of the stack frame, we don't eliminate +%%% subexpressions across instructions that clobber the X registers. +%%% + +ssa_opt_cse({#st{ssa=Linear}=St, FuncDb}) -> + M = #{0=>#{}}, + {St#st{ssa=cse(Linear, #{}, M)}, FuncDb}. + +cse([{L,#b_blk{is=Is0,last=Last0}=Blk}|Bs], Sub0, M0) -> + Es0 = map_get(L, M0), + {Is1,Es,Sub} = cse_is(Is0, Es0, Sub0, []), + Last = sub(Last0, Sub), + M = cse_successors(Is1, Blk, Es, M0), + Is = reverse(Is1), + [{L,Blk#b_blk{is=Is,last=Last}}|cse(Bs, Sub, M)]; +cse([], _, _) -> []. + +cse_successors([#b_set{op=succeeded,args=[Src]},Bif|_], Blk, EsSucc, M0) -> + case cse_suitable(Bif) of + true -> + %% The previous instruction only has a valid value at the success branch. + %% We must remove the substitution for Src from the failure branch. + #b_blk{last=#b_br{succ=Succ,fail=Fail}} = Blk, + M = cse_successors_1([Succ], EsSucc, M0), + EsFail = maps:filter(fun(_, Val) -> Val =/= Src end, EsSucc), + cse_successors_1([Fail], EsFail, M); + false -> + %% There can't be any replacement for Src in EsSucc. No need for + %% any special handling. + cse_successors_1(beam_ssa:successors(Blk), EsSucc, M0) + end; +cse_successors(_Is, Blk, Es, M) -> + cse_successors_1(beam_ssa:successors(Blk), Es, M). + +cse_successors_1([L|Ls], Es0, M) -> + case M of + #{L:=Es1} when map_size(Es1) =:= 0 -> + %% The map is already empty. No need to do anything + %% since the intersection will be empty. + cse_successors_1(Ls, Es0, M); + #{L:=Es1} -> + %% Calculate the intersection of the two maps. + %% Both keys and values must match. + Es = maps:filter(fun(Key, Value) -> + case Es1 of + #{Key:=Value} -> true; + #{} -> false + end + end, Es0), + cse_successors_1(Ls, Es0, M#{L:=Es}); + #{} -> + cse_successors_1(Ls, Es0, M#{L=>Es0}) + end; +cse_successors_1([], _, M) -> M. + +cse_is([#b_set{op=succeeded,dst=Bool,args=[Src]}=I0|Is], Es, Sub0, Acc) -> + I = sub(I0, Sub0), + case I of + #b_set{args=[Src]} -> + cse_is(Is, Es, Sub0, [I|Acc]); + #b_set{} -> + %% The previous instruction has been eliminated. Eliminate the + %% 'succeeded' instruction too. + Sub = Sub0#{Bool=>#b_literal{val=true}}, + cse_is(Is, Es, Sub, Acc) + end; +cse_is([#b_set{dst=Dst}=I0|Is], Es0, Sub0, Acc) -> + I = sub(I0, Sub0), + case beam_ssa:clobbers_xregs(I) of + true -> + %% Retaining the expressions map across calls and other + %% clobbering instructions would work, but it would cause + %% the common subexpressions to be saved to Y registers, + %% which would probably increase the size of the stack + %% frame. + cse_is(Is, #{}, Sub0, [I|Acc]); + false -> + case cse_expr(I) of + none -> + %% Not suitable for CSE. + cse_is(Is, Es0, Sub0, [I|Acc]); + {ok,ExprKey} -> + case Es0 of + #{ExprKey:=Src} -> + Sub = Sub0#{Dst=>Src}, + cse_is(Is, Es0, Sub, Acc); + #{} -> + Es = Es0#{ExprKey=>Dst}, + cse_is(Is, Es, Sub0, [I|Acc]) + end + end + end; +cse_is([], Es, Sub, Acc) -> + {Acc,Es,Sub}. + +cse_expr(#b_set{op=Op,args=Args}=I) -> + case cse_suitable(I) of + true -> {ok,{Op,Args}}; + false -> none + end. + +cse_suitable(#b_set{op=get_hd}) -> true; +cse_suitable(#b_set{op=get_tl}) -> true; +cse_suitable(#b_set{op=put_list}) -> true; +cse_suitable(#b_set{op=get_tuple_element}) -> true; +cse_suitable(#b_set{op=put_tuple}) -> true; +cse_suitable(#b_set{op={bif,tuple_size}}) -> + %% Doing CSE for tuple_size/1 can prevent the + %% creation of test_arity and select_tuple_arity + %% instructions. That could decrease performance + %% and beam_validator could fail to understand + %% that tuple operations that follow are safe. + false; +cse_suitable(#b_set{anno=Anno,op={bif,Name},args=Args}) -> + %% Doing CSE for floating point operators is unsafe. + %% Doing CSE for comparison operators would prevent + %% creation of 'test' instructions. + Arity = length(Args), + not (is_map_key(float_op, Anno) orelse + erl_internal:new_type_test(Name, Arity) orelse + erl_internal:comp_op(Name, Arity) orelse + erl_internal:bool_op(Name, Arity)); +cse_suitable(#b_set{}) -> false. + +%%% +%%% Using floating point instructions. +%%% +%%% Use the special floating points version of arithmetic +%%% instructions, if the operands are known to be floats or the result +%%% of the operation will be a float. +%%% +%%% The float instructions were never used in guards before, so we +%%% will take special care to keep not using them in guards. Using +%%% them in guards would require a new version of the 'fconv' +%%% instruction that would take a failure label. Since it is unlikely +%%% that using float instructions in guards would be benefical, why +%%% bother implementing a new instruction? Also, implementing float +%%% instructions in guards in HiPE could turn out to be a lot of work. +%%% + +-record(fs, + {s=undefined :: 'undefined' | 'cleared', + regs=#{} :: #{beam_ssa:b_var():=beam_ssa:b_var()}, + fail=none :: 'none' | beam_ssa:label(), + non_guards :: gb_sets:set(beam_ssa:label()), + bs :: beam_ssa:block_map() + }). + +ssa_opt_float({#st{ssa=Linear0,cnt=Count0}=St, FuncDb}) -> + NonGuards0 = float_non_guards(Linear0), + NonGuards = gb_sets:from_list(NonGuards0), + Blocks = maps:from_list(Linear0), + Fs = #fs{non_guards=NonGuards,bs=Blocks}, + {Linear,Count} = float_opt(Linear0, Count0, Fs), + {St#st{ssa=Linear,cnt=Count}, FuncDb}. + +float_blk_is_in_guard(#b_blk{last=#b_br{fail=F}}, #fs{non_guards=NonGuards}) -> + not gb_sets:is_member(F, NonGuards); +float_blk_is_in_guard(#b_blk{}, #fs{}) -> + false. + +float_non_guards([{L,#b_blk{is=Is}}|Bs]) -> + case Is of + [#b_set{op=landingpad}|_] -> + [L|float_non_guards(Bs)]; + _ -> + float_non_guards(Bs) + end; +float_non_guards([]) -> [?BADARG_BLOCK]. + +float_opt([{L,Blk}|Bs0], Count0, Fs) -> + case float_blk_is_in_guard(Blk, Fs) of + true -> + %% This block is inside a guard. Don't do + %% any floating point optimizations. + {Bs,Count} = float_opt(Bs0, Count0, Fs), + {[{L,Blk}|Bs],Count}; + false -> + %% This block is not inside a guard. + %% We can do the optimization. + float_opt_1(L, Blk, Bs0, Count0, Fs) + end; +float_opt([], Count, _Fs) -> + {[],Count}. + +float_opt_1(L, #b_blk{is=Is0}=Blk0, Bs0, Count0, Fs0) -> + case float_opt_is(Is0, Fs0, Count0, []) of + {Is1,Fs1,Count1} -> + Fs2 = float_fail_label(Blk0, Fs1), + Fail = Fs2#fs.fail, + {Flush,Blk,Fs,Count2} = float_maybe_flush(Blk0, Fs2, Count1), + Split = float_split_conv(Is1, Blk), + {Blks0,Count3} = float_number(Split, L, Count2), + {Blks,Count4} = float_conv(Blks0, Fail, Count3), + {Bs,Count} = float_opt(Bs0, Count4, Fs), + {Blks++Flush++Bs,Count}; + none -> + {Bs,Count} = float_opt(Bs0, Count0, Fs0), + {[{L,Blk0}|Bs],Count} + end. + +%% Split {float,convert} instructions into individual blocks. +float_split_conv(Is0, Blk) -> + Br = #b_br{bool=#b_literal{val=true},succ=0,fail=0}, + case splitwith(fun(#b_set{op=Op}) -> + Op =/= {float,convert} + end, Is0) of + {Is,[]} -> + [Blk#b_blk{is=Is}]; + {[_|_]=Is1,[#b_set{op={float,convert}}=Conv|Is2]} -> + [#b_blk{is=Is1,last=Br}, + #b_blk{is=[Conv],last=Br}|float_split_conv(Is2, Blk)]; + {[],[#b_set{op={float,convert}}=Conv|Is1]} -> + [#b_blk{is=[Conv],last=Br}|float_split_conv(Is1, Blk)] + end. + +%% Number the blocks that were split. +float_number([B|Bs0], FirstL, Count0) -> + {Bs,Count} = float_number(Bs0, Count0), + {[{FirstL,B}|Bs],Count}. + +float_number([B|Bs0], Count0) -> + {Bs,Count} = float_number(Bs0, Count0+1), + {[{Count0,B}|Bs],Count}; +float_number([], Count) -> + {[],Count}. + +%% Insert 'succeeded' instructions after each {float,convert} +%% instruction. +float_conv([{L,#b_blk{is=Is0}=Blk0}|Bs0], Fail, Count0) -> + case Is0 of + [#b_set{op={float,convert}}=Conv] -> + {Bool0,Count1} = new_reg('@ssa_bool', Count0), + Bool = #b_var{name=Bool0}, + Succeeded = #b_set{op=succeeded,dst=Bool, + args=[Conv#b_set.dst]}, + Is = [Conv,Succeeded], + [{NextL,_}|_] = Bs0, + Br = #b_br{bool=Bool,succ=NextL,fail=Fail}, + Blk = Blk0#b_blk{is=Is,last=Br}, + {Bs,Count} = float_conv(Bs0, Fail, Count1), + {[{L,Blk}|Bs],Count}; + [_|_] -> + case Bs0 of + [{NextL,_}|_] -> + Br = #b_br{bool=#b_literal{val=true}, + succ=NextL,fail=NextL}, + Blk = Blk0#b_blk{last=Br}, + {Bs,Count} = float_conv(Bs0, Fail, Count0), + {[{L,Blk}|Bs],Count}; + [] -> + {[{L,Blk0}],Count0} + end + end. + +float_maybe_flush(Blk0, #fs{s=cleared,fail=Fail,bs=Blocks}=Fs0, Count0) -> + #b_blk{last=#b_br{bool=#b_var{},succ=Succ}=Br} = Blk0, + + %% If the success block starts with a floating point operation, we can + %% defer flushing to that block as long as it isn't a guard. + #b_blk{is=Is} = SuccBlk = map_get(Succ, Blocks), + SuccIsGuard = float_blk_is_in_guard(SuccBlk, Fs0), + + case Is of + [#b_set{anno=#{float_op:=_}}|_] when not SuccIsGuard -> + %% No flush needed. + {[],Blk0,Fs0,Count0}; + _ -> + %% Flush needed. + {Bool0,Count1} = new_reg('@ssa_bool', Count0), + Bool = #b_var{name=Bool0}, + + %% Allocate block numbers. + CheckL = Count1, %For checkerror. + FlushL = Count1 + 1, %For flushing of float regs. + Count = Count1 + 2, + Blk = Blk0#b_blk{last=Br#b_br{succ=CheckL}}, + + %% Build the block with the checkerror instruction. + CheckIs = [#b_set{op={float,checkerror},dst=Bool}], + CheckBr = #b_br{bool=Bool,succ=FlushL,fail=Fail}, + CheckBlk = #b_blk{is=CheckIs,last=CheckBr}, + + %% Build the block that flushes all registers. + FlushIs = float_flush_regs(Fs0), + FlushBr = #b_br{bool=#b_literal{val=true},succ=Succ,fail=Succ}, + FlushBlk = #b_blk{is=FlushIs,last=FlushBr}, + + %% Update state and blocks. + Fs = Fs0#fs{s=undefined,regs=#{},fail=none}, + FlushBs = [{CheckL,CheckBlk},{FlushL,FlushBlk}], + {FlushBs,Blk,Fs,Count} + end; +float_maybe_flush(Blk, Fs, Count) -> + {[],Blk,Fs,Count}. + +float_opt_is([#b_set{op=succeeded,args=[Src]}=I0], + #fs{regs=Rs}=Fs, Count, Acc) -> + case Rs of + #{Src:=Fr} -> + I = I0#b_set{args=[Fr]}, + {reverse(Acc, [I]),Fs,Count}; + #{} -> + {reverse(Acc, [I0]),Fs,Count} + end; +float_opt_is([#b_set{anno=Anno0}=I0|Is0], Fs0, Count0, Acc) -> + case Anno0 of + #{float_op:=FTypes} -> + Anno = maps:remove(float_op, Anno0), + I1 = I0#b_set{anno=Anno}, + {Is,Fs,Count} = float_make_op(I1, FTypes, Fs0, Count0), + float_opt_is(Is0, Fs, Count, reverse(Is, Acc)); + #{} -> + float_opt_is(Is0, Fs0#fs{regs=#{}}, Count0, [I0|Acc]) + end; +float_opt_is([], Fs, _Count, _Acc) -> + #fs{s=undefined} = Fs, %Assertion. + none. + +float_make_op(#b_set{op={bif,Op},dst=Dst,args=As0}=I0, + Ts, #fs{s=S,regs=Rs0}=Fs, Count0) -> + {As1,Rs1,Count1} = float_load(As0, Ts, Rs0, Count0, []), + {As,Is0} = unzip(As1), + {Fr,Count2} = new_reg('@fr', Count1), + FrDst = #b_var{name=Fr}, + I = I0#b_set{op={float,Op},dst=FrDst,args=As}, + Rs = Rs1#{Dst=>FrDst}, + Is = append(Is0) ++ [I], + case S of + undefined -> + {Ignore,Count} = new_reg('@ssa_ignore', Count2), + C = #b_set{op={float,clearerror},dst=#b_var{name=Ignore}}, + {[C|Is],Fs#fs{s=cleared,regs=Rs},Count}; + cleared -> + {Is,Fs#fs{regs=Rs},Count2} + end. + +float_load([A|As], [T|Ts], Rs0, Count0, Acc) -> + {Load,Rs,Count} = float_reg_arg(A, T, Rs0, Count0), + float_load(As, Ts, Rs, Count, [Load|Acc]); +float_load([], [], Rs, Count, Acc) -> + {reverse(Acc),Rs,Count}. + +float_reg_arg(A, T, Rs, Count0) -> + case Rs of + #{A:=Fr} -> + {{Fr,[]},Rs,Count0}; + #{} -> + {Fr,Count} = new_float_copy_reg(Count0), + Dst = #b_var{name=Fr}, + I = float_load_reg(T, A, Dst), + {{Dst,[I]},Rs#{A=>Dst},Count} + end. + +float_load_reg(convert, #b_var{}=Src, Dst) -> + #b_set{op={float,convert},dst=Dst,args=[Src]}; +float_load_reg(convert, #b_literal{val=Val}=Src, Dst) -> + try float(Val) of + F -> + #b_set{op={float,put},dst=Dst,args=[#b_literal{val=F}]} + catch + error:_ -> + %% Let the exception happen at runtime. + #b_set{op={float,convert},dst=Dst,args=[Src]} + end; +float_load_reg(float, Src, Dst) -> + #b_set{op={float,put},dst=Dst,args=[Src]}. + +new_float_copy_reg(Count) -> + new_reg('@fr_copy', Count). + +new_reg(Base, Count) -> + Fr = {Base,Count}, + {Fr,Count+1}. + +float_fail_label(#b_blk{last=Last}, Fs) -> + case Last of + #b_br{bool=#b_var{},fail=Fail} -> + Fs#fs{fail=Fail}; + _ -> + Fs + end. + +float_flush_regs(#fs{regs=Rs}) -> + maps:fold(fun(_, #b_var{name={'@fr_copy',_}}, Acc) -> + Acc; + (Dst, Fr, Acc) -> + [#b_set{op={float,get},dst=Dst,args=[Fr]}|Acc] + end, [], Rs). + +%%% +%%% Live optimization. +%%% +%%% Optimize instructions whose values are not used. They could be +%%% removed if they have no side effects, or in a few cases replaced +%%% with a cheaper instructions +%%% + +ssa_opt_live({#st{ssa=Linear0}=St, FuncDb}) -> + RevLinear = reverse(Linear0), + Blocks0 = maps:from_list(RevLinear), + Blocks = live_opt(RevLinear, #{}, Blocks0), + Linear = beam_ssa:linearize(Blocks), + {St#st{ssa=Linear}, FuncDb}. + +live_opt([{L,Blk0}|Bs], LiveMap0, Blocks) -> + Blk1 = beam_ssa_share:block(Blk0, Blocks), + Successors = beam_ssa:successors(Blk1), + Live0 = live_opt_succ(Successors, L, LiveMap0, gb_sets:empty()), + {Blk,Live} = live_opt_blk(Blk1, Live0), + LiveMap = live_opt_phis(Blk#b_blk.is, L, Live, LiveMap0), + live_opt(Bs, LiveMap, Blocks#{L:=Blk}); +live_opt([], _, Acc) -> Acc. + +live_opt_succ([S|Ss], L, LiveMap, Live0) -> + Key = {S,L}, + case LiveMap of + #{Key:=Live} -> + %% The successor has a phi node, and the value for + %% this block in the phi node is a variable. + live_opt_succ(Ss, L, LiveMap, gb_sets:union(Live, Live0)); + #{S:=Live} -> + %% No phi node in the successor, or the value for + %% this block in the phi node is a literal. + live_opt_succ(Ss, L, LiveMap, gb_sets:union(Live, Live0)); + #{} -> + %% A peek_message block which has not been processed yet. + live_opt_succ(Ss, L, LiveMap, Live0) + end; +live_opt_succ([], _, _, Acc) -> Acc. + +live_opt_phis(Is, L, Live0, LiveMap0) -> + LiveMap = LiveMap0#{L=>Live0}, + Phis = takewhile(fun(#b_set{op=Op}) -> Op =:= phi end, Is), + case Phis of + [] -> + LiveMap; + [_|_] -> + PhiArgs = append([Args || #b_set{args=Args} <- Phis]), + case [{P,V} || {#b_var{}=V,P} <- PhiArgs] of + [_|_]=PhiVars -> + PhiLive0 = rel2fam(PhiVars), + PhiLive = [{{L,P},gb_sets:union(gb_sets:from_list(Vs), Live0)} || + {P,Vs} <- PhiLive0], + maps:merge(LiveMap, maps:from_list(PhiLive)); + [] -> + %% There were only literals in the phi node(s). + LiveMap + end + end. + +live_opt_blk(#b_blk{is=Is0,last=Last}=Blk, Live0) -> + Live1 = gb_sets:union(Live0, gb_sets:from_ordset(beam_ssa:used(Last))), + {Is,Live} = live_opt_is(reverse(Is0), Live1, []), + {Blk#b_blk{is=Is},Live}. + +live_opt_is([#b_set{op=phi,dst=Dst}=I|Is], Live, Acc) -> + case gb_sets:is_member(Dst, Live) of + true -> + live_opt_is(Is, Live, [I|Acc]); + false -> + live_opt_is(Is, Live, Acc) + end; +live_opt_is([#b_set{op=succeeded,dst=SuccDst=SuccDstVar, + args=[Dst]}=SuccI, + #b_set{dst=Dst}=I|Is], Live0, Acc) -> + case gb_sets:is_member(Dst, Live0) of + true -> + Live1 = gb_sets:add(Dst, Live0), + Live = gb_sets:delete_any(SuccDst, Live1), + live_opt_is([I|Is], Live, [SuccI|Acc]); + false -> + case live_opt_unused(I) of + {replace,NewI0} -> + NewI = NewI0#b_set{dst=SuccDstVar}, + live_opt_is([NewI|Is], Live0, Acc); + keep -> + case gb_sets:is_member(SuccDst, Live0) of + true -> + Live1 = gb_sets:add(Dst, Live0), + Live = gb_sets:delete(SuccDst, Live1), + live_opt_is([I|Is], Live, [SuccI|Acc]); + false -> + live_opt_is([I|Is], Live0, Acc) + end + end + end; +live_opt_is([#b_set{dst=Dst}=I|Is], Live0, Acc) -> + case gb_sets:is_member(Dst, Live0) of + true -> + Live1 = gb_sets:union(Live0, gb_sets:from_ordset(beam_ssa:used(I))), + Live = gb_sets:delete(Dst, Live1), + live_opt_is(Is, Live, [I|Acc]); + false -> + case beam_ssa:no_side_effect(I) of + true -> + live_opt_is(Is, Live0, Acc); + false -> + Live = gb_sets:union(Live0, gb_sets:from_ordset(beam_ssa:used(I))), + live_opt_is(Is, Live, [I|Acc]) + end + end; +live_opt_is([], Live, Acc) -> + {Acc,Live}. + +live_opt_unused(#b_set{op=get_map_element}=Set) -> + {replace,Set#b_set{op=has_map_field}}; +live_opt_unused(_) -> keep. + +%%% +%%% Optimize binary matching. +%%% +%%% * If the value of segment is never extracted, rewrite +%%% to a bs_skip instruction. +%%% +%%% * Coalesce adjacent bs_skip instructions and skip instructions +%%% with bs_test_tail. +%%% + +ssa_opt_bsm({#st{ssa=Linear}=St, FuncDb}) -> + Extracted0 = bsm_extracted(Linear), + Extracted = cerl_sets:from_list(Extracted0), + {St#st{ssa=bsm_skip(Linear, Extracted)}, FuncDb}. + +bsm_skip([{L,#b_blk{is=Is0}=Blk}|Bs0], Extracted) -> + Bs = bsm_skip(Bs0, Extracted), + Is = bsm_skip_is(Is0, Extracted), + coalesce_skips({L,Blk#b_blk{is=Is}}, Bs); +bsm_skip([], _) -> []. + +bsm_skip_is([I0|Is], Extracted) -> + case I0 of + #b_set{op=bs_match, + dst=Ctx, + args=[#b_literal{val=T}=Type,PrevCtx|Args0]} + when T =/= string, T =/= skip -> + I = case cerl_sets:is_element(Ctx, Extracted) of + true -> + I0; + false -> + %% The value is never extracted. + Args = [#b_literal{val=skip},PrevCtx,Type|Args0], + I0#b_set{args=Args} + end, + [I|Is]; + #b_set{} -> + [I0|bsm_skip_is(Is, Extracted)] + end; +bsm_skip_is([], _) -> []. + +bsm_extracted([{_,#b_blk{is=Is}}|Bs]) -> + case Is of + [#b_set{op=bs_extract,args=[Ctx]}|_] -> + [Ctx|bsm_extracted(Bs)]; + _ -> + bsm_extracted(Bs) + end; +bsm_extracted([]) -> []. + +coalesce_skips({L,#b_blk{is=[#b_set{op=bs_extract}=Extract|Is0], + last=Last0}=Blk0}, Bs0) -> + case coalesce_skips_is(Is0, Last0, Bs0) of + not_possible -> + [{L,Blk0}|Bs0]; + {Is,Last,Bs} -> + Blk = Blk0#b_blk{is=[Extract|Is],last=Last}, + [{L,Blk}|Bs] + end; +coalesce_skips({L,#b_blk{is=Is0,last=Last0}=Blk0}, Bs0) -> + case coalesce_skips_is(Is0, Last0, Bs0) of + not_possible -> + [{L,Blk0}|Bs0]; + {Is,Last,Bs} -> + Blk = Blk0#b_blk{is=Is,last=Last}, + [{L,Blk}|Bs] + end. + +coalesce_skips_is([#b_set{op=bs_match, + args=[#b_literal{val=skip}, + Ctx0,Type,Flags, + #b_literal{val=Size0}, + #b_literal{val=Unit0}]}=Skip0, + #b_set{op=succeeded}], + #b_br{succ=L2,fail=Fail}=Br0, + Bs0) when is_integer(Size0) -> + case Bs0 of + [{L2,#b_blk{is=[#b_set{op=bs_match, + dst=SkipDst, + args=[#b_literal{val=skip},_,_,_, + #b_literal{val=Size1}, + #b_literal{val=Unit1}]}, + #b_set{op=succeeded}=Succeeded], + last=#b_br{fail=Fail}=Br}}|Bs] when is_integer(Size1) -> + SkipBits = Size0 * Unit0 + Size1 * Unit1, + Skip = Skip0#b_set{dst=SkipDst, + args=[#b_literal{val=skip},Ctx0, + Type,Flags, + #b_literal{val=SkipBits}, + #b_literal{val=1}]}, + Is = [Skip,Succeeded], + {Is,Br,Bs}; + [{L2,#b_blk{is=[#b_set{op=bs_test_tail, + args=[_Ctx,#b_literal{val=TailSkip}]}], + last=#b_br{succ=NextSucc,fail=Fail}}}|Bs] -> + SkipBits = Size0 * Unit0, + TestTail = Skip0#b_set{op=bs_test_tail, + args=[Ctx0,#b_literal{val=SkipBits+TailSkip}]}, + Br = Br0#b_br{bool=TestTail#b_set.dst,succ=NextSucc}, + Is = [TestTail], + {Is,Br,Bs}; + _ -> + not_possible + end; +coalesce_skips_is(_, _, _) -> + not_possible. + +%%% +%%% Short-cutting binary matching instructions. +%%% + +ssa_opt_bsm_shortcut({#st{ssa=Linear}=St, FuncDb}) -> + Positions = bsm_positions(Linear, #{}), + case map_size(Positions) of + 0 -> + %% No binary matching instructions. + {St, FuncDb}; + _ -> + {St#st{ssa=bsm_shortcut(Linear, Positions)}, FuncDb} + end. + +bsm_positions([{L,#b_blk{is=Is,last=Last}}|Bs], PosMap0) -> + PosMap = bsm_positions_is(Is, PosMap0), + case {Is,Last} of + {[#b_set{op=bs_test_tail,dst=Bool,args=[Ctx,#b_literal{val=Bits0}]}], + #b_br{bool=Bool,fail=Fail}} -> + Bits = Bits0 + map_get(Ctx, PosMap0), + bsm_positions(Bs, PosMap#{L=>{Bits,Fail}}); + {_,_} -> + bsm_positions(Bs, PosMap) + end; +bsm_positions([], PosMap) -> PosMap. + +bsm_positions_is([#b_set{op=bs_start_match,dst=New}|Is], PosMap0) -> + PosMap = PosMap0#{New=>0}, + bsm_positions_is(Is, PosMap); +bsm_positions_is([#b_set{op=bs_match,dst=New,args=Args}|Is], PosMap0) -> + [_,Old|_] = Args, + #{Old:=Bits0} = PosMap0, + Bits = bsm_update_bits(Args, Bits0), + PosMap = PosMap0#{New=>Bits}, + bsm_positions_is(Is, PosMap); +bsm_positions_is([_|Is], PosMap) -> + bsm_positions_is(Is, PosMap); +bsm_positions_is([], PosMap) -> PosMap. + +bsm_update_bits([#b_literal{val=string},_,#b_literal{val=String}], Bits) -> + Bits + bit_size(String); +bsm_update_bits([#b_literal{val=utf8}|_], Bits) -> + Bits + 8; +bsm_update_bits([#b_literal{val=utf16}|_], Bits) -> + Bits + 16; +bsm_update_bits([#b_literal{val=utf32}|_], Bits) -> + Bits + 32; +bsm_update_bits([_,_,_,#b_literal{val=Sz},#b_literal{val=U}], Bits) + when is_integer(Sz) -> + Bits + Sz*U; +bsm_update_bits(_, Bits) -> Bits. + +bsm_shortcut([{L,#b_blk{is=Is,last=Last0}=Blk}|Bs], PosMap) -> + case {Is,Last0} of + {[#b_set{op=bs_match,dst=New,args=[_,Old|_]}, + #b_set{op=succeeded,dst=Bool,args=[New]}], + #b_br{bool=Bool,fail=Fail}} -> + case PosMap of + #{Old:=Bits,Fail:={TailBits,NextFail}} when Bits > TailBits -> + Last = Last0#b_br{fail=NextFail}, + [{L,Blk#b_blk{last=Last}}|bsm_shortcut(Bs, PosMap)]; + #{} -> + [{L,Blk}|bsm_shortcut(Bs, PosMap)] + end; + {_,_} -> + [{L,Blk}|bsm_shortcut(Bs, PosMap)] + end; +bsm_shortcut([], _PosMap) -> []. + +%%% +%%% Eliminate redundant bs_test_unit2 instructions. +%%% + +ssa_opt_bsm_units({#st{ssa=Linear}=St, FuncDb}) -> + {St#st{ssa=bsm_units(Linear, #{})}, FuncDb}. + +bsm_units([{L,#b_blk{last=#b_br{succ=Succ,fail=Fail}}=Block0} | Bs], UnitMaps0) -> + UnitsIn = maps:get(L, UnitMaps0, #{}), + {Block, UnitsOut} = bsm_units_skip(Block0, UnitsIn), + UnitMaps1 = bsm_units_join(Succ, UnitsOut, UnitMaps0), + UnitMaps = bsm_units_join(Fail, UnitsIn, UnitMaps1), + [{L, Block} | bsm_units(Bs, UnitMaps)]; +bsm_units([{L,#b_blk{last=#b_switch{fail=Fail,list=Switch}}=Block} | Bs], UnitMaps0) -> + UnitsIn = maps:get(L, UnitMaps0, #{}), + Labels = [Fail | [Lbl || {_Arg, Lbl} <- Switch]], + UnitMaps = foldl(fun(Lbl, UnitMaps) -> + bsm_units_join(Lbl, UnitsIn, UnitMaps) + end, UnitMaps0, Labels), + [{L, Block} | bsm_units(Bs, UnitMaps)]; +bsm_units([{L, Block} | Bs], UnitMaps) -> + [{L, Block} | bsm_units(Bs, UnitMaps)]; +bsm_units([], _UnitMaps) -> + []. + +bsm_units_skip(Block, Units) -> + bsm_units_skip_1(Block#b_blk.is, Block, Units). + +bsm_units_skip_1([#b_set{op=bs_start_match,dst=New}|_], Block, Units) -> + %% We bail early since there can't be more than one match per block. + {Block, Units#{ New => 1 }}; +bsm_units_skip_1([#b_set{op=bs_match, + dst=New, + args=[#b_literal{val=skip}, + Ctx, + #b_literal{val=binary}, + _Flags, + #b_literal{val=all}, + #b_literal{val=OpUnit}]}=Skip | Test], + Block0, Units) -> + [#b_set{op=succeeded,dst=Bool,args=[New]}] = Test, %Assertion. + #b_br{bool=Bool} = Last0 = Block0#b_blk.last, %Assertion. + CtxUnit = map_get(Ctx, Units), + if + CtxUnit rem OpUnit =:= 0 -> + Is = takewhile(fun(I) -> I =/= Skip end, Block0#b_blk.is), + Last = Last0#b_br{bool=#b_literal{val=true}}, + Block = Block0#b_blk{is=Is,last=Last}, + {Block, Units#{ New => CtxUnit }}; + CtxUnit rem OpUnit =/= 0 -> + {Block0, Units#{ New => OpUnit, Ctx => OpUnit }} + end; +bsm_units_skip_1([#b_set{op=bs_match,dst=New,args=Args}|_], Block, Units) -> + [_,Ctx|_] = Args, + CtxUnit = map_get(Ctx, Units), + OpUnit = bsm_op_unit(Args), + {Block, Units#{ New => gcd(OpUnit, CtxUnit) }}; +bsm_units_skip_1([_I | Is], Block, Units) -> + bsm_units_skip_1(Is, Block, Units); +bsm_units_skip_1([], Block, Units) -> + {Block, Units}. + +bsm_op_unit([_,_,_,Size,#b_literal{val=U}]) -> + case Size of + #b_literal{val=Sz} when is_integer(Sz) -> Sz*U; + _ -> U + end; +bsm_op_unit([#b_literal{val=string},_,#b_literal{val=String}]) -> + bit_size(String); +bsm_op_unit([#b_literal{val=utf8}|_]) -> + 8; +bsm_op_unit([#b_literal{val=utf16}|_]) -> + 16; +bsm_op_unit([#b_literal{val=utf32}|_]) -> + 32; +bsm_op_unit(_) -> + 1. + +%% Several paths can lead to the same match instruction and the inferred units +%% may differ between them, so we can only keep the information that is common +%% to all paths. +bsm_units_join(Lbl, MapA, UnitMaps0) when is_map_key(Lbl, UnitMaps0) -> + MapB = map_get(Lbl, UnitMaps0), + Merged = if + map_size(MapB) =< map_size(MapA) -> + bsm_units_join_1(maps:keys(MapB), MapA, MapB); + map_size(MapB) > map_size(MapA) -> + bsm_units_join_1(maps:keys(MapA), MapB, MapA) + end, + UnitMaps0#{Lbl := Merged}; +bsm_units_join(Lbl, MapA, UnitMaps0) when MapA =/= #{} -> + UnitMaps0#{Lbl => MapA}; +bsm_units_join(_Lbl, _MapA, UnitMaps0) -> + UnitMaps0. + +bsm_units_join_1([Key | Keys], Left, Right) when is_map_key(Key, Left) -> + UnitA = map_get(Key, Left), + UnitB = map_get(Key, Right), + bsm_units_join_1(Keys, Left, Right#{Key := gcd(UnitA, UnitB)}); +bsm_units_join_1([Key | Keys], Left, Right) -> + bsm_units_join_1(Keys, Left, maps:remove(Key, Right)); +bsm_units_join_1([], _MapA, Right) -> + Right. + +%%% +%%% Optimize binary construction. +%%% +%%% If an integer segment or a float segment has a literal size and +%%% a literal value, convert to a binary segment. Coalesce adjacent +%%% literal binary segments. Literal binary segments will be converted +%%% to bs_put_string instructions in later pass. +%%% + +ssa_opt_bs_puts({#st{ssa=Linear0,cnt=Count0}=St, FuncDb}) -> + {Linear,Count} = opt_bs_puts(Linear0, Count0, []), + {St#st{ssa=Linear,cnt=Count}, FuncDb}. + +opt_bs_puts([{L,#b_blk{is=Is}=Blk0}|Bs], Count0, Acc0) -> + case Is of + [#b_set{op=bs_put}=I0] -> + case opt_bs_put(L, I0, Blk0, Count0, Acc0) of + not_possible -> + opt_bs_puts(Bs, Count0, [{L,Blk0}|Acc0]); + {Count,Acc1} -> + Acc = opt_bs_puts_merge(Acc1), + opt_bs_puts(Bs, Count, Acc) + end; + _ -> + opt_bs_puts(Bs, Count0, [{L,Blk0}|Acc0]) + end; +opt_bs_puts([], Count, Acc) -> + {reverse(Acc),Count}. + +opt_bs_puts_merge([{L1,#b_blk{is=Is}=Blk0},{L2,#b_blk{is=AccIs}}=BAcc|Acc]) -> + case {AccIs,Is} of + {[#b_set{op=bs_put, + args=[#b_literal{val=binary}, + #b_literal{}, + #b_literal{val=Bin0}, + #b_literal{val=all}, + #b_literal{val=1}]}], + [#b_set{op=bs_put, + args=[#b_literal{val=binary}, + #b_literal{}, + #b_literal{val=Bin1}, + #b_literal{val=all}, + #b_literal{val=1}]}=I0]} -> + %% Coalesce the two segments to one. + Bin = <<Bin0/bitstring,Bin1/bitstring>>, + I = I0#b_set{args=bs_put_args(binary, Bin, all)}, + Blk = Blk0#b_blk{is=[I]}, + [{L2,Blk}|Acc]; + {_,_} -> + [{L1,Blk0},BAcc|Acc] + end. + +opt_bs_put(L, I0, #b_blk{last=Br0}=Blk0, Count0, Acc) -> + case opt_bs_put(I0) of + [Bin] when is_bitstring(Bin) -> + Args = bs_put_args(binary, Bin, all), + I = I0#b_set{args=Args}, + Blk = Blk0#b_blk{is=[I]}, + {Count0,[{L,Blk}|Acc]}; + [{int,Int,Size},Bin] when is_bitstring(Bin) -> + %% Construct a bs_put_integer instruction following + %% by a bs_put_binary instruction. + IntArgs = bs_put_args(integer, Int, Size), + BinArgs = bs_put_args(binary, Bin, all), + {BinL,BinVarNum} = {Count0,Count0+1}, + Count = Count0 + 2, + BinVar = #b_var{name={'@ssa_bool',BinVarNum}}, + BinI = I0#b_set{dst=BinVar,args=BinArgs}, + BinBlk = Blk0#b_blk{is=[BinI],last=Br0#b_br{bool=BinVar}}, + IntI = I0#b_set{args=IntArgs}, + IntBlk = Blk0#b_blk{is=[IntI],last=Br0#b_br{succ=BinL}}, + {Count,[{BinL,BinBlk},{L,IntBlk}|Acc]}; + not_possible -> + not_possible + end. + +opt_bs_put(#b_set{args=[#b_literal{val=binary},_,#b_literal{val=Val}, + #b_literal{val=all},#b_literal{val=Unit}]}) + when is_bitstring(Val) -> + if + bit_size(Val) rem Unit =:= 0 -> + [Val]; + true -> + not_possible + end; +opt_bs_put(#b_set{args=[#b_literal{val=Type},#b_literal{val=Flags}, + #b_literal{val=Val},#b_literal{val=Size}, + #b_literal{val=Unit}]}=I0) when is_integer(Size) -> + EffectiveSize = Size * Unit, + if + EffectiveSize > 0 -> + case {Type,opt_bs_put_endian(Flags)} of + {integer,big} when is_integer(Val) -> + if + EffectiveSize < 64 -> + [<<Val:EffectiveSize>>]; + true -> + opt_bs_put_split_int(Val, EffectiveSize) + end; + {integer,little} when is_integer(Val), EffectiveSize < 128 -> + %% To avoid an explosion in code size, we only try + %% to optimize relatively small fields. + <<Int:EffectiveSize>> = <<Val:EffectiveSize/little>>, + Args = bs_put_args(Type, Int, EffectiveSize), + I = I0#b_set{args=Args}, + opt_bs_put(I); + {binary,_} when is_bitstring(Val) -> + <<Bitstring:EffectiveSize/bits,_/bits>> = Val, + [Bitstring]; + {float,Endian} -> + try + [opt_bs_put_float(Val, EffectiveSize, Endian)] + catch error:_ -> + not_possible + end; + {_,_} -> + not_possible + end; + true -> + not_possible + end; +opt_bs_put(#b_set{}) -> not_possible. + +opt_bs_put_float(N, Sz, Endian) -> + case Endian of + big -> <<N:Sz/big-float-unit:1>>; + little -> <<N:Sz/little-float-unit:1>> + end. + +bs_put_args(Type, Val, Size) -> + [#b_literal{val=Type}, + #b_literal{val=[unsigned,big]}, + #b_literal{val=Val}, + #b_literal{val=Size}, + #b_literal{val=1}]. + +opt_bs_put_endian([big=E|_]) -> E; +opt_bs_put_endian([little=E|_]) -> E; +opt_bs_put_endian([native=E|_]) -> E; +opt_bs_put_endian([_|Fs]) -> opt_bs_put_endian(Fs). + +opt_bs_put_split_int(Int, Size) -> + Pos = opt_bs_put_split_int_1(Int, 0, Size - 1), + UpperSize = Size - Pos, + if + Pos =:= 0 -> + %% Value is 0 or -1 -- keep the original instruction. + not_possible; + UpperSize < 64 -> + %% No or few leading zeroes or ones. + [<<Int:Size>>]; + true -> + %% There are 64 or more leading ones or zeroes in + %% the resulting binary. Split into two separate + %% segments to avoid an explosion in code size. + [{int,Int bsr Pos,UpperSize},<<Int:Pos>>] + end. + +opt_bs_put_split_int_1(_Int, L, R) when L > R -> + 8 * ((L + 7) div 8); +opt_bs_put_split_int_1(Int, L, R) -> + Mid = (L + R) div 2, + case Int bsr Mid of + Upper when Upper =:= 0; Upper =:= -1 -> + opt_bs_put_split_int_1(Int, L, Mid - 1); + _ -> + opt_bs_put_split_int_1(Int, Mid + 1, R) + end. + +%%% +%%% Optimize expressions such as "tuple_size(Var) =:= 2". +%%% +%%% Consider this code: +%%% +%%% 0: +%%% . +%%% . +%%% . +%%% Size = bif:tuple_size Var +%%% BoolVar1 = succeeded Size +%%% br BoolVar1, label 4, label 3 +%%% +%%% 4: +%%% BoolVar2 = bif:'=:=' Size, literal 2 +%%% br BoolVar2, label 6, label 3 +%%% +%%% 6: ... %% OK +%%% +%%% 3: ... %% Not a tuple of size 2 +%%% +%%% The BEAM code will look this: +%%% +%%% {bif,tuple_size,{f,3},[{x,0}],{x,0}}. +%%% {test,is_eq_exact,{f,3},[{x,0},{integer,2}]}. +%%% +%%% Better BEAM code will be produced if we transform the +%%% code like this: +%%% +%%% 0: +%%% . +%%% . +%%% . +%%% br label 10 +%%% +%%% 10: +%%% NewBoolVar = bif:is_tuple Var +%%% br NewBoolVar, label 11, label 3 +%%% +%%% 11: +%%% Size = bif:tuple_size Var +%%% br label 4 +%%% +%%% 4: +%%% BoolVar2 = bif:'=:=' Size, literal 2 +%%% br BoolVar2, label 6, label 3 +%%% +%%% (The key part of the transformation is the removal of +%%% the 'succeeded' instruction to signal to the code generator +%%% that the call to tuple_size/1 can't fail.) +%%% +%%% The BEAM code will look like: +%%% +%%% {test,is_tuple,{f,3},[{x,0}]}. +%%% {test_arity,{f,3},[{x,0},2]}. +%%% +%%% Those two instructions will be combined into a single +%%% is_tuple_of_arity instruction by the loader. +%%% + +ssa_opt_tuple_size({#st{ssa=Linear0,cnt=Count0}=St, FuncDb}) -> + {Linear,Count} = opt_tup_size(Linear0, Count0, []), + {St#st{ssa=Linear,cnt=Count}, FuncDb}. + +opt_tup_size([{L,#b_blk{is=Is,last=Last}=Blk}|Bs], Count0, Acc0) -> + case {Is,Last} of + {[#b_set{op={bif,'=:='},dst=Bool,args=[#b_var{}=Tup,#b_literal{val=Arity}]}], + #b_br{bool=Bool}} when is_integer(Arity), Arity >= 0 -> + {Acc,Count} = opt_tup_size_1(Tup, L, Count0, Acc0), + opt_tup_size(Bs, Count, [{L,Blk}|Acc]); + {_,_} -> + opt_tup_size(Bs, Count0, [{L,Blk}|Acc0]) + end; +opt_tup_size([], Count, Acc) -> + {reverse(Acc),Count}. + +opt_tup_size_1(Size, EqL, Count0, [{L,Blk0}|Acc]) -> + case Blk0 of + #b_blk{is=Is0,last=#b_br{bool=Bool,succ=EqL,fail=Fail}} -> + case opt_tup_size_is(Is0, Bool, Size, []) of + none -> + {[{L,Blk0}|Acc],Count0}; + {PreIs,TupleSizeIs,Tuple} -> + opt_tup_size_2(PreIs, TupleSizeIs, L, EqL, + Tuple, Fail, Count0, Acc) + end; + #b_blk{} -> + {[{L,Blk0}|Acc],Count0} + end; +opt_tup_size_1(_, _, Count, Acc) -> + {Acc,Count}. + +opt_tup_size_2(PreIs, TupleSizeIs, PreL, EqL, Tuple, Fail, Count0, Acc) -> + IsTupleL = Count0, + TupleSizeL = Count0 + 1, + Bool = #b_var{name={'@ssa_bool',Count0+2}}, + Count = Count0 + 3, + + True = #b_literal{val=true}, + PreBr = #b_br{bool=True,succ=IsTupleL,fail=IsTupleL}, + PreBlk = #b_blk{is=PreIs,last=PreBr}, + + IsTupleIs = [#b_set{op={bif,is_tuple},dst=Bool,args=[Tuple]}], + IsTupleBr = #b_br{bool=Bool,succ=TupleSizeL,fail=Fail}, + IsTupleBlk = #b_blk{is=IsTupleIs,last=IsTupleBr}, + + TupleSizeBr = #b_br{bool=True,succ=EqL,fail=EqL}, + TupleSizeBlk = #b_blk{is=TupleSizeIs,last=TupleSizeBr}, + {[{TupleSizeL,TupleSizeBlk}, + {IsTupleL,IsTupleBlk}, + {PreL,PreBlk}|Acc],Count}. + +opt_tup_size_is([#b_set{op={bif,tuple_size},dst=Size,args=[Tuple]}=I, + #b_set{op=succeeded,dst=Bool,args=[Size]}], + Bool, Size, Acc) -> + {reverse(Acc),[I],Tuple}; +opt_tup_size_is([I|Is], Bool, Size, Acc) -> + opt_tup_size_is(Is, Bool, Size, [I|Acc]); +opt_tup_size_is([], _, _, _Acc) -> none. + +%%% +%%% Optimize #b_switch{} instructions. +%%% +%%% If the argument for a #b_switch{} comes from a phi node with all +%%% literals, any values in the switch list which are not in the phi +%%% node can be removed. +%%% +%%% If the values in the phi node and switch list are the same, +%%% the failure label can't be reached and be eliminated. +%%% +%%% A #b_switch{} with only one value can be rewritten to +%%% a #b_br{}. A switch that only verifies that the argument +%%% is 'true' or 'false' can be rewritten to a is_boolean test. +%%% + +ssa_opt_sw({#st{ssa=Linear0,cnt=Count0}=St, FuncDb}) -> + {Linear,Count} = opt_sw(Linear0, Count0, []), + {St#st{ssa=Linear,cnt=Count}, FuncDb}. + +opt_sw([{L,#b_blk{is=Is,last=#b_switch{}=Sw0}=Blk0}|Bs], Count0, Acc) -> + %% Ensure that no label in the switch list is the same + %% as the failure label. + #b_switch{fail=Fail,list=List0} = Sw0, + List = [{Val,Lbl} || {Val,Lbl} <- List0, Lbl =/= Fail], + Sw1 = beam_ssa:normalize(Sw0#b_switch{list=List}), + case Sw1 of + #b_switch{arg=Arg,fail=Fail,list=[{Lit,Lbl}]} -> + %% Rewrite a single value switch to a br. + Bool = #b_var{name={'@ssa_bool',Count0}}, + Count = Count0 + 1, + IsEq = #b_set{op={bif,'=:='},dst=Bool,args=[Arg,Lit]}, + Br = #b_br{bool=Bool,succ=Lbl,fail=Fail}, + Blk = Blk0#b_blk{is=Is++[IsEq],last=Br}, + opt_sw(Bs, Count, [{L,Blk}|Acc]); + #b_switch{arg=Arg,fail=Fail, + list=[{#b_literal{val=B1},Lbl},{#b_literal{val=B2},Lbl}]} + when B1 =:= not B2 -> + %% Replace with is_boolean test. + Bool = #b_var{name={'@ssa_bool',Count0}}, + Count = Count0 + 1, + IsBool = #b_set{op={bif,is_boolean},dst=Bool,args=[Arg]}, + Br = #b_br{bool=Bool,succ=Lbl,fail=Fail}, + Blk = Blk0#b_blk{is=Is++[IsBool],last=Br}, + opt_sw(Bs, Count, [{L,Blk}|Acc]); + Sw0 -> + opt_sw(Bs, Count0, [{L,Blk0}|Acc]); + Sw -> + Blk = Blk0#b_blk{last=Sw}, + opt_sw(Bs, Count0, [{L,Blk}|Acc]) + end; +opt_sw([{L,#b_blk{}=Blk}|Bs], Count, Acc) -> + opt_sw(Bs, Count, [{L,Blk}|Acc]); +opt_sw([], Count, Acc) -> + {reverse(Acc),Count}. + +%%% +%%% Merge blocks. +%%% + +ssa_opt_merge_blocks({#st{ssa=Blocks}=St, FuncDb}) -> + Preds = beam_ssa:predecessors(Blocks), + Merged = merge_blocks_1(beam_ssa:rpo(Blocks), Preds, Blocks), + {St#st{ssa=Merged}, FuncDb}. + +merge_blocks_1([L|Ls], Preds0, Blocks0) -> + case Preds0 of + #{L:=[P]} -> + #{P:=Blk0,L:=Blk1} = Blocks0, + case is_merge_allowed(L, Blk0, Blk1) of + true -> + #b_blk{is=Is0} = Blk0, + #b_blk{is=Is1} = Blk1, + verify_merge_is(Is1), + Is = Is0 ++ Is1, + Blk = Blk1#b_blk{is=Is}, + Blocks1 = maps:remove(L, Blocks0), + Blocks2 = Blocks1#{P:=Blk}, + Successors = beam_ssa:successors(Blk), + Blocks = beam_ssa:update_phi_labels(Successors, L, P, Blocks2), + Preds = merge_update_preds(Successors, L, P, Preds0), + merge_blocks_1(Ls, Preds, Blocks); + false -> + merge_blocks_1(Ls, Preds0, Blocks0) + end; + #{} -> + merge_blocks_1(Ls, Preds0, Blocks0) + end; +merge_blocks_1([], _Preds, Blocks) -> Blocks. + +merge_update_preds([L|Ls], From, To, Preds0) -> + Ps = [rename_label(P, From, To) || P <- map_get(L, Preds0)], + Preds = Preds0#{L:=Ps}, + merge_update_preds(Ls, From, To, Preds); +merge_update_preds([], _, _, Preds) -> Preds. + +rename_label(From, From, To) -> To; +rename_label(Lbl, _, _) -> Lbl. + +verify_merge_is([#b_set{op=Op}|_]) -> + %% The merged block has only one predecessor, so it should not have any phi + %% nodes. + true = Op =/= phi; %Assertion. +verify_merge_is(_) -> + ok. + +is_merge_allowed(_, #b_blk{}, #b_blk{is=[#b_set{op=peek_message}|_]}) -> + false; +is_merge_allowed(L, #b_blk{last=#b_br{}}=Blk, #b_blk{}) -> + %% The predecessor block must have exactly one successor (L) for + %% the merge to be safe. + case beam_ssa:successors(Blk) of + [L] -> true; + [_|_] -> false + end; +is_merge_allowed(_, #b_blk{last=#b_switch{}}, #b_blk{}) -> + false. + +%%% +%%% When a tuple is matched, the pattern matching compiler generates a +%%% get_tuple_element instruction for every tuple element that will +%%% ever be used in the rest of the function. That often forces the +%%% extracted tuple elements to be stored in Y registers until it's +%%% time to use them. It could also mean that there could be execution +%%% paths that will never use the extracted elements. +%%% +%%% This optimization will sink get_tuple_element instructions, that +%%% is, move them forward in the execution stream to the last possible +%%% block there they will still dominate all uses. That may reduce the +%%% size of stack frames, reduce register shuffling, and avoid +%%% extracting tuple elements on execution paths that never use the +%%% extracted values. +%%% + +ssa_opt_sink({#st{ssa=Blocks0}=St, FuncDb}) -> + Linear = beam_ssa:linearize(Blocks0), + + %% Create a map with all variables that define get_tuple_element + %% instructions. The variable name map to the block it is defined in. + case def_blocks(Linear) of + [] -> + %% No get_tuple_element instructions, so there is nothing to do. + {St, FuncDb}; + [_|_]=Defs0 -> + Defs = maps:from_list(Defs0), + {do_ssa_opt_sink(Linear, Defs, St), FuncDb} + end. + +do_ssa_opt_sink(Linear, Defs, #st{ssa=Blocks0}=St) -> + %% Now find all the blocks that use variables defined by get_tuple_element + %% instructions. + Used = used_blocks(Linear, Defs, []), + + %% Calculate dominators. + {Dom,Numbering} = beam_ssa:dominators(Blocks0), + + %% It is not safe to move get_tuple_element instructions to blocks + %% that begin with certain instructions. It is also unsafe to move + %% the instructions into any part of a receive. To avoid such + %% unsafe moves, pretend that the unsuitable blocks are not + %% dominators. + Unsuitable = unsuitable(Linear, Blocks0), + + %% Calculate new positions for get_tuple_element instructions. The new + %% position is a block that dominates all uses of the variable. + DefLoc = new_def_locations(Used, Defs, Dom, Numbering, Unsuitable), + + %% Now move all suitable get_tuple_element instructions to their + %% new blocks. + Blocks = foldl(fun({V,To}, A) -> + From = map_get(V, Defs), + move_defs(V, From, To, A) + end, Blocks0, DefLoc), + St#st{ssa=Blocks}. + +def_blocks([{L,#b_blk{is=Is}}|Bs]) -> + def_blocks_is(Is, L, def_blocks(Bs)); +def_blocks([]) -> []. + +def_blocks_is([#b_set{op=get_tuple_element,dst=Dst}|Is], L, Acc) -> + def_blocks_is(Is, L, [{Dst,L}|Acc]); +def_blocks_is([_|Is], L, Acc) -> + def_blocks_is(Is, L, Acc); +def_blocks_is([], _, Acc) -> Acc. + +used_blocks([{L,Blk}|Bs], Def, Acc0) -> + Used = beam_ssa:used(Blk), + Acc = [{V,L} || V <- Used, maps:is_key(V, Def)] ++ Acc0, + used_blocks(Bs, Def, Acc); +used_blocks([], _Def, Acc) -> + rel2fam(Acc). + +%% unsuitable(Linear, Blocks) -> Unsuitable. +%% Return an ordset of block labels for the blocks that are not +%% suitable for sinking of get_tuple_element instructions. + +unsuitable(Linear, Blocks) -> + Predecessors = beam_ssa:predecessors(Blocks), + Unsuitable0 = unsuitable_1(Linear), + Unsuitable1 = unsuitable_recv(Linear, Blocks, Predecessors), + gb_sets:from_list(Unsuitable0 ++ Unsuitable1). + +unsuitable_1([{L,#b_blk{is=[#b_set{op=Op}|_]}}|Bs]) -> + Unsuitable = case Op of + bs_extract -> true; + bs_put -> true; + {float,_} -> true; + landingpad -> true; + peek_message -> true; + wait_timeout -> true; + _ -> false + end, + case Unsuitable of + true -> + [L|unsuitable_1(Bs)]; + false -> + unsuitable_1(Bs) + end; +unsuitable_1([{_,#b_blk{}}|Bs]) -> + unsuitable_1(Bs); +unsuitable_1([]) -> []. + +unsuitable_recv([{L,#b_blk{is=[#b_set{op=Op}|_]}}|Bs], Blocks, Predecessors) -> + Ls = case Op of + remove_message -> + unsuitable_loop(L, Blocks, Predecessors); + recv_next -> + unsuitable_loop(L, Blocks, Predecessors); + _ -> + [] + end, + Ls ++ unsuitable_recv(Bs, Blocks, Predecessors); +unsuitable_recv([_|Bs], Blocks, Predecessors) -> + unsuitable_recv(Bs, Blocks, Predecessors); +unsuitable_recv([], _, _) -> []. + +unsuitable_loop(L, Blocks, Predecessors) -> + unsuitable_loop(L, Blocks, Predecessors, []). + +unsuitable_loop(L, Blocks, Predecessors, Acc) -> + Ps = map_get(L, Predecessors), + unsuitable_loop_1(Ps, Blocks, Predecessors, Acc). + +unsuitable_loop_1([P|Ps], Blocks, Predecessors, Acc0) -> + case map_get(P, Blocks) of + #b_blk{is=[#b_set{op=peek_message}|_]} -> + unsuitable_loop_1(Ps, Blocks, Predecessors, Acc0); + #b_blk{} -> + case ordsets:is_element(P, Acc0) of + false -> + Acc1 = ordsets:add_element(P, Acc0), + Acc = unsuitable_loop(P, Blocks, Predecessors, Acc1), + unsuitable_loop_1(Ps, Blocks, Predecessors, Acc); + true -> + unsuitable_loop_1(Ps, Blocks, Predecessors, Acc0) + end + end; +unsuitable_loop_1([], _, _, Acc) -> Acc. + +%% new_def_locations([{Variable,[UsedInBlock]}|Vs], Defs, +%% Dominators, Numbering, Unsuitable) -> +%% [{Variable,NewDefinitionBlock}] +%% +%% Calculate new locations for get_tuple_element instructions. For +%% each variable, the new location is a block that dominates all uses +%% of the variable and as near to the uses of as possible. + +new_def_locations([{V,UsedIn}|Vs], Defs, Dom, Numbering, Unsuitable) -> + DefIn = map_get(V, Defs), + Common = common_dominator(UsedIn, Dom, Numbering, Unsuitable), + case member(Common, map_get(DefIn, Dom)) of + true -> + %% The common dominator is either DefIn or an + %% ancestor of DefIn. + new_def_locations(Vs, Defs, Dom, Numbering, Unsuitable); + false -> + %% We have found a suitable descendant of DefIn, + %% to which the get_tuple_element instruction can + %% be sunk. + [{V,Common}|new_def_locations(Vs, Defs, Dom, Numbering, Unsuitable)] + end; +new_def_locations([], _, _, _, _) -> []. + +common_dominator(Ls0, Dom, Numbering, Unsuitable) -> + [Common|_] = beam_ssa:common_dominators(Ls0, Dom, Numbering), + case gb_sets:is_member(Common, Unsuitable) of + true -> + %% It is not allowed to place the instruction here. Try + %% to find another suitable dominating block by going up + %% one step in the dominator tree. + [Common,OneUp|_] = map_get(Common, Dom), + common_dominator([OneUp], Dom, Numbering, Unsuitable); + false -> + Common + end. + +%% Move get_tuple_element instructions to their new locations. + +move_defs(V, From, To, Blocks) -> + #{From:=FromBlk0,To:=ToBlk0} = Blocks, + {Def,FromBlk} = remove_def(V, FromBlk0), + try insert_def(V, Def, ToBlk0) of + ToBlk -> + %%io:format("~p: ~p => ~p\n", [V,From,To]), + Blocks#{From:=FromBlk,To:=ToBlk} + catch + throw:not_possible -> + Blocks + end. + +remove_def(V, #b_blk{is=Is0}=Blk) -> + {Def,Is} = remove_def_is(Is0, V, []), + {Def,Blk#b_blk{is=Is}}. + +remove_def_is([#b_set{dst=Dst}=Def|Is], Dst, Acc) -> + {Def,reverse(Acc, Is)}; +remove_def_is([I|Is], Dst, Acc) -> + remove_def_is(Is, Dst, [I|Acc]). + +insert_def(V, Def, #b_blk{is=Is0}=Blk) -> + Is = insert_def_is(Is0, V, Def), + Blk#b_blk{is=Is}. + +insert_def_is([#b_set{op=phi}=I|Is], V, Def) -> + case member(V, beam_ssa:used(I)) of + true -> + throw(not_possible); + false -> + [I|insert_def_is(Is, V, Def)] + end; +insert_def_is([#b_set{op=Op}=I|Is]=Is0, V, Def) -> + Action0 = case Op of + call -> beyond; + 'catch_end' -> beyond; + timeout -> beyond; + _ -> here + end, + Action = case Is of + [#b_set{op=succeeded}|_] -> here; + _ -> Action0 + end, + case Action of + beyond -> + case member(V, beam_ssa:used(I)) of + true -> + %% The variable is used by this instruction. We must + %% place the definition before this instruction. + [Def|Is0]; + false -> + %% Place it beyond the current instruction. + [I|insert_def_is(Is, V, Def)] + end; + here -> + [Def|Is0] + end; +insert_def_is([], _V, Def) -> + [Def]. + +%%% +%%% Order consecutive get_tuple_element instructions in ascending +%%% position order. This will give the loader more opportunities +%%% for combining get_tuple_element instructions. +%%% + +ssa_opt_get_tuple_element({#st{ssa=Blocks0}=St, FuncDb}) -> + Blocks = opt_get_tuple_element(maps:to_list(Blocks0), Blocks0), + {St#st{ssa=Blocks}, FuncDb}. + +opt_get_tuple_element([{L,#b_blk{is=Is0}=Blk0}|Bs], Blocks) -> + case opt_get_tuple_element_is(Is0, false, []) of + {yes,Is} -> + Blk = Blk0#b_blk{is=Is}, + opt_get_tuple_element(Bs, Blocks#{L:=Blk}); + no -> + opt_get_tuple_element(Bs, Blocks) + end; +opt_get_tuple_element([], Blocks) -> Blocks. + +opt_get_tuple_element_is([#b_set{op=get_tuple_element, + args=[#b_var{}=Src,_]}=I0|Is0], + _AnyChange, Acc) -> + {GetIs0,Is} = collect_get_tuple_element(Is0, Src, [I0]), + GetIs1 = sort([{Pos,I} || #b_set{args=[_,Pos]}=I <- GetIs0]), + GetIs = [I || {_,I} <- GetIs1], + opt_get_tuple_element_is(Is, true, reverse(GetIs, Acc)); +opt_get_tuple_element_is([I|Is], AnyChange, Acc) -> + opt_get_tuple_element_is(Is, AnyChange, [I|Acc]); +opt_get_tuple_element_is([], AnyChange, Acc) -> + case AnyChange of + true -> {yes,reverse(Acc)}; + false -> no + end. + +collect_get_tuple_element([#b_set{op=get_tuple_element, + args=[Src,_]}=I|Is], Src, Acc) -> + collect_get_tuple_element(Is, Src, [I|Acc]); +collect_get_tuple_element(Is, _Src, Acc) -> + {Acc,Is}. + +%%% +%%% Common utilities. +%%% + +gcd(A, B) -> + case A rem B of + 0 -> B; + X -> gcd(B, X) + end. + +rel2fam(S0) -> + S1 = sofs:relation(S0), + S = sofs:rel2fam(S1), + sofs:to_external(S). + +sub(I, Sub) -> + beam_ssa:normalize(sub_1(I, Sub)). + +sub_1(#b_set{op=phi,args=Args}=I, Sub) -> + I#b_set{args=[{sub_arg(A, Sub),P} || {A,P} <- Args]}; +sub_1(#b_set{args=Args}=I, Sub) -> + I#b_set{args=[sub_arg(A, Sub) || A <- Args]}; +sub_1(#b_br{bool=#b_var{}=Old}=Br, Sub) -> + New = sub_arg(Old, Sub), + Br#b_br{bool=New}; +sub_1(#b_switch{arg=#b_var{}=Old}=Sw, Sub) -> + New = sub_arg(Old, Sub), + Sw#b_switch{arg=New}; +sub_1(#b_ret{arg=#b_var{}=Old}=Ret, Sub) -> + New = sub_arg(Old, Sub), + Ret#b_ret{arg=New}; +sub_1(Last, _) -> Last. + +sub_arg(#b_remote{mod=Mod,name=Name}=Rem, Sub) -> + Rem#b_remote{mod=sub_arg(Mod, Sub),name=sub_arg(Name, Sub)}; +sub_arg(Old, Sub) -> + case Sub of + #{Old:=New} -> New; + #{} -> Old + end. + +new_var(#b_var{name={Base,N}}, Count) -> + true = is_integer(N), %Assertion. + {#b_var{name={Base,Count}},Count+1}; +new_var(#b_var{name=Base}, Count) -> + {#b_var{name={Base,Count}},Count+1}. diff --git a/lib/compiler/src/beam_ssa_opt.hrl b/lib/compiler/src/beam_ssa_opt.hrl new file mode 100644 index 0000000000..37711a6f48 --- /dev/null +++ b/lib/compiler/src/beam_ssa_opt.hrl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-include("beam_ssa.hrl"). + +-record(func_info, + {%% Local calls going in/out of this function. + in = ordsets:new() :: ordsets:ordset(func_id()), + out = ordsets:new() :: ordsets:ordset(func_id()), + + %% Whether the function is exported or not; some optimizations may + %% need to be suppressed if it is. + exported = true :: boolean(), + + %% The inferred types of each argument (as opposed to parameter), + %% indexed by call site. + %% + %% This is more effective than the naive approach of joining into a + %% "parameter_type" as we go as it lets us narrow parameter types + %% without having to visit all callers on each pass, which helps a lot + %% when dealing with co-recursive functions. + arg_types = [] :: list(arg_type_map()), + + %% The inferred return type of this function, this is either [type()] + %% or [] to note absence. + ret_type = [] :: list()}). + +-type arg_key() :: {CallerId :: func_id(), + CallDst :: beam_ssa:b_var()}. +-type arg_type_map() :: #{ arg_key() => term() }. + +%% Per-function metadata used by various optimization passes to perform +%% module-level optimization. If a function is absent it means that +%% module-level optimization has been turned off for said function. +-type func_id() :: beam_ssa:b_local(). +-type func_info_db() :: #{ func_id() => #func_info{} }. diff --git a/lib/compiler/src/beam_ssa_pp.erl b/lib/compiler/src/beam_ssa_pp.erl new file mode 100644 index 0000000000..34ac08b32e --- /dev/null +++ b/lib/compiler/src/beam_ssa_pp.erl @@ -0,0 +1,238 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(beam_ssa_pp). + +-export([format_function/1,format_instr/1,format_var/1]). + +-include("beam_ssa.hrl"). + +-spec format_function(beam_ssa:b_function()) -> iolist(). + +format_function(#b_function{anno=Anno0,args=Args, + bs=Blocks,cnt=Counter}) -> + #{func_info:={M,F,_}} = Anno0, + Anno = maps:without([func_info,location,live_intervals,registers], Anno0), + FuncAnno = case Anno0 of + #{live_intervals:=Intervals} -> + Anno0#{live_intervals:=maps:from_list(Intervals)}; + #{} -> + Anno0 + end, + ReachableBlocks = beam_ssa:rpo(Blocks), + All = maps:keys(Blocks), + Unreachable = ordsets:subtract(ordsets:from_list(All), + ordsets:from_list(ReachableBlocks)), + [case Anno0 of + #{location:={Filename,Line}} -> + io_lib:format("%% ~ts:~p\n", [Filename,Line]); + #{} -> + [] + end, + io_lib:format("%% Counter = ~p\n", [Counter]), + [format_anno(Key, Value) || + {Key,Value} <- lists:sort(maps:to_list(Anno))], + io_lib:format("function ~p:~p(~ts) {\n", [M,F,format_args(Args, FuncAnno)]), + [format_live_interval(Var, FuncAnno) || Var <- Args], + format_blocks(ReachableBlocks, Blocks, FuncAnno), + case Unreachable of + [] -> + []; + [_|_] -> + ["\n%% Unreachable blocks\n\n", + format_blocks(Unreachable, Blocks, FuncAnno)] + end, + + "}\n"]. + + +-spec format_instr(beam_ssa:b_set()) -> iolist(). + +format_instr(#b_set{}=I) -> + Cs = lists:flatten(format_instr(I#b_set{anno=#{}}, #{}, true)), + string:trim(Cs, leading); +format_instr(I0) -> + I = setelement(2, I0, #{}), + Cs = lists:flatten(format_terminator(I, #{})), + string:trim(Cs, both). + +-spec format_var(beam_ssa:b_var()) -> iolist(). + +format_var(V) -> + Cs = lists:flatten(format_var(V, #{})), + string:trim(Cs, leading). + +%%% +%%% Local functions. +%%% + +format_anno(Key, Map) when is_map(Map) -> + Sorted = lists:sort(maps:to_list(Map)), + [io_lib:format("%% ~s:\n", [Key]), + [io_lib:format("%% ~w => ~w\n", [K,V]) || {K,V} <- Sorted]]; +format_anno(Key, Value) -> + io_lib:format("%% ~s: ~p\n", [Key,Value]). + +format_blocks(Ls, Blocks, Anno) -> + PP = [format_block(L, Blocks, Anno) || L <- Ls], + lists:join($\n, PP). + +format_block(L, Blocks, FuncAnno) -> + #b_blk{anno=Anno,is=Is,last=Last} = maps:get(L, Blocks), + [case map_size(Anno) of + 0 -> []; + _ -> io_lib:format("%% ~p\n", [Anno]) + end, + io_lib:format("~p:", [L]), + format_instrs(Is, FuncAnno, true), + $\n, + format_terminator(Last, FuncAnno)]. + +format_instrs([I|Is], FuncAnno, First) -> + [$\n, + format_instr(I, FuncAnno, First), + format_instrs(Is, FuncAnno, false)]; +format_instrs([], _FuncAnno, _First) -> + []. + +format_instr(#b_set{anno=Anno,op=Op,dst=Dst,args=Args}, + FuncAnno, First) -> + AnnoStr = format_anno(Anno), + LiveIntervalStr = format_live_interval(Dst, FuncAnno), + [if + First -> + []; + AnnoStr =/= []; LiveIntervalStr =/= [] -> + $\n; + true -> + [] + end, + AnnoStr, + LiveIntervalStr, + io_lib:format(" ~s~ts = ~ts", [format_i_number(Anno), + format_var(Dst, FuncAnno), + format_op(Op)]), + case Args of + [] -> + []; + [_|_] -> + io_lib:format(" ~ts", [format_args(Args, FuncAnno)]) + end]. + +format_i_number(#{n:=N}) -> + io_lib:format("[~p] ", [N]); +format_i_number(#{}) -> []. + +format_terminator(#b_br{anno=A,bool=#b_literal{val=true},succ=Lbl}, _) -> + io_lib:format(" ~sbr label ~p\n", [format_i_number(A),Lbl]); +format_terminator(#b_br{anno=A,bool=#b_literal{val=false},fail=Lbl}, _) -> + io_lib:format(" ~sbr label ~p\n", [format_i_number(A),Lbl]); +format_terminator(#b_br{anno=A,bool=Bool,succ=Succ,fail=Fail}, FuncAnno) -> + io_lib:format(" ~sbr ~ts, label ~p, label ~p\n", + [format_i_number(A),format_arg(Bool, FuncAnno),Succ,Fail]); +format_terminator(#b_switch{anno=A,arg=Arg,fail=Fail,list=List}, FuncAnno) -> + io_lib:format(" ~sswitch ~ts, label ~p, ~ts\n", + [format_i_number(A),format_arg(Arg, FuncAnno),Fail, + format_list(List,FuncAnno)]); +format_terminator(#b_ret{anno=A,arg=Arg}, FuncAnno) -> + io_lib:format(" ~sret ~ts\n", [format_i_number(A),format_arg(Arg, FuncAnno)]). + +format_op({Prefix,Name}) -> + io_lib:format("~p:~p", [Prefix,Name]); +format_op(Name) -> + io_lib:format("~p", [Name]). + +format_register(#b_var{}=V, #{registers:=Regs}) -> + {Tag,N} = maps:get(V, Regs), + io_lib:format("~p~p", [Tag,N]); +format_register(_, #{}) -> "". + +format_var(Var, FuncAnno) -> + VarString = format_var_1(Var), + case format_register(Var, FuncAnno) of + [] -> VarString; + [_|_]=Reg -> [Reg,$/,VarString] + end. + +format_var_1(#b_var{name={Name,Uniq}}) -> + if + is_atom(Name) -> + io_lib:format("~ts:~p", [Name,Uniq]); + is_integer(Name) -> + io_lib:format("_~p:~p", [Name,Uniq]) + end; +format_var_1(#b_var{name=Name}) when is_atom(Name) -> + atom_to_list(Name); +format_var_1(#b_var{name=Name}) when is_integer(Name) -> + "_"++integer_to_list(Name). + +format_args(Args, FuncAnno) -> + Ss = [format_arg(Arg, FuncAnno) || Arg <- Args], + lists:join(", ", Ss). + +format_arg(#b_var{}=Arg, FuncAnno) -> + format_var(Arg, FuncAnno); +format_arg(#b_literal{val=Val}, _FuncAnno) -> + io_lib:format("literal ~p", [Val]); +format_arg(#b_remote{mod=Mod,name=Name,arity=Arity}, FuncAnno) -> + io_lib:format("remote (~ts):(~ts)/~p", + [format_arg(Mod, FuncAnno),format_arg(Name, FuncAnno),Arity]); +format_arg(#b_local{name=Name,arity=Arity}, FuncAnno) -> + io_lib:format("local ~ts/~p", [format_arg(Name, FuncAnno),Arity]); +format_arg({Value,Label}, FuncAnno) when is_integer(Label) -> + io_lib:format("{ ~ts, ~p }", [format_arg(Value, FuncAnno),Label]); +format_arg(Other, _) -> + io_lib:format("*** ~p ***", [Other]). + +format_list(List, FuncAnno) -> + Ss = [io_lib:format("{ ~ts, ~ts }", [format_arg(Val, FuncAnno),format_label(L)]) || + {Val,L} <- List], + io_lib:format("[ ~ts ]", [lists:join(", ", Ss)]). + +format_label(L) -> + ["label ",integer_to_list(L)]. + +format_anno(#{n:=_}=Anno) -> + format_anno(maps:remove(n, Anno)); +format_anno(#{location:={File,Line}}=Anno0) -> + Anno = maps:remove(location, Anno0), + [io_lib:format(" %% ~ts:~p\n", [File,Line])|format_anno_1(Anno)]; +format_anno(Anno) -> + format_anno_1(Anno). + +format_anno_1(Anno) -> + case map_size(Anno) of + 0 -> + []; + _ -> + [io_lib:format(" %% Anno: ~p\n", [Anno])] + end. + +format_live_interval(#b_var{}=Dst, #{live_intervals:=Intervals}) -> + case Intervals of + #{Dst:=Rs0} -> + Rs1 = [io_lib:format("~p..~p", [Start,End]) || + {Start,End} <- Rs0], + Rs = lists:join(" ", Rs1), + io_lib:format(" %% ~ts: ~s\n", [format_var_1(Dst),Rs]); + #{} -> + [] + end; +format_live_interval(_, _) -> []. + diff --git a/lib/compiler/src/beam_ssa_pre_codegen.erl b/lib/compiler/src/beam_ssa_pre_codegen.erl new file mode 100644 index 0000000000..bf99e8fc26 --- /dev/null +++ b/lib/compiler/src/beam_ssa_pre_codegen.erl @@ -0,0 +1,2850 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% Purpose: Prepare for code generation, including register allocation. +%% +%% The output of this compiler pass is still in the SSA format, but +%% it has been annotated and transformed to help the code generator. +%% +%% * Some instructions are translated to other instructions closer to +%% the BEAM instructions. For example, the binary matching +%% instructions are transformed from the optimization-friendly +%% internal format to instruction more similar to the actual BEAM +%% instructions. +%% +%% * Blocks that will need an instruction for allocating a stack frame +%% are annotated with a {frame_size,Size} annotation. +%% +%% * 'copy' instructions are added for all variables that need +%% to be saved to the stack frame. Additional 'copy' instructions +%% can be added as an optimization to reuse y registers (see +%% the copy_retval sub pass). +%% +%% * Each function is annotated with a {register,RegisterMap} +%% annotation that maps each variable to a BEAM register. The linear +%% scan algorithm is used to allocate registers. +%% +%% There are four kind of registers. x, y, fr (floating point register), +%% and z. A variable will be allocated to a z register if it is only +%% used by the instruction following the instruction that defines the +%% the variable. The code generator will typically combine those +%% instructions to a test instruction. z registers are also used for +%% some instructions that don't have a return value. +%% +%% References: +%% +%% [1] H. Mössenböck and M. Pfeiffer. Linear scan register allocation +%% in the context of SSA form and register constraints. In Proceedings +%% of the International Conference on Compiler Construction, pages +%% 229–246. LNCS 2304, Springer-Verlag, 2002. +%% +%% [2] C. Wimmer and H. Mössenböck. Optimized interval splitting in a +%% linear scan register allocator. In Proceedings of the ACM/USENIX +%% International Conference on Virtual Execution Environments, pages +%% 132–141. ACM Press, 2005. +%% +%% [3] C. Wimmer and M. Franz. Linear Scan Register Allocation on SSA +%% Form. In Proceedings of the International Symposium on Code +%% Generation and Optimization, pages 170-179. ACM Press, 2010. +%% + +-module(beam_ssa_pre_codegen). + +-export([module/2]). + +-include("beam_ssa.hrl"). + +-import(lists, [all/2,any/2,append/1,duplicate/2, + foldl/3,last/1,map/2,member/2,partition/2, + reverse/1,reverse/2,sort/1,splitwith/2,zip/2]). + +-spec module(beam_ssa:b_module(), [compile:option()]) -> + {'ok',beam_ssa:b_module()}. + +module(#b_module{body=Fs0}=Module, Opts) -> + UseBSM3 = not proplists:get_bool(no_bsm3, Opts), + Ps = passes(Opts), + Fs = functions(Fs0, Ps, UseBSM3), + {ok,Module#b_module{body=Fs}}. + +functions([F|Fs], Ps, UseBSM3) -> + [function(F, Ps, UseBSM3)|functions(Fs, Ps, UseBSM3)]; +functions([], _Ps, _UseBSM3) -> []. + +-type b_var() :: beam_ssa:b_var(). +-type var_name() :: beam_ssa:var_name(). +-type instr_number() :: pos_integer(). +-type range() :: {instr_number(),instr_number()}. +-type reg_num() :: beam_asm:reg_num(). +-type xreg() :: {'x',reg_num()}. +-type yreg() :: {'y',reg_num()}. +-type ypool() :: {'y',beam_ssa:label()}. +-type reservation() :: 'fr' | {'prefer',xreg()} | 'x' | {'x',xreg()} | + ypool() | {yreg(),ypool()} | 'z'. +-type ssa_register() :: beam_ssa_codegen:ssa_register(). + +-define(TC(Body), tc(fun() -> Body end, ?FILE, ?LINE)). +-record(st, {ssa :: beam_ssa:block_map(), + args :: [b_var()], + cnt :: beam_ssa:label(), + use_bsm3 :: boolean(), + frames=[] :: [beam_ssa:label()], + intervals=[] :: [{b_var(),[range()]}], + res=[] :: [{b_var(),reservation()}] | #{b_var():=reservation()}, + regs=#{} :: #{b_var():=ssa_register()}, + extra_annos=[] :: [{atom(),term()}] + }). +-define(PASS(N), {N,fun N/1}). + +passes(Opts) -> + AddPrecgAnnos = proplists:get_bool(dprecg, Opts), + FixTuples = proplists:get_bool(no_put_tuple2, Opts), + Ps = [?PASS(assert_no_critical_edges), + + %% Preliminaries. + ?PASS(fix_bs), + ?PASS(sanitize), + case FixTuples of + false -> ignore; + true -> ?PASS(fix_tuples) + end, + ?PASS(use_set_tuple_element), + ?PASS(place_frames), + ?PASS(fix_receives), + + %% Find and reserve Y registers. + ?PASS(find_yregs), + ?PASS(reserve_yregs), + + %% Handle legacy binary match instruction that don't + %% accept a Y register as destination. + ?PASS(legacy_bs), + + %% Improve reuse of Y registers to potentially + %% reduce the size of the stack frame. + ?PASS(copy_retval), + ?PASS(opt_get_list), + + %% Calculate live intervals. + ?PASS(number_instructions), + ?PASS(live_intervals), + ?PASS(reserve_regs), + + %% If needed for a .precg file, save the live intervals + %% so they can be included in an annotation. + case AddPrecgAnnos of + false -> ignore; + true -> ?PASS(save_live_intervals) + end, + + %% Allocate registers. + ?PASS(linear_scan), + ?PASS(frame_size), + ?PASS(turn_yregs)], + [P || P <- Ps, P =/= ignore]. + +function(#b_function{anno=Anno,args=Args,bs=Blocks0,cnt=Count0}=F0, + Ps, UseBSM3) -> + try + St0 = #st{ssa=Blocks0,args=Args,use_bsm3=UseBSM3,cnt=Count0}, + St = compile:run_sub_passes(Ps, St0), + #st{ssa=Blocks,cnt=Count,regs=Regs,extra_annos=ExtraAnnos} = St, + F1 = add_extra_annos(F0, ExtraAnnos), + F = beam_ssa:add_anno(registers, Regs, F1), + F#b_function{bs=Blocks,cnt=Count} + catch + Class:Error:Stack -> + #{func_info:={_,Name,Arity}} = Anno, + io:fwrite("Function: ~w/~w\n", [Name,Arity]), + erlang:raise(Class, Error, Stack) + end. + +save_live_intervals(#st{intervals=Intervals}=St) -> + St#st{extra_annos=[{live_intervals,Intervals}]}. + +%% Add extra annotations when a .precg listing file is being produced. +add_extra_annos(F, Annos) -> + foldl(fun({Name,Value}, Acc) -> + beam_ssa:add_anno(Name, Value, Acc) + end, F, Annos). + +%% assert_no_critical_edges(St0) -> St. +%% The code generator will not work if there are critial edges. +%% Abort if any critical edges are found. + +assert_no_critical_edges(#st{ssa=Blocks}=St) -> + F = fun assert_no_ces/3, + beam_ssa:fold_rpo(F, Blocks, Blocks), + St. + +assert_no_ces(_, #b_blk{is=[#b_set{op=phi,args=[_,_]=Phis}|_]}, Blocks) -> + %% This block has multiple predecessors. Make sure that none + %% of the precessors have more than one successor. + true = all(fun({_,P}) -> + length(beam_ssa:successors(P, Blocks)) =:= 1 + end, Phis), %Assertion. + Blocks; +assert_no_ces(_, _, Blocks) -> Blocks. + +%% fix_bs(St0) -> St. +%% Fix up the binary matching instructions: +%% +%% * Insert bs_save and bs_restore instructions where needed. +%% +%% * Combine bs_match and bs_extract instructions to bs_get +%% instructions. + +fix_bs(#st{ssa=Blocks,cnt=Count0,use_bsm3=UseBSM3}=St) -> + F = fun(#b_set{op=bs_start_match,dst=Dst}, A) -> + %% Mark the root of the match context list. + [{Dst,{context,Dst}}|A]; + (#b_set{op=bs_match,dst=Dst,args=[_,ParentCtx|_]}, A) -> + %% Link this match context the previous match context. + [{Dst,ParentCtx}|A]; + (_, A) -> + A + end, + case beam_ssa:fold_instrs_rpo(F, [0], [],Blocks) of + [] -> + %% No binary matching in this function. + St; + [_|_]=M -> + CtxChain = maps:from_list(M), + Linear0 = beam_ssa:linearize(Blocks), + + %% Insert position instructions where needed. + {Linear1,Count} = case UseBSM3 of + true -> + bs_pos_bsm3(Linear0, CtxChain, Count0); + false -> + bs_pos_bsm2(Linear0, CtxChain, Count0) + end, + + %% Rename instructions. + Linear = bs_instrs(Linear1, CtxChain, []), + + St#st{ssa=maps:from_list(Linear),cnt=Count} + end. + +%% Insert bs_get_position and bs_set_position instructions as needed. +bs_pos_bsm3(Linear0, CtxChain, Count0) -> + Rs0 = bs_restores(Linear0, CtxChain, #{}, #{}), + Rs = maps:values(Rs0), + S0 = sofs:relation(Rs, [{context,save_point}]), + S1 = sofs:relation_to_family(S0), + S = sofs:to_external(S1), + + {SavePoints,Count1} = make_bs_pos_dict(S, Count0, []), + {Gets,Count2} = make_bs_setpos_map(Rs, SavePoints, Count1, []), + {Sets,Count} = make_bs_getpos_map(maps:to_list(Rs0), SavePoints, Count2, []), + + %% Now insert all saves and restores. + {bs_insert_bsm3(Linear0, Gets, Sets, SavePoints),Count}. + +make_bs_setpos_map([{Ctx,Save}=Ps|T], SavePoints, Count, Acc) -> + SavePoint = get_savepoint(Ps, SavePoints), + I = #b_set{op=bs_get_position,dst=SavePoint,args=[Ctx]}, + make_bs_setpos_map(T, SavePoints, Count+1, [{Save,I}|Acc]); +make_bs_setpos_map([], _, Count, Acc) -> + {maps:from_list(Acc),Count}. + +make_bs_getpos_map([{Bef,{Ctx,_}=Ps}|T], SavePoints, Count, Acc) -> + Ignored = #b_var{name={'@ssa_ignored',Count}}, + Args = [Ctx, get_savepoint(Ps, SavePoints)], + I = #b_set{op=bs_set_position,dst=Ignored,args=Args}, + make_bs_getpos_map(T, SavePoints, Count+1, [{Bef,I}|Acc]); +make_bs_getpos_map([], _, Count, Acc) -> + {maps:from_list(Acc),Count}. + +get_savepoint({_,_}=Ps, SavePoints) -> + Name = {'@ssa_bs_position', map_get(Ps, SavePoints)}, + #b_var{name=Name}. + +make_bs_pos_dict([{Ctx,Pts}|T], Count0, Acc0) -> + {Acc, Count} = make_bs_pos_dict_1(Pts, Ctx, Count0, Acc0), + make_bs_pos_dict(T, Count, Acc); +make_bs_pos_dict([], Count, Acc) -> + {maps:from_list(Acc), Count}. + +make_bs_pos_dict_1([H|T], Ctx, I, Acc) -> + make_bs_pos_dict_1(T, Ctx, I+1, [{{Ctx,H},I}|Acc]); +make_bs_pos_dict_1([], Ctx, I, Acc) -> + {[{Ctx,I}|Acc], I}. + +%% As bs_position but without OTP-22 instructions. This is only used when +%% cross-compiling to older versions. +bs_pos_bsm2(Linear0, CtxChain, Count0) -> + Rs0 = bs_restores(Linear0, CtxChain, #{}, #{}), + Rs = maps:values(Rs0), + S0 = sofs:relation(Rs, [{context,save_point}]), + S1 = sofs:relation_to_family(S0), + S = sofs:to_external(S1), + Slots = make_save_point_dict(S, []), + {Saves,Count1} = make_save_map(Rs, Slots, Count0, []), + {Restores,Count} = make_restore_map(maps:to_list(Rs0), Slots, Count1, []), + + %% Now insert all saves and restores. + {bs_insert_bsm2(Linear0, Saves, Restores, Slots),Count}. + +make_save_map([{Ctx,Save}=Ps|T], Slots, Count, Acc) -> + Ignored = #b_var{name={'@ssa_ignored',Count}}, + case make_slot(Ps, Slots) of + #b_literal{val=start} -> + make_save_map(T, Slots, Count, Acc); + Slot -> + I = #b_set{op=bs_save,dst=Ignored,args=[Ctx,Slot]}, + make_save_map(T, Slots, Count+1, [{Save,I}|Acc]) + end; +make_save_map([], _, Count, Acc) -> + {maps:from_list(Acc),Count}. + +make_restore_map([{Bef,{Ctx,_}=Ps}|T], Slots, Count, Acc) -> + Ignored = #b_var{name={'@ssa_ignored',Count}}, + I = #b_set{op=bs_restore,dst=Ignored,args=[Ctx,make_slot(Ps, Slots)]}, + make_restore_map(T, Slots, Count+1, [{Bef,I}|Acc]); +make_restore_map([], _, Count, Acc) -> + {maps:from_list(Acc),Count}. + +make_slot({Same,Same}, _Slots) -> + #b_literal{val=start}; +make_slot({_,_}=Ps, Slots) -> + #b_literal{val=map_get(Ps, Slots)}. + +make_save_point_dict([{Ctx,Pts}|T], Acc0) -> + Acc = make_save_point_dict_1(Pts, Ctx, 0, Acc0), + make_save_point_dict(T, Acc); +make_save_point_dict([], Acc) -> + maps:from_list(Acc). + +make_save_point_dict_1([Ctx|T], Ctx, I, Acc) -> + %% Special {atom,start} save point. Does not need a + %% bs_save instruction. + make_save_point_dict_1(T, Ctx, I, Acc); +make_save_point_dict_1([H|T], Ctx, I, Acc) -> + make_save_point_dict_1(T, Ctx, I+1, [{{Ctx,H},I}|Acc]); +make_save_point_dict_1([], Ctx, I, Acc) -> + [{Ctx,I}|Acc]. + +bs_restores([{L,#b_blk{is=Is,last=Last}}|Bs], CtxChain, D0, Rs0) -> + InPos = maps:get(L, D0, #{}), + {SuccPos, FailPos, Rs} = bs_restores_is(Is, CtxChain, InPos, InPos, Rs0), + + D = bs_update_successors(Last, SuccPos, FailPos, D0), + bs_restores(Bs, CtxChain, D, Rs); +bs_restores([], _, _, Rs) -> Rs. + +bs_update_successors(#b_br{succ=Succ,fail=Fail}, SPos, FPos, D) -> + join_positions([{Succ,SPos},{Fail,FPos}], D); +bs_update_successors(#b_switch{fail=Fail,list=List}, SPos, FPos, D) -> + SPos = FPos, %Assertion. + Update = [{L,SPos} || {_,L} <- List] ++ [{Fail,SPos}], + join_positions(Update, D); +bs_update_successors(#b_ret{}, SPos, FPos, D) -> + SPos = FPos, %Assertion. + D. + +join_positions([{L,MapPos0}|T], D) -> + case D of + #{L:=MapPos0} -> + %% Same map. + join_positions(T, D); + #{L:=MapPos1} -> + %% Different maps. + MapPos = join_positions_1(MapPos0, MapPos1), + join_positions(T, D#{L:=MapPos}); + #{} -> + join_positions(T, D#{L=>MapPos0}) + end; +join_positions([], D) -> D. + +join_positions_1(MapPos0, MapPos1) -> + MapPos2 = maps:map(fun(Start, Pos) -> + case MapPos0 of + #{Start:=Pos} -> Pos; + #{Start:=_} -> unknown; + #{} -> Pos + end + end, MapPos1), + maps:merge(MapPos0, MapPos2). + +%% +%% Updates the restore and position maps according to the given instructions. +%% +%% Note that positions may be updated even when a match fails; if a match +%% requires a restore, the position at the fail block will be the position +%% we've *restored to* and not the one we entered the current block with. +%% + +bs_restores_is([#b_set{op=bs_start_match,dst=Start}|Is], + CtxChain, SPos0, FPos, Rs) -> + %% We only allow one match per block. + SPos0 = FPos, %Assertion. + SPos = SPos0#{Start=>Start}, + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); +bs_restores_is([#b_set{op=bs_match,dst=NewPos,args=Args}=I|Is], + CtxChain, SPos0, FPos0, Rs0) -> + SPos0 = FPos0, %Assertion. + Start = bs_subst_ctx(NewPos, CtxChain), + [_,FromPos|_] = Args, + case SPos0 of + #{Start:=FromPos} -> + %% Same position, no restore needed. + SPos = case bs_match_type(I) of + plain -> + %% Update position to new position. + SPos0#{Start:=NewPos}; + _ -> + %% Position will not change (test_unit + %% instruction or no instruction at + %% all). + SPos0#{Start:=FromPos} + end, + bs_restores_is(Is, CtxChain, SPos, FPos0, Rs0); + #{Start:=_} -> + %% Different positions, might need a restore instruction. + case bs_match_type(I) of + none -> + %% This is a tail test that will be optimized away. + %% There's no need to do a restore, and all + %% positions are unchanged. + bs_restores_is(Is, CtxChain, SPos0, FPos0, Rs0); + test_unit -> + %% This match instruction will be replaced by + %% a test_unit instruction. We will need a + %% restore. The new position will be the position + %% restored to (NOT NewPos). + SPos = SPos0#{Start:=FromPos}, + FPos = FPos0#{Start:=FromPos}, + Rs = Rs0#{NewPos=>{Start,FromPos}}, + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); + plain -> + %% Match or skip. Position will be changed. + SPos = SPos0#{Start:=NewPos}, + FPos = FPos0#{Start:=FromPos}, + Rs = Rs0#{NewPos=>{Start,FromPos}}, + bs_restores_is(Is, CtxChain, SPos, FPos, Rs) + end + end; +bs_restores_is([#b_set{op=bs_extract,args=[FromPos|_]}|Is], + CtxChain, SPos, FPos, Rs) -> + Start = bs_subst_ctx(FromPos, CtxChain), + #{Start:=FromPos} = SPos, %Assertion. + #{Start:=FromPos} = FPos, %Assertion. + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); +bs_restores_is([#b_set{op=call,dst=Dst,args=Args}|Is], + CtxChain, SPos0, FPos0, Rs0) -> + {Rs, SPos1, FPos1} = bs_restore_args(Args, SPos0, FPos0, CtxChain, Dst, Rs0), + {SPos, FPos} = bs_invalidate_pos(Args, SPos1, FPos1, CtxChain), + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); +bs_restores_is([#b_set{op=landingpad}|Is], CtxChain, SPos0, FPos0, Rs) -> + %% We can land here from any point, so all positions are invalid. + Invalidate = fun(_Start,_Pos) -> unknown end, + SPos = maps:map(Invalidate, SPos0), + FPos = maps:map(Invalidate, FPos0), + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); +bs_restores_is([#b_set{op=Op,dst=Dst,args=Args}|Is], + CtxChain, SPos0, FPos0, Rs0) + when Op =:= bs_test_tail; + Op =:= bs_get_tail -> + {Rs, SPos, FPos} = bs_restore_args(Args, SPos0, FPos0, CtxChain, Dst, Rs0), + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); +bs_restores_is([_|Is], CtxChain, SPos, FPos, Rs) -> + bs_restores_is(Is, CtxChain, SPos, FPos, Rs); +bs_restores_is([], _CtxChain, SPos, FPos, Rs) -> + {SPos, FPos, Rs}. + +bs_match_type(#b_set{args=[#b_literal{val=skip},_Ctx, + #b_literal{val=binary},_Flags, + #b_literal{val=all},#b_literal{val=U}]}) -> + case U of + 1 -> none; + _ -> test_unit + end; +bs_match_type(_) -> + plain. + +%% Call instructions leave the match position in an undefined state, +%% requiring us to invalidate each affected argument. +bs_invalidate_pos([#b_var{}=Arg|Args], SPos0, FPos0, CtxChain) -> + Start = bs_subst_ctx(Arg, CtxChain), + case SPos0 of + #{Start:=_} -> + SPos = SPos0#{Start:=unknown}, + FPos = FPos0#{Start:=unknown}, + bs_invalidate_pos(Args, SPos, FPos, CtxChain); + #{} -> + %% Not a match context. + bs_invalidate_pos(Args, SPos0, FPos0, CtxChain) + end; +bs_invalidate_pos([_|Args], SPos, FPos, CtxChain) -> + bs_invalidate_pos(Args, SPos, FPos, CtxChain); +bs_invalidate_pos([], SPos, FPos, _CtxChain) -> + {SPos, FPos}. + +bs_restore_args([#b_var{}=Arg|Args], SPos0, FPos0, CtxChain, Dst, Rs0) -> + Start = bs_subst_ctx(Arg, CtxChain), + case SPos0 of + #{Start:=Arg} -> + %% Same position, no restore needed. + bs_restore_args(Args, SPos0, FPos0, CtxChain, Dst, Rs0); + #{Start:=_} -> + %% Different positions, need a restore instruction. + SPos = SPos0#{Start:=Arg}, + FPos = FPos0#{Start:=Arg}, + Rs = Rs0#{Dst=>{Start,Arg}}, + bs_restore_args(Args, SPos, FPos, CtxChain, Dst, Rs); + #{} -> + %% Not a match context. + bs_restore_args(Args, SPos0, FPos0, CtxChain, Dst, Rs0) + end; +bs_restore_args([_|Args], SPos, FPos, CtxChain, Dst, Rs) -> + bs_restore_args(Args, SPos, FPos, CtxChain, Dst, Rs); +bs_restore_args([], SPos, FPos, _CtxChain, _Dst, Rs) -> + {Rs,SPos,FPos}. + +%% Insert all bs_save and bs_restore instructions. + +bs_insert_bsm3(Blocks, Saves, Restores, SavePoints) -> + bs_insert_1(Blocks, Saves, Restores, SavePoints, fun(I) -> I end). + +bs_insert_bsm2(Blocks, Saves, Restores, SavePoints) -> + %% The old instructions require bs_start_match to be annotated with the + %% number of position slots it needs. + bs_insert_1(Blocks, Saves, Restores, SavePoints, + fun(#b_set{op=bs_start_match,dst=Dst}=I0) -> + NumSlots = case SavePoints of + #{Dst:=NumSlots0} -> NumSlots0; + #{} -> 0 + end, + beam_ssa:add_anno(num_slots, NumSlots, I0); + (I) -> + I + end). + +bs_insert_1([{L,#b_blk{is=Is0}=Blk}|Bs0], Saves, Restores, Slots, XFrm) -> + Is = bs_insert_is_1(Is0, Restores, Slots, XFrm), + Bs = bs_insert_saves(Is, Bs0, Saves), + [{L,Blk#b_blk{is=Is}}|bs_insert_1(Bs, Saves, Restores, Slots, XFrm)]; +bs_insert_1([], _, _, _, _) -> []. + +bs_insert_is_1([#b_set{op=Op,dst=Dst}=I0|Is], Restores, SavePoints, XFrm) -> + I = XFrm(I0), + if + Op =:= bs_test_tail; + Op =:= bs_get_tail; + Op =:= bs_match; + Op =:= call -> + Rs = case Restores of + #{Dst:=R} -> [R]; + #{} -> [] + end, + Rs ++ [I|bs_insert_is_1(Is, Restores, SavePoints, XFrm)]; + true -> + [I|bs_insert_is_1(Is, Restores, SavePoints, XFrm)] + end; +bs_insert_is_1([], _, _, _) -> []. + +bs_insert_saves([#b_set{dst=Dst}|Is], Bs, Saves) -> + case Saves of + #{Dst:=S} -> + bs_insert_save(S, Bs); + #{} -> + bs_insert_saves(Is, Bs, Saves) + end; +bs_insert_saves([], Bs, _) -> Bs. + +bs_insert_save(Save, [{L,#b_blk{is=Is0}=Blk}|Bs]) -> + Is = case Is0 of + [#b_set{op=bs_extract}=Ex|Is1] -> + [Ex,Save|Is1]; + _ -> + [Save|Is0] + end, + [{L,Blk#b_blk{is=Is}}|Bs]. + +%% Translate bs_match instructions to bs_get, bs_match_string, +%% or bs_skip. Also rename match context variables to use the +%% variable assigned to by the start_match instruction. + +bs_instrs([{L,#b_blk{is=Is0}=Blk}|Bs], CtxChain, Acc0) -> + case bs_instrs_is(Is0, CtxChain, []) of + [#b_set{op=bs_extract,dst=Dst,args=[Ctx]}|Is] -> + %% Drop this instruction. Rewrite the corresponding + %% bs_match instruction in the previous block to + %% a bs_get instruction. + Acc = bs_combine(Dst, Ctx, Acc0), + bs_instrs(Bs, CtxChain, [{L,Blk#b_blk{is=Is}}|Acc]); + Is -> + bs_instrs(Bs, CtxChain, [{L,Blk#b_blk{is=Is}}|Acc0]) + end; +bs_instrs([], _, Acc) -> + reverse(Acc). + +bs_instrs_is([#b_set{op=Op,args=Args0}=I0|Is], CtxChain, Acc) -> + Args = [bs_subst_ctx(A, CtxChain) || A <- Args0], + I1 = I0#b_set{args=Args}, + I = case {Op,Args} of + {bs_match,[#b_literal{val=skip},Ctx,Type|As]} -> + I1#b_set{op=bs_skip,args=[Type,Ctx|As]}; + {bs_match,[#b_literal{val=string},Ctx|As]} -> + I1#b_set{op=bs_match_string,args=[Ctx|As]}; + {bs_get_tail,[Ctx|As]} -> + I1#b_set{op=bs_get_tail,args=[Ctx|As]}; + {_,_} -> + I1 + end, + bs_instrs_is(Is, CtxChain, [I|Acc]); +bs_instrs_is([], _, Acc) -> + reverse(Acc). + +%% Combine a bs_match instruction with the destination register +%% taken from a bs_extract instruction. + +bs_combine(Dst, Ctx, [{L,#b_blk{is=Is0}=Blk}|Acc]) -> + [#b_set{}=Succeeded, + #b_set{op=bs_match,args=[Type,_|As]}=BsMatch|Is1] = reverse(Is0), + Is = reverse(Is1, [BsMatch#b_set{op=bs_get,dst=Dst,args=[Type,Ctx|As]}, + Succeeded#b_set{args=[Dst]}]), + [{L,Blk#b_blk{is=Is}}|Acc]. + +bs_subst_ctx(#b_var{}=Var, CtxChain) -> + case CtxChain of + #{Var:={context,Ctx}} -> + Ctx; + #{Var:=ParentCtx} -> + bs_subst_ctx(ParentCtx, CtxChain); + #{} -> + %% Not a match context variable. + Var + end; +bs_subst_ctx(Other, _CtxChain) -> + Other. + +%% legacy_bs(St0) -> St. +%% Binary matching instructions in OTP 21 and earlier don't support +%% a Y register as destination. If St#st.use_bsm3 is false, +%% we will need to rewrite those instructions so that the result +%% is first put in an X register and then moved to a Y register +%% if the operation succeeded. + +legacy_bs(#st{use_bsm3=false,ssa=Blocks0,cnt=Count0,res=Res}=St) -> + IsYreg = maps:from_list([{V,true} || {V,{y,_}} <- Res]), + Linear0 = beam_ssa:linearize(Blocks0), + {Linear,Count} = legacy_bs(Linear0, IsYreg, Count0, #{}, []), + Blocks = maps:from_list(Linear), + St#st{ssa=Blocks,cnt=Count}; +legacy_bs(#st{use_bsm3=true}=St) -> St. + +legacy_bs([{L,Blk}|Bs], IsYreg, Count0, Copies0, Acc) -> + #b_blk{is=Is0,last=Last} = Blk, + Is1 = case Copies0 of + #{L:=Copy} -> [Copy|Is0]; + #{} -> Is0 + end, + {Is,Count,Copies} = legacy_bs_is(Is1, Last, IsYreg, Count0, Copies0, []), + legacy_bs(Bs, IsYreg, Count, Copies, [{L,Blk#b_blk{is=Is}}|Acc]); +legacy_bs([], _IsYreg, Count, _Copies, Acc) -> + {Acc,Count}. + +legacy_bs_is([#b_set{op=Op,dst=Dst}=I0, + #b_set{op=succeeded,dst=SuccDst,args=[Dst]}=SuccI0], + Last, IsYreg, Count0, Copies0, Acc) -> + NeedsFix = is_map_key(Dst, IsYreg) andalso + case Op of + bs_get -> true; + bs_init -> true; + _ -> false + end, + case NeedsFix of + true -> + TempDst = #b_var{name={'@bs_temp_dst',Count0}}, + Count = Count0 + 1, + I = I0#b_set{dst=TempDst}, + SuccI = SuccI0#b_set{args=[TempDst]}, + Copy = #b_set{op=copy,dst=Dst,args=[TempDst]}, + #b_br{bool=SuccDst,succ=SuccL} = Last, + Copies = Copies0#{SuccL=>Copy}, + legacy_bs_is([], Last, IsYreg, Count, Copies, [SuccI,I|Acc]); + false -> + legacy_bs_is([], Last, IsYreg, Count0, Copies0, [SuccI0,I0|Acc]) + end; +legacy_bs_is([I|Is], Last, IsYreg, Count, Copies, Acc) -> + legacy_bs_is(Is, Last, IsYreg, Count, Copies, [I|Acc]); +legacy_bs_is([], _Last, _IsYreg, Count, Copies, Acc) -> + {reverse(Acc),Count,Copies}. + +%% sanitize(St0) -> St. +%% Remove constructs that can cause problems later: +%% +%% * Unreachable blocks may cause problems for determination of +%% dominators. +%% +%% * Some instructions (such as get_hd) don't accept literal +%% arguments. Evaluate the instructions and remove them. + +sanitize(#st{ssa=Blocks0,cnt=Count0}=St) -> + Ls = beam_ssa:rpo(Blocks0), + {Blocks,Count} = sanitize(Ls, Count0, Blocks0, #{}), + St#st{ssa=Blocks,cnt=Count}. + +sanitize([L|Ls], Count0, Blocks0, Values0) -> + #b_blk{is=Is0} = Blk0 = map_get(L, Blocks0), + case sanitize_is(Is0, Count0, Values0, false, []) of + no_change -> + sanitize(Ls, Count0, Blocks0, Values0); + {Is,Count,Values} -> + Blk = Blk0#b_blk{is=Is}, + Blocks = Blocks0#{L:=Blk}, + sanitize(Ls, Count, Blocks, Values) + end; +sanitize([], Count, Blocks0, Values) -> + Blocks = if + map_size(Values) =:= 0 -> + Blocks0; + true -> + beam_ssa:rename_vars(Values, [0], Blocks0) + end, + + %% Unreachable blocks can cause problems for the dominator calculations. + Ls = beam_ssa:rpo(Blocks), + Reachable = gb_sets:from_list(Ls), + {case map_size(Blocks) =:= gb_sets:size(Reachable) of + true -> Blocks; + false -> remove_unreachable(Ls, Blocks, Reachable, []) + end,Count}. + +sanitize_is([#b_set{op=get_map_element,args=Args0}=I0|Is], + Count0, Values, Changed, Acc) -> + case sanitize_args(Args0, Values) of + [#b_literal{}=Map,Key] -> + %% Bind the literal map to a variable. + {MapVar,Count} = new_var('@ssa_map', Count0), + I = I0#b_set{args=[MapVar,Key]}, + Copy = #b_set{op=copy,dst=MapVar,args=[Map]}, + sanitize_is(Is, Count, Values, true, [I,Copy|Acc]); + [_,_]=Args0 -> + sanitize_is(Is, Count0, Values, Changed, [I0|Acc]); + [_,_]=Args -> + I = I0#b_set{args=Args}, + sanitize_is(Is, Count0, Values, Changed, [I|Acc]) + end; +sanitize_is([#b_set{op=Op,dst=Dst,args=Args0}=I0|Is0], + Count, Values, Changed0, Acc) -> + Args = sanitize_args(Args0, Values), + case sanitize_instr(Op, Args, I0) of + {value,Value0} -> + Value = #b_literal{val=Value0}, + sanitize_is(Is0, Count, Values#{Dst=>Value}, true, Acc); + {ok,I} -> + sanitize_is(Is0, Count, Values, true, [I|Acc]); + ok -> + I = I0#b_set{args=Args}, + Changed = Changed0 orelse Args =/= Args0, + sanitize_is(Is0, Count, Values, Changed, [I|Acc]) + end; +sanitize_is([], Count, Values, Changed, Acc) -> + case Changed of + true -> + {reverse(Acc),Count,Values}; + false -> + no_change + end. + +sanitize_args(Args, Values) -> + map(fun(Var) -> + case Values of + #{Var:=New} -> New; + #{} -> Var + end + end, Args). + +sanitize_instr({bif,Bif}, [#b_literal{val=Lit}], _I) -> + case erl_bifs:is_pure(erlang, Bif, 1) of + false -> + ok; + true -> + try + {value,erlang:Bif(Lit)} + catch + error:_ -> + ok + end + end; +sanitize_instr({bif,Bif}, [#b_literal{val=Lit1},#b_literal{val=Lit2}], _I) -> + true = erl_bifs:is_pure(erlang, Bif, 2), %Assertion. + try + {value,erlang:Bif(Lit1, Lit2)} + catch + error:_ -> + ok + end; +sanitize_instr(get_hd, [#b_literal{val=[Hd|_]}], _I) -> + {value,Hd}; +sanitize_instr(get_tl, [#b_literal{val=[_|Tl]}], _I) -> + {value,Tl}; +sanitize_instr(get_tuple_element, [#b_literal{val=T}, + #b_literal{val=I}], _I) + when I < tuple_size(T) -> + {value,element(I+1, T)}; +sanitize_instr(is_nonempty_list, [#b_literal{val=Lit}], _I) -> + {value,case Lit of + [_|_] -> true; + _ -> false + end}; +sanitize_instr(is_tagged_tuple, [#b_literal{val=Tuple}, + #b_literal{val=Arity}, + #b_literal{val=Tag}], _I) + when is_integer(Arity), is_atom(Tag) -> + if + tuple_size(Tuple) =:= Arity, element(1, Tuple) =:= Tag -> + {value,true}; + true -> + {value,false} + end; +sanitize_instr(bs_init, [#b_literal{val=new},#b_literal{val=Sz}|_], I0) -> + if + is_integer(Sz), Sz >= 0 -> ok; + true -> {ok,sanitize_badarg(I0)} + end; +sanitize_instr(bs_init, [#b_literal{val=append},_,#b_literal{val=Sz}|_], I0) -> + if + is_integer(Sz), Sz >= 0 -> ok; + true -> {ok,sanitize_badarg(I0)} + end; +sanitize_instr(succeeded, [#b_literal{}], _I) -> + {value,true}; +sanitize_instr(_, _, _) -> ok. + +sanitize_badarg(I) -> + Func = #b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=error},arity=1}, + I#b_set{op=call,args=[Func,#b_literal{val=badarg}]}. + +remove_unreachable([L|Ls], Blocks, Reachable, Acc) -> + #b_blk{is=Is0} = Blk0 = map_get(L, Blocks), + case split_phis(Is0) of + {[_|_]=Phis,Rest} -> + Is = [prune_phi(Phi, Reachable) || Phi <- Phis] ++ Rest, + Blk = Blk0#b_blk{is=Is}, + remove_unreachable(Ls, Blocks, Reachable, [{L,Blk}|Acc]); + {[],_} -> + remove_unreachable(Ls, Blocks, Reachable, [{L,Blk0}|Acc]) + end; +remove_unreachable([], _Blocks, _, Acc) -> + maps:from_list(Acc). + +prune_phi(#b_set{args=Args0}=Phi, Reachable) -> + Args = [A || {_,Pred}=A <- Args0, + gb_sets:is_element(Pred, Reachable)], + Phi#b_set{args=Args}. + +%%% +%%% Fix tuples. +%%% + +%% fix_tuples(St0) -> St. +%% If compatibility with a previous version of Erlang has been +%% requested, tuple creation must be split into two instruction to +%% mirror the the way tuples are created in BEAM prior to OTP 22. +%% Each put_tuple instruction is split into put_tuple_arity followed +%% by put_tuple_elements. + +fix_tuples(#st{ssa=Blocks0,cnt=Count0}=St) -> + F = fun (#b_set{op=put_tuple,args=Args}=Put, C0) -> + Arity = #b_literal{val=length(Args)}, + {Ignore,C} = new_var('@ssa_ignore', C0), + {[Put#b_set{op=put_tuple_arity,args=[Arity]}, + #b_set{dst=Ignore,op=put_tuple_elements,args=Args}],C}; + (I, C) -> {[I],C} + end, + {Blocks,Count} = beam_ssa:flatmapfold_instrs_rpo(F, [0], Count0, Blocks0), + St#st{ssa=Blocks,cnt=Count}. + +%%% +%%% Introduce the set_tuple_element instructions to make +%%% multiple-field record updates faster. +%%% +%%% The expansion of record field updates, when more than one field is +%%% updated, but not a majority of the fields, will create a sequence of +%%% calls to `erlang:setelement(Index, Value, Tuple)` where Tuple in the +%%% first call is the original record tuple, and in the subsequent calls +%%% Tuple is the result of the previous call. Furthermore, all Index +%%% values are constant positive integers, and the first call to +%%% `setelement` will have the greatest index. Thus all the following +%%% calls do not actually need to test at run-time whether Tuple has type +%%% tuple, nor that the index is within the tuple bounds. +%%% +%%% Since this optimization introduces destructive updates, it used to +%%% be done as the very last Core Erlang pass before going to +%%% lower-level code. However, it turns out that this kind of destructive +%%% updates are awkward also in SSA code and can prevent or complicate +%%% type analysis and aggressive optimizations. +%%% +%%% NOTE: Because there no write barriers in the system, this kind of +%%% optimization can only be done when we are sure that garbage +%%% collection will not be triggered between the creation of the tuple +%%% and the destructive updates - otherwise we might insert pointers +%%% from an older generation to a newer. +%%% + +use_set_tuple_element(#st{ssa=Blocks0}=St) -> + Uses = count_uses(Blocks0), + RPO = reverse(beam_ssa:rpo(Blocks0)), + Blocks = use_ste_1(RPO, Uses, Blocks0), + St#st{ssa=Blocks}. + +use_ste_1([L|Ls], Uses, Blocks0) -> + {Blk0,Blocks} = use_ste_across(L, Uses, Blocks0), + #b_blk{is=Is0} = Blk0, + case use_ste_is(Is0, Uses) of + Is0 -> + use_ste_1(Ls, Uses, Blocks); + Is -> + Blk = Blk0#b_blk{is=Is}, + use_ste_1(Ls, Uses, Blocks#{L:=Blk}) + end; +use_ste_1([], _, Blocks) -> Blocks. + +%%% Optimize within a single block. + +use_ste_is([#b_set{}=I|Is0], Uses) -> + Is = use_ste_is(Is0, Uses), + case extract_ste(I) of + none -> + [I|Is]; + Extracted -> + use_ste_call(Extracted, I, Is, Uses) + end; +use_ste_is([], _Uses) -> []. + +use_ste_call({Dst0,Pos0,_Var0,_Val0}, Call1, Is0, Uses) -> + case get_ste_call(Is0, []) of + {Prefix,{Dst1,Pos1,Dst0,Val1},Call2,Is} + when Pos1 > 0, Pos0 > Pos1 -> + case is_single_use(Dst0, Uses) of + true -> + Call = Call1#b_set{dst=Dst1}, + Args = [Val1,Dst1,#b_literal{val=Pos1-1}], + Dsetel = Call2#b_set{op=set_tuple_element, + dst=Dst0, + args=Args}, + [Call|Prefix] ++ [Dsetel|Is]; + false -> + [Call1|Is0] + end; + _ -> + [Call1|Is0] + end. + +get_ste_call([#b_set{op=get_tuple_element}=I|Is], Acc) -> + get_ste_call(Is, [I|Acc]); +get_ste_call([#b_set{op=call}=I|Is], Acc) -> + case extract_ste(I) of + none -> + none; + Extracted -> + {reverse(Acc),Extracted,I,Is} + end; +get_ste_call(_, _) -> none. + +extract_ste(#b_set{op=call,dst=Dst, + args=[#b_remote{mod=#b_literal{val=M}, + name=#b_literal{val=F}}|Args]}) -> + case {M,F,Args} of + {erlang,setelement,[#b_literal{val=Pos},Tuple,Val]} -> + {Dst,Pos,Tuple,Val}; + {_,_,_} -> + none + end; +extract_ste(#b_set{}) -> none. + +%%% Optimize accross blocks within a try/catch block. + +use_ste_across(L, Uses, Blocks) -> + case map_get(L, Blocks) of + #b_blk{last=#b_br{bool=#b_var{}}}=Blk -> + try + use_ste_across_1(L, Blk, Uses, Blocks) + catch + throw:not_possible -> + {Blk,Blocks} + end; + #b_blk{}=Blk -> + {Blk,Blocks} + end. + +use_ste_across_1(L, Blk0, Uses, Blocks0) -> + #b_blk{is=IsThis,last=#b_br{bool=Bool,succ=Next}} = Blk0, + case reverse(IsThis) of + [#b_set{op=succeeded,dst=Bool,args=[Result]}=Succ0, + #b_set{op=call,args=[#b_remote{}|_],dst=Result}=Call1|Prefix] -> + case is_single_use(Bool, Uses) andalso + is_n_uses(2, Result, Uses) of + true -> ok; + false -> throw(not_possible) + end, + Call2 = use_ste_across_next(Next, Uses, Blocks0), + Is = [Call1,Call2], + case use_ste_is(Is, decrement_uses(Result, Uses)) of + [#b_set{}=Call,#b_set{op=set_tuple_element}=Ste] -> + Blocks1 = use_ste_fix_next(Ste, Next, Blocks0), + Succ = Succ0#b_set{args=[Call#b_set.dst]}, + Blk = Blk0#b_blk{is=reverse(Prefix, [Call,Succ])}, + Blocks = Blocks1#{L:=Blk}, + {Blk,Blocks}; + _ -> + throw(not_possible) + end; + _ -> + throw(not_possible) + end. + +use_ste_across_next(Next, Uses, Blocks) -> + case map_get(Next, Blocks) of + #b_blk{is=[#b_set{op=call,dst=Result,args=[#b_remote{}|_]}=Call, + #b_set{op=succeeded,dst=Bool,args=[Result]}], + last=#b_br{bool=Bool}} -> + case is_single_use(Bool, Uses) andalso + is_n_uses(2, Result, Uses) of + true -> ok; + false -> throw(not_possible) + end, + Call; + #b_blk{} -> + throw(not_possible) + end. + +use_ste_fix_next(Ste, Next, Blocks) -> + Blk0 = map_get(Next, Blocks), + #b_blk{is=[#b_set{op=call},#b_set{op=succeeded}],last=Br0} = Blk0, + Br = beam_ssa:normalize(Br0#b_br{bool=#b_literal{val=true}}), + Blk = Blk0#b_blk{is=[Ste],last=Br}, + Blocks#{Next:=Blk}. + +%% Count how many times each variable is used. + +count_uses(Blocks) -> + count_uses_blk(maps:values(Blocks), #{}). + +count_uses_blk([#b_blk{is=Is,last=Last}|Bs], CountMap0) -> + F = fun(I, CountMap) -> + foldl(fun(Var, Acc) -> + case Acc of + #{Var:=3} -> Acc; + #{Var:=C} -> Acc#{Var:=C+1}; + #{} -> Acc#{Var=>1} + end + end, CountMap, beam_ssa:used(I)) + end, + CountMap = F(Last, foldl(F, CountMap0, Is)), + count_uses_blk(Bs, CountMap); +count_uses_blk([], CountMap) -> CountMap. + +decrement_uses(V, Uses) -> + #{V:=C} = Uses, + Uses#{V:=C-1}. + +is_n_uses(N, V, Uses) -> + case Uses of + #{V:=N} -> true; + #{} -> false + end. + +is_single_use(V, Uses) -> + case Uses of + #{V:=1} -> true; + #{} -> false + end. + +%%% +%%% Find out where frames should be placed. +%%% + +%% place_frames(St0) -> St. +%% Return a list of the labels for the blocks that need stack frame +%% allocation instructions. +%% +%% This function attempts to place stack frames as tight as possible +%% around the code, to avoid building stack frames for code paths +%% that don't need one. +%% +%% Stack frames are placed in blocks that dominate all of their +%% descendants. That guarantees that the deallocation instructions +%% cannot be reached from other execution paths that didn't set up +%% a stack frame or set up a stack frame with a different size. + +place_frames(#st{ssa=Blocks}=St) -> + {Doms,_} = beam_ssa:dominators(Blocks), + Ls = beam_ssa:rpo(Blocks), + Tried = gb_sets:empty(), + Frames0 = [], + {Frames,_} = place_frames_1(Ls, Blocks, Doms, Tried, Frames0), + St#st{frames=Frames}. + +place_frames_1([L|Ls], Blocks, Doms, Tried0, Frames0) -> + Blk = map_get(L, Blocks), + case need_frame(Blk) of + true -> + %% This block needs a frame. Try to place it here. + {Frames,Tried} = do_place_frame(L, Blocks, Doms, Tried0, Frames0), + + %% Successfully placed. Try to place more frames in descendants + %% that are not dominated by this block. + place_frames_1(Ls, Blocks, Doms, Tried, Frames); + false -> + try + place_frames_1(Ls, Blocks, Doms, Tried0, Frames0) + catch + throw:{need_frame,For,Tried1}=Reason -> + %% An descendant block needs a stack frame. Try to + %% place it here. + case is_dominated_by(For, L, Doms) of + true -> + %% Try to place a frame here. + {Frames,Tried} = do_place_frame(L, Blocks, Doms, + Tried1, Frames0), + place_frames_1(Ls, Blocks, Doms, Tried, Frames); + false -> + %% Wrong place. This block does not dominate + %% the block that needs the frame. Pass it on + %% to our ancestors. + throw(Reason) + end + end + end; +place_frames_1([], _, _, Tried, Frames) -> + {Frames,Tried}. + +%% do_place_frame(Label, Blocks, Dominators, Tried0, Frames0) -> {Frames,Tried}. +%% Try to place a frame in this block. This function returns +%% successfully if it either succeds at placing a frame in this +%% block, if an ancestor that dominates this block has already placed +%% a frame, or if we have already tried to put a frame in this block. +%% +%% An {need_frame,Label,Tried} exception will be thrown if this block +%% block is not suitable for having a stack frame (i.e. it does not dominate +%% all of its descendants). The exception means that an ancestor will have to +%% place the frame needed by this block. + +do_place_frame(L, Blocks, Doms, Tried0, Frames) -> + case gb_sets:is_element(L, Tried0) of + true -> + %% We have already tried to put a frame in this block. + {Frames,Tried0}; + false -> + %% Try to place a frame in this block. + Tried = gb_sets:insert(L, Tried0), + case place_frame_here(L, Blocks, Doms, Frames) of + yes -> + %% We need a frame and it is safe to place it here. + {[L|Frames],Tried}; + no -> + %% An ancestor has a frame. Not needed. + {Frames,Tried}; + ancestor -> + %% This block does not dominate all of its + %% descendants. We must place the frame in + %% an ancestor. + throw({need_frame,L,Tried}) + end + end. + +%% place_frame_here(Label, Blocks, Doms, Frames) -> no|yes|ancestor. +%% Determine whether a frame should be placed in block Label. + +place_frame_here(L, Blocks, Doms, Frames) -> + B0 = any(fun(DomBy) -> + is_dominated_by(L, DomBy, Doms) + end, Frames), + case B0 of + true -> + %% This block is dominated by an ancestor block that + %% defines a frame. Not needed/allowed to put a frame + %% here. + no; + false -> + %% No frame in any ancestor. We need a frame. + %% Now check whether the frame can be placed here. + %% If this block dominates all of its descendants + %% and the predecessors of any phi nodes it can be + %% placed here. + Descendants = beam_ssa:rpo([L], Blocks), + PhiPredecessors = phi_predecessors(L, Blocks), + MustDominate = ordsets:from_list(PhiPredecessors ++ Descendants), + Dominates = all(fun(?BADARG_BLOCK) -> + %% This block defines no variables and calls + %% erlang:error(badarg). It does not matter + %% whether L dominates ?BADARG_BLOCK or not; + %% it is still safe to put the frame in L. + true; + (Bl) -> + is_dominated_by(Bl, L, Doms) + end, MustDominate), + + %% Also, this block must not be a loop header. + IsLoopHeader = is_loop_header(L, Blocks), + case Dominates andalso not IsLoopHeader of + true -> yes; + false -> ancestor + end + end. + +%% phi_predecessors(Label, Blocks) -> +%% Return all predecessors referenced in phi nodes. + +phi_predecessors(L, Blocks) -> + #b_blk{is=Is} = map_get(L, Blocks), + [P || #b_set{op=phi,args=Args} <- Is, {_,P} <- Args]. + +%% is_dominated_by(Label, DominatedBy, Dominators) -> true|false. +%% Test whether block Label is dominated by block DominatedBy. + +is_dominated_by(L, DomBy, Doms) -> + DominatedBy = map_get(L, Doms), + member(DomBy, DominatedBy). + +%% need_frame(#b_blk{}) -> true|false. +%% Test whether any of the instructions in the block requires a stack frame. + +need_frame(#b_blk{is=Is,last=#b_ret{arg=Ret}}) -> + need_frame_1(Is, {return,Ret}); +need_frame(#b_blk{is=Is}) -> + need_frame_1(Is, body). + +need_frame_1([#b_set{op=make_fun,dst=Fun}|Is], {return,_}=Context) -> + %% Since make_fun clobbers X registers, a stack frame is needed if + %% any of the following instructions use any other variable than + %% the one holding the reference to the created fun. + need_frame_1(Is, Context) orelse + case beam_ssa:used(#b_blk{is=Is,last=#b_ret{arg=Fun}}) of + [Fun] -> false; + [_|_] -> true + end; +need_frame_1([#b_set{op=new_try_tag}|_], _) -> + true; +need_frame_1([#b_set{op=call,dst=Val}]=Is, {return,Ret}) -> + if + Val =:= Ret -> need_frame_1(Is, tail); + true -> need_frame_1(Is, body) + end; +need_frame_1([#b_set{op=call,args=[Func|_]}|Is], Context) -> + case Func of + #b_remote{mod=#b_literal{val=Mod}, + name=#b_literal{val=Name}, + arity=Arity} when is_atom(Mod), is_atom(Name) -> + case erl_bifs:is_exit_bif(Mod, Name, Arity) of + true -> + false; + false -> + Context =:= body orelse + Is =/= [] orelse + is_trap_bif(Mod, Name, Arity) + end; + #b_remote{} -> + %% This is an apply(), which always needs a frame. + true; + #b_local{} -> + Context =:= body orelse Is =/= []; + _ -> + %% A fun call always needs a frame. + true + end; +need_frame_1([I|Is], Context) -> + beam_ssa:clobbers_xregs(I) orelse need_frame_1(Is, Context); +need_frame_1([], _) -> false. + +%% is_trap_bif(Mod, Name, Arity) -> true|false. +%% Test whether we need a stack frame for this BIF. + +is_trap_bif(erlang, '!', 2) -> true; +is_trap_bif(erlang, link, 1) -> true; +is_trap_bif(erlang, unlink, 1) -> true; +is_trap_bif(erlang, monitor_node, 2) -> true; +is_trap_bif(erlang, group_leader, 2) -> true; +is_trap_bif(erlang, exit, 2) -> true; +is_trap_bif(_, _, _) -> false. + +%%% +%%% Fix variables used in matching in receive. +%%% +%%% The loop_rec/2 instruction may return a reference to a +%%% message outside of any heap or heap fragment. If the message +%%% does not match, it is not allowed to store any reference to +%%% the message (or part of the message) on the stack. If we do, +%%% the message will be corrupted if there happens to be a GC. +%%% +%%% Here we make sure to introduce copies of variables that are +%%% matched out and subsequently used after the remove_message/0 +%%% instructions. That will make sure that only X registers are +%%% used during matching. +%%% +%%% Depending on where variables are defined and used, they must +%%% be handled in two different ways. +%%% +%%% Variables that are always defined in the receive (before branching +%%% out into the different clauses of the receive) and used after the +%%% receive must be handled in the following way: Before each +%%% remove_message instruction, each such variable must be copied, and +%%% all variables must be consolidated using a phi node in the +%%% common exit block for the receive. +%%% +%%% Variables that are matched out and used in the same clause +%%% need copy instructions before the remove_message instruction +%%% in that clause. +%%% + +fix_receives(#st{ssa=Blocks0,cnt=Count0}=St) -> + {Blocks,Count} = fix_receives_1(maps:to_list(Blocks0), + Blocks0, Count0), + St#st{ssa=Blocks,cnt=Count}. + +fix_receives_1([{L,Blk}|Ls], Blocks0, Count0) -> + case Blk of + #b_blk{is=[#b_set{op=peek_message}|_]} -> + Rm = find_rm_blocks(L, Blocks0), + LoopExit = find_loop_exit(Rm, Blocks0), + Defs0 = beam_ssa:def([L], Blocks0), + CommonUsed = recv_common(Defs0, LoopExit, Blocks0), + {Blocks1,Count1} = recv_fix_common(CommonUsed, LoopExit, Rm, + Blocks0, Count0), + Defs = ordsets:subtract(Defs0, CommonUsed), + {Blocks,Count} = fix_receive(Rm, Defs, Blocks1, Count1), + fix_receives_1(Ls, Blocks, Count); + #b_blk{} -> + fix_receives_1(Ls, Blocks0, Count0) + end; +fix_receives_1([], Blocks, Count) -> + {Blocks,Count}. + +recv_common(_Defs, none, _Blocks) -> + %% There is no common exit block because receive is used + %% in the tail position of a function. + []; +recv_common(Defs, Exit, Blocks) -> + {ExitDefs,ExitUsed} = beam_ssa:def_used([Exit], Blocks), + Def = ordsets:subtract(Defs, ExitDefs), + ordsets:intersection(Def, ExitUsed). + +%% recv_fix_common([CommonVar], LoopExit, [RemoveMessageLabel], +%% Blocks0, Count0) -> {Blocks,Count}. +%% Handle variables alwys defined in a receive and used +%% in the exit block following the receive. + +recv_fix_common([Msg0|T], Exit, Rm, Blocks0, Count0) -> + {Msg,Count1} = new_var('@recv', Count0), + Blocks1 = beam_ssa:rename_vars(#{Msg0=>Msg}, [Exit], Blocks0), + N = length(Rm), + {MsgVars,Count} = new_vars(duplicate(N, '@recv'), Count1), + PhiArgs = fix_exit_phi_args(MsgVars, Rm, Exit, Blocks1), + Phi = #b_set{op=phi,dst=Msg,args=PhiArgs}, + ExitBlk0 = map_get(Exit, Blocks1), + ExitBlk = ExitBlk0#b_blk{is=[Phi|ExitBlk0#b_blk.is]}, + Blocks2 = Blocks1#{Exit:=ExitBlk}, + Blocks = recv_fix_common_1(MsgVars, Rm, Msg0, Blocks2), + recv_fix_common(T, Exit, Rm, Blocks, Count); +recv_fix_common([], _, _, Blocks, Count) -> + {Blocks,Count}. + +recv_fix_common_1([V|Vs], [Rm|Rms], Msg, Blocks0) -> + Ren = #{Msg=>V}, + Blocks1 = beam_ssa:rename_vars(Ren, [Rm], Blocks0), + #b_blk{is=Is0} = Blk0 = map_get(Rm, Blocks1), + Copy = #b_set{op=copy,dst=V,args=[Msg]}, + Is = insert_after_phis(Is0, [Copy]), + Blk = Blk0#b_blk{is=Is}, + Blocks = Blocks1#{Rm:=Blk}, + recv_fix_common_1(Vs, Rms, Msg, Blocks); +recv_fix_common_1([], [], _Msg, Blocks) -> Blocks. + +fix_exit_phi_args([V|Vs], [Rm|Rms], Exit, Blocks) -> + Path = beam_ssa:rpo([Rm], Blocks), + Preds = exit_predecessors(Path, Exit, Blocks), + [{V,Pred} || Pred <- Preds] ++ fix_exit_phi_args(Vs, Rms, Exit, Blocks); +fix_exit_phi_args([], [], _, _) -> []. + +exit_predecessors([L|Ls], Exit, Blocks) -> + Blk = map_get(L, Blocks), + case member(Exit, beam_ssa:successors(Blk)) of + true -> + [L|exit_predecessors(Ls, Exit, Blocks)]; + false -> + exit_predecessors(Ls, Exit, Blocks) + end; +exit_predecessors([], _Exit, _Blocks) -> []. + +%% fix_receive([Label], Defs, Blocks0, Count0) -> {Blocks,Count}. +%% Add a copy instruction for all variables that are matched out and +%% later used within a clause of the receive. + +fix_receive([L|Ls], Defs, Blocks0, Count0) -> + {RmDefs,Used0} = beam_ssa:def_used([L], Blocks0), + Def = ordsets:subtract(Defs, RmDefs), + Used = ordsets:intersection(Def, Used0), + {NewVars,Count} = new_vars([Base || #b_var{name=Base} <- Used], Count0), + Ren = zip(Used, NewVars), + Blocks1 = beam_ssa:rename_vars(Ren, [L], Blocks0), + #b_blk{is=Is0} = Blk1 = map_get(L, Blocks1), + CopyIs = [#b_set{op=copy,dst=New,args=[Old]} || {Old,New} <- Ren], + Is = insert_after_phis(Is0, CopyIs), + Blk = Blk1#b_blk{is=Is}, + Blocks = Blocks1#{L:=Blk}, + fix_receive(Ls, Defs, Blocks, Count); +fix_receive([], _Defs, Blocks, Count) -> + {Blocks,Count}. + +%% find_loop_exit([Label], Blocks) -> Label | none. +%% Find the block to which control is transferred when the +%% the receive loop is exited. + +find_loop_exit([L1,L2|_Ls], Blocks) -> + Path1 = beam_ssa:rpo([L1], Blocks), + Path2 = beam_ssa:rpo([L2], Blocks), + find_loop_exit_1(reverse(Path1), reverse(Path2), none); +find_loop_exit(_, _) -> none. + +find_loop_exit_1([H|T1], [H|T2], _) -> + find_loop_exit_1(T1, T2, H); +find_loop_exit_1(_, _, Exit) -> Exit. + +%% find_rm_blocks(StartLabel, Blocks) -> [Label]. +%% Find all blocks that start with remove_message within the receive +%% loop whose peek_message label is StartLabel. + +find_rm_blocks(L, Blocks) -> + Seen = gb_sets:singleton(L), + Blk = map_get(L, Blocks), + Succ = beam_ssa:successors(Blk), + find_rm_blocks_1(Succ, Seen, Blocks). + +find_rm_blocks_1([L|Ls], Seen0, Blocks) -> + case gb_sets:is_member(L, Seen0) of + true -> + find_rm_blocks_1(Ls, Seen0, Blocks); + false -> + Seen = gb_sets:insert(L, Seen0), + Blk = map_get(L, Blocks), + case find_rm_act(Blk#b_blk.is) of + prune -> + %% Looping back. Don't look at any successors. + find_rm_blocks_1(Ls, Seen, Blocks); + continue -> + %% Neutral block. Do nothing here, but look at + %% all successors. + Succ = beam_ssa:successors(Blk), + find_rm_blocks_1(Succ++Ls, Seen, Blocks); + found -> + %% Found remove_message instruction. + [L|find_rm_blocks_1(Ls, Seen, Blocks)] + end + end; +find_rm_blocks_1([], _, _) -> []. + +find_rm_act([#b_set{op=Op}|Is]) -> + case Op of + remove_message -> found; + peek_message -> prune; + recv_next -> prune; + wait_timeout -> prune; + wait -> prune; + _ -> find_rm_act(Is) + end; +find_rm_act([]) -> + continue. + +%%% +%%% Find out which variables need to be stored in Y registers. +%%% + +-record(dk, {d :: ordsets:ordset(var_name()), + k :: ordsets:ordset(var_name()) + }). + +%% find_yregs(St0) -> St. +%% Find all variables that must be stored in Y registers. Annotate +%% the blocks that allocate frames with the set of Y registers +%% used within that stack frame. +%% +%% Basically, we following all execution paths starting from a block +%% that allocates a frame, keeping track of of all defined registers +%% and all registers killed by an instruction that clobbers X +%% registers. For every use of a variable, we check if if it is in +%% the set of killed variables; if it is, it must be stored in an Y +%% register. + +find_yregs(#st{frames=[]}=St) -> + St; +find_yregs(#st{frames=[_|_]=Frames,args=Args,ssa=Blocks0}=St) -> + FrameDefs = find_defs(Frames, Blocks0, [V || #b_var{}=V <- Args]), + Blocks = find_yregs_1(FrameDefs, Blocks0), + St#st{ssa=Blocks}. + +find_yregs_1([{F,Defs}|Fs], Blocks0) -> + DK = #dk{d=Defs,k=[]}, + D0 = #{F=>DK}, + Ls = beam_ssa:rpo([F], Blocks0), + Yregs0 = [], + Yregs = find_yregs_2(Ls, Blocks0, D0, Yregs0), + Blk0 = map_get(F, Blocks0), + Blk = beam_ssa:add_anno(yregs, Yregs, Blk0), + Blocks = Blocks0#{F:=Blk}, + find_yregs_1(Fs, Blocks); +find_yregs_1([], Blocks) -> Blocks. + +find_yregs_2([L|Ls], Blocks0, D0, Yregs0) -> + Blk0 = map_get(L, Blocks0), + #b_blk{is=Is,last=Last} = Blk0, + Ys0 = map_get(L, D0), + {Yregs1,Ys} = find_yregs_is(Is, Ys0, Yregs0), + Yregs = find_yregs_terminator(Last, Ys, Yregs1), + Successors = beam_ssa:successors(Blk0), + D = find_update_succ(Successors, Ys, D0), + find_yregs_2(Ls, Blocks0, D, Yregs); +find_yregs_2([], _Blocks, _D, Yregs) -> Yregs. + +find_defs(Frames, Blocks, Defs) -> + Seen = gb_sets:empty(), + FramesSet = gb_sets:from_list(Frames), + {FrameDefs,_} = find_defs_1([0], Blocks, FramesSet, Seen, Defs, []), + FrameDefs. + +find_defs_1([L|Ls], Blocks, Frames, Seen0, Defs0, Acc0) -> + case gb_sets:is_member(L, Frames) of + true -> + OrderedDefs = ordsets:from_list(Defs0), + find_defs_1(Ls, Blocks, Frames, Seen0, Defs0, + [{L,OrderedDefs}|Acc0]); + false -> + case gb_sets:is_member(L, Seen0) of + true -> + find_defs_1(Ls, Blocks, Frames, Seen0, Defs0, Acc0); + false -> + Seen1 = gb_sets:insert(L, Seen0), + {Acc,Seen} = find_defs_1(Ls, Blocks, Frames, Seen1, Defs0, Acc0), + #b_blk{is=Is} = Blk = map_get(L, Blocks), + Defs = find_defs_is(Is, Defs0), + Successors = beam_ssa:successors(Blk), + find_defs_1(Successors, Blocks, Frames, Seen, Defs, Acc) + end + end; +find_defs_1([], _, _, Seen, _, Acc) -> + {Acc,Seen}. + +find_defs_is([#b_set{dst=Dst}|Is], Acc) -> + find_defs_is(Is, [Dst|Acc]); +find_defs_is([], Acc) -> Acc. + +find_update_succ([S|Ss], #dk{d=Defs0,k=Killed0}=DK0, D0) -> + case D0 of + #{S:=#dk{d=Defs1,k=Killed1}} -> + Defs = ordsets:intersection(Defs0, Defs1), + Killed = ordsets:union(Killed0, Killed1), + DK = #dk{d=Defs,k=Killed}, + D = D0#{S:=DK}, + find_update_succ(Ss, DK0, D); + #{} -> + D = D0#{S=>DK0}, + find_update_succ(Ss, DK0, D) + end; +find_update_succ([], _, D) -> D. + +find_yregs_is([#b_set{dst=Dst}=I|Is], #dk{d=Defs0,k=Killed0}=Ys, Yregs0) -> + Used = beam_ssa:used(I), + Yregs1 = ordsets:intersection(Used, Killed0), + Yregs = ordsets:union(Yregs0, Yregs1), + case beam_ssa:clobbers_xregs(I) of + false -> + Defs = ordsets:add_element(Dst, Defs0), + find_yregs_is(Is, Ys#dk{d=Defs}, Yregs); + true -> + Killed = ordsets:union(Defs0, Killed0), + Defs = [Dst], + find_yregs_is(Is, Ys#dk{d=Defs,k=Killed}, Yregs) + end; +find_yregs_is([], Ys, Yregs) -> {Yregs,Ys}. + +find_yregs_terminator(Terminator, #dk{k=Killed}, Yregs0) -> + Used = beam_ssa:used(Terminator), + Yregs = ordsets:intersection(Used, Killed), + ordsets:union(Yregs0, Yregs). + +%%% +%%% Try to reduce the size of the stack frame, by adding an explicit +%%% 'copy' instructions for return values from 'call' and 'make_fun' that +%%% need to be saved in Y registers. Here is an example to show +%%% how that's useful. First, here is the Erlang code: +%%% +%%% f(Pid) -> +%%% Res = foo(42), +%%% _ = node(Pid), +%%% bar(), +%%% Res. +%%% +%%% Compiled to SSA format, the main part of the code looks like this: +%%% +%%% 0: +%%% Res = call local literal foo/1, literal 42 +%%% _1 = bif:node Pid +%%% @ssa_bool = succeeded _1 +%%% br @ssa_bool, label 3, label 1 +%%% 3: +%%% @ssa_ignored = call local literal bar/0 +%%% ret Res +%%% +%%% It can be seen that the variables Pid and Res must be saved in Y +%%% registers in order to survive the function calls. A previous sub +%%% pass has inserted a 'copy' instruction to save the value of the +%%% variable Pid: +%%% +%%% 0: +%%% Pid:4 = copy Pid +%%% Res = call local literal foo/1, literal 42 +%%% _1 = bif:node Pid:4 +%%% @ssa_bool = succeeded _1 +%%% br @ssa_bool, label 3, label 1 +%%% +%%% 3: +%%% @ssa_ignored = call local literal bar/0 +%%% ret Res +%%% +%%% The Res and Pid:4 variables must be assigned to different Y registers +%%% because they are live at the same time. copy_retval() inserts a +%%% 'copy' instruction to copy Res to a new variable: +%%% +%%% 0: +%%% Pid:4 = copy Pid +%%% Res:6 = call local literal foo/1, literal 42 +%%% _1 = bif:node Pid:4 +%%% @ssa_bool = succeeded _1 +%%% br @ssa_bool, label 3, label 1 +%%% +%%% 3: +%%% Res = copy Res:6 +%%% @ssa_ignored = call local literal bar/0 +%%% ret Res +%%% +%%% The new variable Res:6 is used to capture the return value from the call. +%%% The variables Pid:4 and Res are no longer live at the same time, so they +%%% can be assigned to the same Y register. +%%% + +copy_retval(#st{frames=Frames,ssa=Blocks0,cnt=Count0}=St) -> + {Blocks,Count} = copy_retval_1(Frames, Blocks0, Count0), + St#st{ssa=Blocks,cnt=Count}. + +copy_retval_1([F|Fs], Blocks0, Count0) -> + #b_blk{anno=#{yregs:=Yregs0},is=Is} = map_get(F, Blocks0), + Yregs1 = gb_sets:from_list(Yregs0), + Yregs = collect_yregs(Is, Yregs1), + Ls = beam_ssa:rpo([F], Blocks0), + {Blocks,Count} = copy_retval_2(Ls, Yregs, none, Blocks0, Count0), + copy_retval_1(Fs, Blocks, Count); +copy_retval_1([], Blocks, Count) -> + {Blocks,Count}. + +collect_yregs([#b_set{op=copy,dst=Y,args=[#b_var{}=X]}|Is], + Yregs0) -> + true = gb_sets:is_member(X, Yregs0), %Assertion. + Yregs = gb_sets:insert(Y, gb_sets:delete(X, Yregs0)), + collect_yregs(Is, Yregs); +collect_yregs([#b_set{}|Is], Yregs) -> + collect_yregs(Is, Yregs); +collect_yregs([], Yregs) -> Yregs. + +copy_retval_2([L|Ls], Yregs, Copy0, Blocks0, Count0) -> + #b_blk{is=Is0,last=Last} = Blk = map_get(L, Blocks0), + RC = case {Last,Ls} of + {#b_br{succ=Succ,fail=?BADARG_BLOCK},[Succ|_]} -> + true; + {_,_} -> + false + end, + case copy_retval_is(Is0, RC, Yregs, Copy0, Count0, []) of + {Is,Count} -> + case Copy0 =:= none andalso Count0 =:= Count of + true -> + copy_retval_2(Ls, Yregs, none, Blocks0, Count0); + false -> + Blocks = Blocks0#{L=>Blk#b_blk{is=Is}}, + copy_retval_2(Ls, Yregs, none, Blocks, Count) + end; + {Is,Count,Copy} -> + Blocks = Blocks0#{L=>Blk#b_blk{is=Is}}, + copy_retval_2(Ls, Yregs, Copy, Blocks, Count) + end; +copy_retval_2([], _Yregs, none, Blocks, Count) -> + {Blocks,Count}. + +copy_retval_is([#b_set{op=put_tuple_elements,args=Args0}=I0], false, _Yregs, + Copy, Count, Acc) -> + I = I0#b_set{args=copy_sub_args(Args0, Copy)}, + {reverse(Acc, [I|acc_copy([], Copy)]),Count}; +copy_retval_is([#b_set{op=Op}=I0], false, Yregs, Copy, Count0, Acc0) + when Op =:= call; Op =:= make_fun -> + {I,Count,Acc} = place_retval_copy(I0, Yregs, Copy, Count0, Acc0), + {reverse(Acc, [I]),Count}; +copy_retval_is([#b_set{}]=Is, false, _Yregs, Copy, Count, Acc) -> + {reverse(Acc, acc_copy(Is, Copy)),Count}; +copy_retval_is([#b_set{},#b_set{op=succeeded}]=Is, false, _Yregs, Copy, Count, Acc) -> + {reverse(Acc, acc_copy(Is, Copy)),Count}; +copy_retval_is([#b_set{op=Op,dst=#b_var{name=RetName}=Dst}=I0|Is], RC, Yregs, + Copy0, Count0, Acc0) when Op =:= call; Op =:= make_fun -> + {I1,Count1,Acc} = place_retval_copy(I0, Yregs, Copy0, Count0, Acc0), + case gb_sets:is_member(Dst, Yregs) of + true -> + {NewVar,Count} = new_var(RetName, Count1), + Copy = #b_set{op=copy,dst=Dst,args=[NewVar]}, + I = I1#b_set{dst=NewVar}, + copy_retval_is(Is, RC, Yregs, Copy, Count, [I|Acc]); + false -> + copy_retval_is(Is, RC, Yregs, none, Count1, [I1|Acc]) + end; +copy_retval_is([#b_set{args=Args0}=I0|Is], RC, Yregs, Copy, Count, Acc) -> + I = I0#b_set{args=copy_sub_args(Args0, Copy)}, + case beam_ssa:clobbers_xregs(I) of + true -> + copy_retval_is(Is, RC, Yregs, none, Count, [I|acc_copy(Acc, Copy)]); + false -> + copy_retval_is(Is, RC, Yregs, Copy, Count, [I|Acc]) + end; +copy_retval_is([], RC, _, Copy, Count, Acc) -> + case {Copy,RC} of + {none,_} -> + {reverse(Acc),Count}; + {#b_set{},true} -> + {reverse(Acc),Count,Copy}; + {#b_set{},false} -> + {reverse(Acc, [Copy]),Count} + end. + +%% +%% Consider this code: +%% +%% Var = ... +%% ... +%% A1 = call foo/0 +%% A = copy A1 +%% B = call bar/1, Var +%% +%% If the Var variable is no longer used after this code, its Y register +%% can't be reused for A. To allow the Y register to be reused +%% we will need to insert 'copy' instructions for arguments that are +%% in Y registers: +%% +%% Var = ... +%% ... +%% A1 = call foo/0 +%% Var1 = copy Var +%% A = copy A1 +%% B = call bar/1, Var1 +%% + +place_retval_copy(I, _Yregs, none, Count, Acc) -> + {I,Count,Acc}; +place_retval_copy(#b_set{args=[F|Args0]}=I, Yregs, Copy, Count0, Acc0) -> + #b_set{dst=Avoid} = Copy, + {Args,Acc1,Count} = copy_func_args(Args0, Yregs, Avoid, Acc0, [], Count0), + Acc = [Copy|Acc1], + {I#b_set{args=[F|Args]},Count,Acc}. + +copy_func_args([#b_var{name=AName}=A|As], Yregs, Avoid, CopyAcc, Acc, Count0) -> + case gb_sets:is_member(A, Yregs) of + true when A =/= Avoid -> + {NewVar,Count} = new_var(AName, Count0), + Copy = #b_set{op=copy,dst=NewVar,args=[A]}, + copy_func_args(As, Yregs, Avoid, [Copy|CopyAcc], [NewVar|Acc], Count); + _ -> + copy_func_args(As, Yregs, Avoid, CopyAcc, [A|Acc], Count0) + end; +copy_func_args([A|As], Yregs, Avoid, CopyAcc, Acc, Count) -> + copy_func_args(As, Yregs, Avoid, CopyAcc, [A|Acc], Count); +copy_func_args([], _Yregs, _Avoid, CopyAcc, Acc, Count) -> + {reverse(Acc),CopyAcc,Count}. + +acc_copy(Acc, none) -> Acc; +acc_copy(Acc, #b_set{}=Copy) -> [Copy|Acc]. + +copy_sub_args(Args, none) -> + Args; +copy_sub_args(Args, #b_set{dst=Dst,args=[Src]}) -> + [sub_arg(A, Dst, Src) || A <- Args]. + +sub_arg(Old, Old, New) -> New; +sub_arg(Old, _, _) -> Old. + +%%% +%%% Consider: +%%% +%%% x1/Hd = get_hd x0/Cons +%%% y0/Tl = get_tl x0/Cons +%%% +%%% Register x0 can't be reused for Hd. If Hd needs to be in x0, +%%% a 'move' instruction must be inserted. +%%% +%%% If we swap get_hd and get_tl when Tl is in a Y register, +%%% x0 can be used for Hd if Cons is not used again: +%%% +%%% y0/Tl = get_tl x0/Cons +%%% x0/Hd = get_hd x0/Cons +%%% + +opt_get_list(#st{ssa=Blocks,res=Res}=St) -> + ResMap = maps:from_list(Res), + Ls = beam_ssa:rpo(Blocks), + St#st{ssa=opt_get_list_1(Ls, ResMap, Blocks)}. + +opt_get_list_1([L|Ls], Res, Blocks0) -> + #b_blk{is=Is0} = Blk = map_get(L, Blocks0), + case opt_get_list_is(Is0, Res, [], false) of + no -> + opt_get_list_1(Ls, Res, Blocks0); + {yes,Is} -> + Blocks = Blocks0#{L:=Blk#b_blk{is=Is}}, + opt_get_list_1(Ls, Res, Blocks) + end; +opt_get_list_1([], _, Blocks) -> Blocks. + +opt_get_list_is([#b_set{op=get_hd,dst=Hd, + args=[Cons]}=GetHd, + #b_set{op=get_tl,dst=Tl, + args=[Cons]}=GetTl|Is], + Res, Acc, Changed) -> + %% Note that when this pass is run, only Y registers have + %% reservations. The absence of an entry for a variable therefore + %% means that the variable will be in an X register. + case Res of + #{Hd:={y,_}} -> + %% Hd will be in a Y register. Don't swap. + opt_get_list_is([GetTl|Is], Res, [GetHd|Acc], Changed); + #{Tl:={y,_}} -> + %% Tl will be in a Y register. Swap. + opt_get_list_is([GetHd|Is], Res, [GetTl|Acc], true); + #{} -> + %% Both are in X registers. Nothing to do. + opt_get_list_is([GetTl|Is], Res, [GetHd|Acc], Changed) + end; +opt_get_list_is([I|Is], Res, Acc, Changed) -> + opt_get_list_is(Is, Res, [I|Acc], Changed); +opt_get_list_is([], _Res, Acc, Changed) -> + case Changed of + true -> + {yes,reverse(Acc)}; + false -> + no + end. + +%%% +%%% Number instructions in the order they are executed. +%%% + +%% number_instructions(St0) -> St. +%% Number instructions in the order they are executed. Use a step +%% size of 2. Don't number phi instructions. All phi variables in +%% a block will be live one unit before the first non-phi instruction +%% in the block. + +number_instructions(#st{ssa=Blocks0}=St) -> + Ls = beam_ssa:rpo(Blocks0), + St#st{ssa=number_is_1(Ls, 1, Blocks0)}. + +number_is_1([L|Ls], N0, Blocks0) -> + #b_blk{is=Is0,last=Last0} = Bl0 = map_get(L, Blocks0), + {Is,N1} = number_is_2(Is0, N0, []), + Last = beam_ssa:add_anno(n, N1, Last0), + N = N1 + 2, + Bl = Bl0#b_blk{is=Is,last=Last}, + Blocks = Blocks0#{L:=Bl}, + number_is_1(Ls, N, Blocks); +number_is_1([], _, Blocks) -> Blocks. + +number_is_2([#b_set{op=phi}=I|Is], N, Acc) -> + number_is_2(Is, N, [I|Acc]); +number_is_2([I0|Is], N, Acc) -> + I = beam_ssa:add_anno(n, N, I0), + number_is_2(Is, N+2, [I|Acc]); +number_is_2([], N, Acc) -> + {reverse(Acc),N}. + +%%% +%%% Calculate live intervals. +%%% + +live_intervals(#st{args=Args,ssa=Blocks}=St) -> + Vars0 = [{V,{0,1}} || #b_var{}=V <- Args], + F = fun(L, _, A) -> live_interval_blk(L, Blocks, A) end, + LiveMap0 = #{}, + Acc0 = {[],LiveMap0}, + {Vars,_} = beam_ssa:fold_po(F, Acc0, Blocks), + Intervals = merge_ranges(rel2fam(Vars0++Vars)), + St#st{intervals=Intervals}. + +merge_ranges([{V,Rs}|T]) -> + [{V,merge_ranges_1(Rs)}|merge_ranges(T)]; +merge_ranges([]) -> []. + +merge_ranges_1([{A,N},{N,Z}|Rs]) -> + merge_ranges_1([{A,Z}|Rs]); +merge_ranges_1([R|Rs]) -> + [R|merge_ranges_1(Rs)]; +merge_ranges_1([]) -> []. + +live_interval_blk(L, Blocks, {Vars0,LiveMap0}) -> + Live0 = [], + Successors = beam_ssa:successors(L, Blocks), + Live1 = update_successors(Successors, L, Blocks, LiveMap0, Live0), + + %% Add ranges for all variables that are live in the successors. + #b_blk{is=Is,last=Last} = map_get(L, Blocks), + End = beam_ssa:get_anno(n, Last), + Use = [{V,{use,End+1}} || V <- Live1], + + %% Determine used and defined variables in this block. + FirstNumber = first_number(Is, Last), + UseDef0 = live_interval_blk_1([Last|reverse(Is)], FirstNumber, Use), + UseDef = rel2fam(UseDef0), + + %% Update what is live at the beginning of this block and + %% store it. + Used = [V || {V,[{use,_}|_]} <- UseDef], + Live2 = ordsets:union(Live1, Used), + Killed = [V || {V,[{def,_}|_]} <- UseDef], + Live = ordsets:subtract(Live2, Killed), + LiveMap = LiveMap0#{L=>Live}, + + %% Construct the ranges for this block. + Vars = make_block_ranges(UseDef, FirstNumber, Vars0), + {Vars,LiveMap}. + +make_block_ranges([{V,[{def,Def}]}|Vs], First, Acc) -> + make_block_ranges(Vs, First, [{V,{Def,Def}}|Acc]); +make_block_ranges([{V,[{def,Def}|Uses]}|Vs], First, Acc) -> + {use,Last} = last(Uses), + make_block_ranges(Vs, First, [{V,{Def,Last}}|Acc]); +make_block_ranges([{V,[{use,_}|_]=Uses}|Vs], First, Acc) -> + {use,Last} = last(Uses), + make_block_ranges(Vs, First, [{V,{First,Last}}|Acc]); +make_block_ranges([], _, Acc) -> Acc. + +live_interval_blk_1([#b_set{op=phi,dst=Dst}|Is], FirstNumber, Acc0) -> + Acc = [{Dst,{def,FirstNumber}}|Acc0], + live_interval_blk_1(Is, FirstNumber, Acc); +live_interval_blk_1([#b_set{op=bs_start_match}=I|Is], + FirstNumber, Acc0) -> + N = beam_ssa:get_anno(n, I), + #b_set{dst=Dst} = I, + Acc1 = [{Dst,{def,N}}|Acc0], + Acc = [{V,{use,N}} || V <- beam_ssa:used(I)] ++ Acc1, + live_interval_blk_1(Is, FirstNumber, Acc); +live_interval_blk_1([I|Is], FirstNumber, Acc0) -> + N = beam_ssa:get_anno(n, I), + Acc1 = case I of + #b_set{dst=Dst} -> + [{Dst,{def,N}}|Acc0]; + _ -> + Acc0 + end, + Used = beam_ssa:used(I), + Acc = [{V,{use,N}} || V <- Used] ++ Acc1, + live_interval_blk_1(Is, FirstNumber, Acc); +live_interval_blk_1([], _FirstNumber, Acc) -> + Acc. + +%% first_number([#b_set{}]) -> InstructionNumber. +%% Return the number for the first instruction for the block. +%% Note that this number is one less than the first +%% non-phi instruction in the block. + +first_number([#b_set{op=phi}|Is], Last) -> + first_number(Is, Last); +first_number([I|_], _) -> + beam_ssa:get_anno(n, I) - 1; +first_number([], Last) -> + beam_ssa:get_anno(n, Last) - 1. + +update_successors([L|Ls], Pred, Blocks, LiveMap, Live0) -> + Live1 = ordsets:union(Live0, get_live(L, LiveMap)), + #b_blk{is=Is} = map_get(L, Blocks), + Live = update_live_phis(Is, Pred, Live1), + update_successors(Ls, Pred, Blocks, LiveMap, Live); +update_successors([], _, _, _, Live) -> Live. + +get_live(L, LiveMap) -> + case LiveMap of + #{L:=Live} -> Live; + #{} -> [] + end. + +update_live_phis([#b_set{op=phi,dst=Killed,args=Args}|Is], + Pred, Live0) -> + Used = [V || {#b_var{}=V,L} <- Args, L =:= Pred], + Live1 = ordsets:union(ordsets:from_list(Used), Live0), + Live = ordsets:del_element(Killed, Live1), + update_live_phis(Is, Pred, Live); +update_live_phis(_, _, Live) -> Live. + +%%% +%%% Reserve Y registers. +%%% + +%% reserve_yregs(St0) -> St. +%% In each block that allocates a stack frame, insert instructions +%% that copy variables that must be in Y registers (given by +%% the `yregs` annotation) to new variables. +%% +%% Also allocate specific Y registers for try and catch tags. +%% The outermost try/catch tag is placed in y0, any directly +%% nested tag in y1, and so on. Note that this is the reversed +%% order as required by BEAM; it will be corrected later by +%% turn_yregs(). + +reserve_yregs(#st{frames=Frames}=St0) -> + foldl(fun reserve_yregs_1/2, St0, Frames). + +reserve_yregs_1(L, #st{ssa=Blocks0,cnt=Count0,res=Res0}=St) -> + Blk = map_get(L, Blocks0), + Yregs = beam_ssa:get_anno(yregs, Blk), + {Def,Used} = beam_ssa:def_used([L], Blocks0), + UsedYregs = ordsets:intersection(Yregs, Used), + DefBefore = ordsets:subtract(UsedYregs, Def), + {BeforeVars,Blocks,Count} = rename_vars(DefBefore, L, Blocks0, Count0), + InsideVars = ordsets:subtract(UsedYregs, DefBefore), + ResTryTags0 = reserve_try_tags(L, Blocks), + ResTryTags = [{V,{Reg,Count}} || {V,Reg} <- ResTryTags0], + Vars = BeforeVars ++ InsideVars, + Res = [{V,{y,Count}} || V <- Vars] ++ ResTryTags ++ Res0, + St#st{res=Res,ssa=Blocks,cnt=Count+1}. + +reserve_try_tags(L, Blocks) -> + Seen = gb_sets:empty(), + {Res0,_} = reserve_try_tags_1([L], Blocks, Seen, #{}), + Res1 = [maps:to_list(M) || {_,M} <- maps:to_list(Res0)], + Res = [{V,{y,Y}} || {V,Y} <- append(Res1)], + ordsets:from_list(Res). + +reserve_try_tags_1([L|Ls], Blocks, Seen0, ActMap0) -> + case gb_sets:is_element(L, Seen0) of + true -> + reserve_try_tags_1(Ls, Blocks, Seen0, ActMap0); + false -> + Seen1 = gb_sets:insert(L, Seen0), + #b_blk{is=Is} = Blk = map_get(L, Blocks), + Active0 = get_active(L, ActMap0), + Active = reserve_try_tags_is(Is, Active0), + Successors = beam_ssa:successors(Blk), + ActMap1 = update_act_map(Successors, Active, ActMap0), + {ActMap,Seen} = reserve_try_tags_1(Ls, Blocks, Seen1, ActMap1), + reserve_try_tags_1(Successors, Blocks, Seen,ActMap) + end; +reserve_try_tags_1([], _Blocks, Seen, ActMap) -> + {ActMap,Seen}. + +get_active(L, ActMap) -> + case ActMap of + #{L:=Active} -> Active; + #{} -> #{} + end. + +reserve_try_tags_is([#b_set{op=new_try_tag,dst=V}|Is], Active) -> + N = map_size(Active), + reserve_try_tags_is(Is, Active#{V=>N}); +reserve_try_tags_is([#b_set{op=kill_try_tag,args=[Tag]}|Is], Active) -> + reserve_try_tags_is(Is, maps:remove(Tag, Active)); +reserve_try_tags_is([_|Is], Active) -> + reserve_try_tags_is(Is, Active); +reserve_try_tags_is([], Active) -> Active. + +update_act_map([L|Ls], Active0, ActMap0) -> + case ActMap0 of + #{L:=Active1} -> + ActMap = ActMap0#{L=>maps:merge(Active0, Active1)}, + update_act_map(Ls, Active0, ActMap); + #{} -> + ActMap = ActMap0#{L=>Active0}, + update_act_map(Ls, Active0, ActMap) + end; +update_act_map([], _, ActMap) -> ActMap. + +rename_vars([], _, Blocks, Count) -> + {[],Blocks,Count}; +rename_vars(Vs, L, Blocks0, Count0) -> + {NewVars,Count} = new_vars([Base || #b_var{name=Base} <- Vs], Count0), + Ren = zip(Vs, NewVars), + Blocks1 = beam_ssa:rename_vars(Ren, [L], Blocks0), + #b_blk{is=Is0} = Blk0 = map_get(L, Blocks1), + CopyIs = [#b_set{op=copy,dst=New,args=[Old]} || {Old,New} <- Ren], + Is = insert_after_phis(Is0, CopyIs), + Blk = Blk0#b_blk{is=Is}, + Blocks = Blocks1#{L:=Blk}, + {NewVars,Blocks,Count}. + +insert_after_phis([#b_set{op=phi}=I|Is], InsertIs) -> + [I|insert_after_phis(Is, InsertIs)]; +insert_after_phis(Is, InsertIs) -> + InsertIs ++ Is. + +%% frame_size(St0) -> St. +%% Calculate the frame size for each block that allocates a frame. +%% Annotate the block with the frame size. Also annotate all +%% return instructions with {deallocate,FrameSize} to simplify +%% code generation. + +frame_size(#st{frames=Frames,regs=Regs,ssa=Blocks0}=St) -> + Blocks = foldl(fun(L, Blks) -> + frame_size_1(L, Regs, Blks) + end, Blocks0, Frames), + St#st{ssa=Blocks}. + +frame_size_1(L, Regs, Blocks0) -> + Def = beam_ssa:def([L], Blocks0), + Yregs0 = [map_get(V, Regs) || V <- Def, is_yreg(map_get(V, Regs))], + Yregs = ordsets:from_list(Yregs0), + FrameSize = length(ordsets:from_list(Yregs)), + if + FrameSize =/= 0 -> + [{y,0}|_] = Yregs, %Assertion. + {y,Last} = last(Yregs), + Last = FrameSize - 1, %Assertion. + ok; + true -> + ok + end, + Blk0 = map_get(L, Blocks0), + Blk = beam_ssa:add_anno(frame_size, FrameSize, Blk0), + + %% Insert an annotation for frame deallocation on + %% each #b_ret{}. + Blocks = Blocks0#{L:=Blk}, + Reachable = beam_ssa:rpo([L], Blocks), + frame_deallocate(Reachable, FrameSize, Blocks). + +frame_deallocate([L|Ls], Size, Blocks0) -> + Blk0 = map_get(L, Blocks0), + Blk = case Blk0 of + #b_blk{last=#b_ret{}=Ret0} -> + Ret = beam_ssa:add_anno(deallocate, Size, Ret0), + Blk0#b_blk{last=Ret}; + #b_blk{} -> + Blk0 + end, + Blocks = Blocks0#{L:=Blk}, + frame_deallocate(Ls, Size, Blocks); +frame_deallocate([], _, Blocks) -> Blocks. + + +%% turn_yregs(St0) -> St. +%% Renumber y registers so that {y,0} becomes {y,FrameSize-1}, +%% {y,FrameSize-1} becomes {y,0} and so on. This is to make nested +%% catches work. The register allocator (linear_scan()) has given +%% a lower number to the outermost catch. + +turn_yregs(#st{frames=Frames,regs=Regs0,ssa=Blocks}=St) -> + Regs1 = foldl(fun(L, A) -> + Blk = map_get(L, Blocks), + FrameSize = beam_ssa:get_anno(frame_size, Blk), + Def = beam_ssa:def([L], Blocks), + [turn_yregs_1(Def, FrameSize, Regs0)|A] + end, [], Frames), + Regs = maps:merge(Regs0, maps:from_list(append(Regs1))), + St#st{regs=Regs}. + +turn_yregs_1(Def, FrameSize, Regs) -> + Yregs0 = [{map_get(V, Regs),V} || V <- Def, is_yreg(map_get(V, Regs))], + Yregs1 = rel2fam(Yregs0), + FrameSize = length(Yregs1), + Yregs2 = [{{y,FrameSize-Y-1},Vs} || {{y,Y},Vs} <- Yregs1], + R0 = sofs:family(Yregs2), + R1 = sofs:family_to_relation(R0), + R = sofs:converse(R1), + sofs:to_external(R). + +%%% +%%% Reserving registers before register allocation. +%%% + +%% reserve_regs(St0) -> St. +%% Reserve registers prior to register allocation. Y registers +%% have already been reserved. This function will reserve z, +%% fr, and specific x registers. + +reserve_regs(#st{args=Args,ssa=Blocks,intervals=Intervals,res=Res0}=St) -> + %% Reserve x0, x1, and so on for the function arguments. + Res1 = reserve_arg_regs(Args, 0, Res0), + + %% Reserve Z registers (dummy registers) for instructions with no + %% return values (e.g. remove_message) or pseudo-return values + %% (e.g. landingpad). + Res2 = reserve_zregs(Blocks, Intervals, Res1), + + %% Reserve float registers. + Res3 = reserve_fregs(Blocks, Res2), + + %% Reserve all remaining unreserved variables as X registers. + Res = maps:from_list(Res3), + St#st{res=reserve_xregs(Blocks, Res)}. + +reserve_arg_regs([#b_var{}=Arg|Is], N, Acc) -> + reserve_arg_regs(Is, N+1, [{Arg,{x,N}}|Acc]); +reserve_arg_regs([], _, Acc) -> Acc. + +reserve_zregs(Blocks, Intervals, Res) -> + ShortLived0 = [V || {V,[{Start,End}]} <- Intervals, Start+2 =:= End], + ShortLived = cerl_sets:from_list(ShortLived0), + F = fun(_, #b_blk{is=Is,last=Last}, A) -> + reserve_zreg(Is, Last, ShortLived, A) + end, + beam_ssa:fold_rpo(F, [0], Res, Blocks). + +reserve_zreg([#b_set{op=Op,dst=Dst}], + #b_br{bool=Dst}, _ShortLived, A) when Op =:= call; + Op =:= get_tuple_element -> + %% If type optimization has determined that the result of these + %% instructions can be used directly in a branch, we must avoid reserving a + %% z register or code generation will fail. + A; +reserve_zreg([#b_set{op={bif,tuple_size},dst=Dst}, + #b_set{op={bif,'=:='},args=[Dst,Val]}], Last, ShortLived, A0) -> + case {Val,Last} of + {#b_literal{val=Arity},#b_br{bool=#b_var{}}} when Arity bsr 32 =:= 0 -> + %% These two instructions can be combined to a test_arity + %% instruction provided that the arity variable is short-lived. + reserve_zreg_1(Dst, ShortLived, A0); + {_,_} -> + %% Either the arity is too big, or the boolean value is not + %% used in a conditional branch. + A0 + end; +reserve_zreg([#b_set{op={bif,tuple_size},dst=Dst}], + #b_switch{}, ShortLived, A) -> + reserve_zreg_1(Dst, ShortLived, A); +reserve_zreg([#b_set{op={bif,'xor'}}], _Last, _ShortLived, A) -> + %% There is no short, easy way to rewrite 'xor' to a series of + %% test instructions. + A; +reserve_zreg([#b_set{op={bif,is_record}}], _Last, _ShortLived, A) -> + %% There is no short, easy way to rewrite is_record/2 to a series of + %% test instructions. + A; +reserve_zreg([#b_set{op=Op,dst=Dst}|Is], Last, ShortLived, A0) -> + IsZReg = case Op of + bs_match_string -> true; + bs_save -> true; + bs_restore -> true; + bs_set_position -> true; + {float,clearerror} -> true; + kill_try_tag -> true; + landingpad -> true; + put_tuple_elements -> true; + remove_message -> true; + set_tuple_element -> true; + succeeded -> true; + timeout -> true; + wait_timeout -> true; + _ -> false + end, + A = case IsZReg of + true -> [{Dst,z}|A0]; + false -> A0 + end, + reserve_zreg(Is, Last, ShortLived, A); +reserve_zreg([], #b_br{bool=Bool}, ShortLived, A) -> + reserve_zreg_1(Bool, ShortLived, A); +reserve_zreg([], _, _, A) -> A. + +reserve_zreg_1(#b_var{}=V, ShortLived, A) -> + case cerl_sets:is_element(V, ShortLived) of + true -> [{V,z}|A]; + false -> A + end; +reserve_zreg_1(#b_literal{}, _, A) -> A. + +reserve_fregs(Blocks, Res) -> + F = fun(_, #b_blk{is=Is}, A) -> + reserve_freg(Is, A) + end, + beam_ssa:fold_rpo(F, [0], Res, Blocks). + +reserve_freg([#b_set{op={float,Op},dst=V}|Is], Res) -> + case Op of + get -> + reserve_freg(Is, Res); + _ -> + reserve_freg(Is, [{V,fr}|Res]) + end; +reserve_freg([_|Is], Res) -> + reserve_freg(Is, Res); +reserve_freg([], Res) -> Res. + +%% reserve_xregs(St0) -> St. +%% Reserve all remaining variables as X registers. +%% +%% If a variable will need to be in a specific X register for a +%% 'call' or 'make_fun' (and there is nothing that will kill it +%% between the definition and use), reserve the register using a +%% {prefer,{x,X} annotation. That annotation means that the linear +%% scan algorithm will place the variable in the preferred register, +%% unless that register is already occupied. +%% +%% All remaining variables are reserved as X registers. Linear scan +%% will allocate the lowest free X register for the variable. + +reserve_xregs(Blocks, Res) -> + Ls = reverse(beam_ssa:rpo(Blocks)), + reserve_xregs(Ls, Blocks, #{}, Res). + +reserve_xregs([L|Ls], Blocks, XsMap0, Res0) -> + #b_blk{anno=Anno,is=Is0,last=Last} = map_get(L, Blocks), + + %% Calculate mapping from variable name to the preferred + %% register. + Xs0 = reserve_terminator(L, Is0, Last, Blocks, XsMap0, Res0), + + %% We need to figure out where the code generator will + %% place instructions that will do a garbage collection. + %% Insert 'gc' markers as pseudo-instructions in the + %% instruction sequence. + Is1 = reverse(Is0), + Is2 = res_place_gc_instrs(Is1, []), + Is = res_place_allocate(Anno, Is2), + + %% Add register hints for variables that are defined + %% in the (reversed) instruction sequence. + {Res,Xs} = reserve_xregs_is(Is, Res0, Xs0, []), + + XsMap = XsMap0#{L=>Xs}, + reserve_xregs(Ls, Blocks, XsMap, Res); +reserve_xregs([], _, _, Res) -> Res. + +%% Insert explicit 'gc' markers points where there will +%% be a garbage collection. (Note that the instruction +%% sequence passed to this function is reversed.) + +res_place_gc_instrs([#b_set{op=phi}=I|Is], Acc) -> + res_place_gc_instrs(Is, [I|Acc]); +res_place_gc_instrs([#b_set{op=Op}=I|Is], Acc) + when Op =:= call; Op =:= make_fun -> + case Acc of + [] -> + res_place_gc_instrs(Is, [I|Acc]); + [GC|_] when GC =:= gc; GC =:= test_heap -> + res_place_gc_instrs(Is, [I,gc|Acc]); + [_|_] -> + res_place_gc_instrs(Is, [I,gc|Acc]) + end; +res_place_gc_instrs([#b_set{op=Op,args=Args}=I|Is], Acc0) -> + case beam_ssa_codegen:classify_heap_need(Op, Args) of + neutral -> + case Acc0 of + [test_heap|Acc] -> + res_place_gc_instrs(Is, [test_heap,I|Acc]); + Acc -> + res_place_gc_instrs(Is, [I|Acc]) + end; + {put,_} -> + case Acc0 of + [test_heap|Acc] -> + res_place_gc_instrs(Is, [test_heap,I|Acc]); + Acc -> + res_place_gc_instrs(Is, [test_heap,I|Acc]) + end; + _ -> + res_place_gc_instrs(Is, [gc,I|Acc0]) + end; +res_place_gc_instrs([], Acc) -> + %% Reverse and replace 'test_heap' markers with 'gc'. + %% (The distinction is no longer useful.) + res_place_gc_instrs_rev(Acc, []). + +res_place_gc_instrs_rev([test_heap|Is], [gc|_]=Acc) -> + res_place_gc_instrs_rev(Is, Acc); +res_place_gc_instrs_rev([test_heap|Is], Acc) -> + res_place_gc_instrs_rev(Is, [gc|Acc]); +res_place_gc_instrs_rev([gc|Is], [gc|_]=Acc) -> + res_place_gc_instrs_rev(Is, Acc); +res_place_gc_instrs_rev([I|Is], Acc) -> + res_place_gc_instrs_rev(Is, [I|Acc]); +res_place_gc_instrs_rev([], Acc) -> Acc. + +res_place_allocate(#{yregs:=_}, Is) -> + %% There will be an 'allocate' instruction inserted here. + Is ++ [gc]; +res_place_allocate(#{}, Is) -> Is. + +reserve_xregs_is([gc|Is], Res, Xs0, Used) -> + %% At this point, the code generator will place an instruction + %% that does a garbage collection. We must prune the remembered + %% registers. + Xs = res_xregs_prune(Xs0, Used, Res), + reserve_xregs_is(Is, Res, Xs, Used); +reserve_xregs_is([#b_set{op=Op,dst=Dst,args=Args}=I|Is], Res0, Xs0, Used0) -> + Res = reserve_xreg(Dst, Xs0, Res0), + Used1 = ordsets:union(Used0, beam_ssa:used(I)), + Used = ordsets:del_element(Dst, Used1), + case Op of + call -> + Xs = reserve_call_args(tl(Args)), + reserve_xregs_is(Is, Res, Xs, Used); + make_fun -> + Xs = reserve_call_args(tl(Args)), + reserve_xregs_is(Is, Res, Xs, Used); + _ -> + reserve_xregs_is(Is, Res, Xs0, Used) + end; +reserve_xregs_is([], Res, Xs, _Used) -> + {Res,Xs}. + +%% Pick up register hints from the successors of this blocks. +reserve_terminator(_L, _Is, #b_br{bool=#b_var{},succ=Succ,fail=?BADARG_BLOCK}, + _Blocks, XsMap, _Res) -> + %% We know that no variables are used at ?BADARG_BLOCK, so + %% any register hints from the success blocks are safe to use. + map_get(Succ, XsMap); +reserve_terminator(L, Is, #b_br{bool=#b_var{},succ=Succ,fail=Fail}, + Blocks, XsMap, Res) when Succ =/= Fail -> + #{Succ:=SuccBlk,Fail:=FailBlk} = Blocks, + case {SuccBlk,FailBlk} of + {#b_blk{is=[],last=#b_br{succ=PhiL,fail=PhiL}}, + #b_blk{is=[],last=#b_br{succ=PhiL,fail=PhiL}}} -> + %% Both branches ultimately transfer to the same + %% block (via two blocks with no instructions). + %% Pick up register hints from the phi nodes + %% in the common block. + #{PhiL:=#b_blk{is=PhiIs}} = Blocks, + Xs = res_xregs_from_phi(PhiIs, Succ, Res, #{}), + res_xregs_from_phi(PhiIs, Fail, Res, Xs); + {_,_} when Is =/= [] -> + case last(Is) of + #b_set{op=succeeded,args=[Arg]} -> + %% We know that Arg will not be used at the failure + %% label, so we can pick up register hints from the + %% success label. + Br = #b_br{bool=#b_literal{val=true},succ=Succ,fail=Succ}, + case reserve_terminator(L, [], Br, Blocks, XsMap, Res) of + #{Arg:=Reg} -> #{Arg=>Reg}; + #{} -> #{} + end; + _ -> + %% Register hints from the success block may not + %% be safe at the failure block, and vice versa. + #{} + end; + {_,_} -> + %% Register hints from the success block may not + %% be safe at the failure block, and vice versa. + #{} + end; +reserve_terminator(L, Is, #b_br{bool=#b_literal{val=true},succ=Succ}, + Blocks, XsMap, Res) -> + case map_get(Succ, Blocks) of + #b_blk{is=[],last=Last} -> + reserve_terminator(Succ, Is, Last, Blocks, XsMap, Res); + #b_blk{is=[_|_]=PhiIs} -> + res_xregs_from_phi(PhiIs, L, Res, #{}) + end; +reserve_terminator(_, _, _, _, _, _) -> #{}. + +%% Pick up a reservation from a phi node. +res_xregs_from_phi([#b_set{op=phi,dst=Dst,args=Args}|Is], + Pred, Res, Acc) -> + case [V || {#b_var{}=V,L} <- Args, L =:= Pred] of + [] -> + %% The value of the phi node for this predecessor + %% is a literal. Nothing to do here. + res_xregs_from_phi(Is, Pred, Res, Acc); + [V] -> + case Res of + #{Dst:={prefer,Reg}} -> + %% Try placing V in the same register as for + %% the phi node. + res_xregs_from_phi(Is, Pred, Res, Acc#{V=>Reg}); + #{Dst:=_} -> + res_xregs_from_phi(Is, Pred, Res, Acc) + end + end; +res_xregs_from_phi(_, _, _, Acc) -> Acc. + +reserve_call_args(Args) -> + reserve_call_args(Args, 0, #{}). + +reserve_call_args([#b_var{}=Var|As], X, Xs) -> + reserve_call_args(As, X+1, Xs#{Var=>{x,X}}); +reserve_call_args([#b_literal{}|As], X, Xs) -> + reserve_call_args(As, X+1, Xs); +reserve_call_args([], _, Xs) -> Xs. + +reserve_xreg(V, Xs, Res) -> + case Res of + #{V:=_} -> + %% Already reserved (but not as an X register). + Res; + #{} -> + case Xs of + #{V:=X} -> + %% Add a hint that this specific X register is + %% preferred, unless it is already in use. + Res#{V=>{prefer,X}}; + #{} -> + %% Reserve as an X register in general. + Res#{V=>x} + end + end. + +%% res_xregs_prune(PreferredRegs, Used, Res) -> PreferredRegs. +%% Prune the list of preferred registers, to make sure that +%% there are no "holes" (uninitialized X registers) when +%% invoking the garbage collector. + +res_xregs_prune(Xs, Used, Res) when map_size(Xs) =/= 0 -> + %% The number of safe registers is the number of the X registers + %% used after this point. The actual number of safe registers may + %% be higher than this number, but this is a conservative safe + %% estimate. + NumSafe = foldl(fun(V, N) -> + case Res of + #{V:={x,_}} -> N + 1; + #{V:=_} -> N; + #{} -> N + 1 + end + end, 0, Used), + + %% Remove unsafe registers from the list of potential + %% preferred registers. + maps:filter(fun(_, {x,X}) -> X < NumSafe end, Xs); +res_xregs_prune(Xs, _Used, _Res) -> Xs. + +%%% +%%% Register allocation using linear scan. +%%% + +-record(i, + {sort=1 :: instr_number(), + reg=none :: i_reg(), + pool=x :: pool_id(), + var=#b_var{} :: b_var(), + rs=[] :: [range()] + }). + +-record(l, + {cur=#i{} :: interval(), + unhandled_res=[] :: [interval()], + unhandled_any=[] :: [interval()], + active=[] :: [interval()], + inactive=[] :: [interval()], + free=#{} :: #{var_name()=>pool(), + {'next',pool_id()}:=reg_num()}, + regs=[] :: [{b_var(),ssa_register()}] + }). + +-type interval() :: #i{}. +-type i_reg() :: ssa_register() | {'prefer',xreg()} | 'none'. +-type pool_id() :: 'fr' | 'x' | 'z' | instr_number(). +-type pool() :: ordsets:ordset(ssa_register()). + +linear_scan(#st{intervals=Intervals0,res=Res}=St0) -> + St = St0#st{intervals=[],res=[]}, + Free = init_free(maps:to_list(Res)), + Intervals1 = [init_interval(Int, Res) || Int <- Intervals0], + Intervals = sort(Intervals1), + IsReserved = fun(#i{reg=Reg}) -> + case Reg of + none -> false; + {prefer,{_,_}} -> false; + {_,_} -> true + end + end, + {UnhandledRes,Unhandled} = partition(IsReserved, Intervals), + L = #l{unhandled_res=UnhandledRes, + unhandled_any=Unhandled,free=Free}, + #l{regs=Regs} = do_linear(L), + St#st{regs=maps:from_list(Regs)}. + +init_interval({V,[{Start,_}|_]=Rs}, Res) -> + Info = map_get(V, Res), + Pool = case Info of + {prefer,{x,_}} -> x; + x -> x; + {x,_} -> x; + {y,Uniq} -> Uniq; + {{y,_},Uniq} -> Uniq; + z -> z; + fr -> fr + end, + Reg = case Info of + {prefer,{x,_}} -> Info; + {x,_} -> Info; + {{y,_}=Y,_} -> Y; + _ -> none + end, + #i{sort=Start,var=V,reg=Reg,pool=Pool,rs=Rs}. + +init_free(Res) -> + Free0 = rel2fam([{x,{x,0}}|init_free_1(Res)]), + #{x:=Xs0} = Free1 = maps:from_list(Free0), + Xs = init_xregs(Xs0), + Free = Free1#{x:=Xs}, + Next = maps:fold(fun(K, V, A) -> [{{next,K},length(V)}|A] end, [], Free), + maps:merge(Free, maps:from_list(Next)). + +init_free_1([{_,{prefer,{x,_}=Reg}}|Res]) -> + [{x,Reg}|init_free_1(Res)]; +init_free_1([{_,{x,_}=Reg}|Res]) -> + [{x,Reg}|init_free_1(Res)]; +init_free_1([{_,{y,Uniq}}|Res]) -> + [{Uniq,{y,0}}|init_free_1(Res)]; +init_free_1([{_,{{y,_}=Reg,Uniq}}|Res]) -> + [{Uniq,Reg}|init_free_1(Res)]; +init_free_1([{_,z}|Res]) -> + [{z,{z,0}}|init_free_1(Res)]; +init_free_1([{_,fr}|Res]) -> + [{fr,{fr,0}}|init_free_1(Res)]; +init_free_1([{_,x}|Res]) -> + init_free_1(Res); +init_free_1([]) -> []. + +%% Make sure that the pool of xregs is contiguous. +init_xregs([{x,N},{x,M}|Is]) when N+1 =:= M -> + [{x,N}|init_xregs([{x,M}|Is])]; +init_xregs([{x,N}|[{x,_}|_]=Is]) -> + [{x,N}|init_xregs([{x,N+1}|Is])]; +init_xregs([{x,_}]=Is) -> Is. + +do_linear(L0) -> + case set_next_current(L0) of + done -> + L0; + L1 -> + L2 = expire_active(L1), + L3 = check_inactive(L2), + Available = collect_available(L3), + L4 = select_register(Available, L3), + L = make_cur_active(L4), + do_linear(L) + end. + +set_next_current(#l{unhandled_res=[Cur1|T1], + unhandled_any=[Cur2|T2]}=L) -> + case {Cur1,Cur2} of + {#i{sort=N1},#i{sort=N2}} when N1 < N2 -> + L#l{cur=Cur1,unhandled_res=T1}; + {_,_} -> + L#l{cur=Cur2,unhandled_any=T2} + end; +set_next_current(#l{unhandled_res=[], + unhandled_any=[Cur|T]}=L) -> + L#l{cur=Cur,unhandled_any=T}; +set_next_current(#l{unhandled_res=[Cur|T], + unhandled_any=[]}=L) -> + L#l{cur=Cur,unhandled_res=T}; +set_next_current(#l{unhandled_res=[],unhandled_any=[]}) -> + done. + +expire_active(#l{cur=#i{sort=CurBegin},active=Act0}=L0) -> + {Act,L} = expire_active(Act0, CurBegin, L0, []), + L#l{active=Act}. + +expire_active([#i{reg=Reg,rs=Rs0}=I|Is], CurBegin, L0, Acc) -> + {_,_} = Reg, %Assertion. + case overlap_status(Rs0, CurBegin) of + ends_before_cur -> + L = free_reg(I, L0), + expire_active(Is, CurBegin, L, Acc); + overlapping -> + expire_active(Is, CurBegin, L0, [I|Acc]); + not_overlapping -> + Rs = strip_before_current(Rs0, CurBegin), + L1 = free_reg(I, L0), + L = L1#l{inactive=[I#i{rs=Rs}|L1#l.inactive]}, + expire_active(Is, CurBegin, L, Acc) + end; +expire_active([], _CurBegin, L, Acc) -> + {Acc,L}. + +check_inactive(#l{cur=#i{sort=CurBegin},inactive=InAct0}=L0) -> + {InAct,L} = check_inactive(InAct0, CurBegin, L0, []), + L#l{inactive=InAct}. + +check_inactive([#i{rs=Rs0}=I|Is], CurBegin, L0, Acc) -> + case overlap_status(Rs0, CurBegin) of + ends_before_cur -> + check_inactive(Is, CurBegin, L0, Acc); + not_overlapping -> + check_inactive(Is, CurBegin, L0, [I|Acc]); + overlapping -> + Rs = strip_before_current(Rs0, CurBegin), + L1 = L0#l{active=[I#i{rs=Rs}|L0#l.active]}, + L = reserve_reg(I, L1), + check_inactive(Is, CurBegin, L, Acc) + end; +check_inactive([], _CurBegin, L, Acc) -> + {Acc,L}. + +strip_before_current([{_,E}|Rs], CurBegin) when E =< CurBegin -> + strip_before_current(Rs, CurBegin); +strip_before_current(Rs, _CurBegin) -> Rs. + +collect_available(#l{cur=#i{reg={prefer,{_,_}=Prefer}}=I}=L) -> + %% Use the preferred register if it is available. + Avail = collect_available(L#l{cur=I#i{reg=none}}), + case member(Prefer, Avail) of + true -> [Prefer]; + false -> Avail + end; +collect_available(#l{cur=#i{reg={_,_}=ReservedReg}}) -> + %% Return the already reserved register. + [ReservedReg]; +collect_available(#l{unhandled_res=Unhandled,cur=Cur}=L) -> + Free = get_pool(Cur, L), + + %% Note that since the live intervals are constructed from + %% SSA form, there cannot be any overlap of the current interval + %% with any inactive interval. See [3], page 175. Therefore we + %% only have check the unhandled intervals for overlap with + %% the current interval. As a further optimization, we only need + %% to check the intervals that have reserved registers. + collect_available(Unhandled, Cur, Free). + +collect_available([#i{pool=Pool1}|Is], #i{pool=Pool2}=Cur, Free) + when Pool1 =/= Pool2 -> + %% Wrong pool. Ignore this interval. + collect_available(Is, Cur, Free); +collect_available([#i{reg={_,_}=Reg}=I|Is], Cur, Free0) -> + case overlaps(I, Cur) of + true -> + Free = ordsets:del_element(Reg, Free0), + collect_available(Is, Cur, Free); + false -> + collect_available(Is, Cur, Free0) + end; +collect_available([], _, Free) -> Free. + +select_register([{_,_}=Reg|_], #l{cur=Cur0,regs=Regs}=L) -> + Cur = Cur0#i{reg=Reg}, + reserve_reg(Cur, L#l{cur=Cur,regs=[{Cur#i.var,Reg}|Regs]}); +select_register([], #l{cur=Cur0,regs=Regs}=L0) -> + %% Allocate a new register in the pool. + {Reg,L1} = get_next_free(Cur0, L0), + Cur = Cur0#i{reg=Reg}, + L = L1#l{cur=Cur,regs=[{Cur#i.var,Reg}|Regs]}, + reserve_reg(Cur, L). + +make_cur_active(#l{cur=Cur,active=Act}=L) -> + L#l{active=[Cur|Act]}. + +overlaps(#i{rs=Rs1}, #i{rs=Rs2}) -> + are_overlapping(Rs1, Rs2). + +overlap_status([{S,E}], CurBegin) -> + if + E =< CurBegin -> ends_before_cur; + CurBegin < S -> not_overlapping; + true -> overlapping + end; +overlap_status([{S,E}|Rs], CurBegin) -> + if + E =< CurBegin -> + overlap_status(Rs, CurBegin); + S =< CurBegin -> + overlapping; + true -> + not_overlapping + end. + +reserve_reg(#i{reg={_,_}=Reg}=I, L) -> + FreeRegs0 = get_pool(I, L), + FreeRegs = ordsets:del_element(Reg, FreeRegs0), + update_pool(I, FreeRegs, L). + +free_reg(#i{reg={_,_}=Reg}=I, L) -> + FreeRegs0 = get_pool(I, L), + FreeRegs = ordsets:add_element(Reg, FreeRegs0), + update_pool(I, FreeRegs, L). + +get_pool(#i{pool=Pool}, #l{free=Free}) -> + map_get(Pool, Free). + +update_pool(#i{pool=Pool}, New, #l{free=Free0}=L) -> + Free = Free0#{Pool:=New}, + L#l{free=Free}. + +get_next_free(#i{pool=Pool}, #l{free=Free0}=L0) -> + K = {next,Pool}, + N = map_get(K, Free0), + Free = Free0#{K:=N+1}, + L = L0#l{free=Free}, + if + is_integer(Pool) -> {{y,N},L}; + is_atom(Pool) -> {{Pool,N},L} + end. + +%%% +%%% Interval utilities. +%%% + +are_overlapping([R|Rs1], Rs2) -> + case are_overlapping_1(R, Rs2) of + true -> + true; + false -> + are_overlapping(Rs1, Rs2) + end; +are_overlapping([], _) -> false. + +are_overlapping_1({_S1,E1}, [{S2,_E2}|_]) when E1 < S2 -> + false; +are_overlapping_1({S1,E1}=R, [{S2,E2}|Rs]) -> + (S2 < E1 andalso E2 > S1) orelse are_overlapping_1(R, Rs); +are_overlapping_1({_,_}, []) -> false. + +%%% +%%% Utilities. +%%% + +%% is_loop_header(L, Blocks) -> false|true. +%% Check whether the block is a loop header. + +is_loop_header(L, Blocks) -> + %% We KNOW that a loop header must start with a peek_message + %% instruction. + case map_get(L, Blocks) of + #b_blk{is=[#b_set{op=peek_message}|_]} -> true; + _ -> false + end. + +rel2fam(S0) -> + S1 = sofs:relation(S0), + S = sofs:rel2fam(S1), + sofs:to_external(S). + +split_phis(Is) -> + splitwith(fun(#b_set{op=Op}) -> Op =:= phi end, Is). + +is_yreg({y,_}) -> true; +is_yreg({x,_}) -> false; +is_yreg({z,_}) -> false; +is_yreg({fr,_}) -> false. + +new_vars([Base|Vs0], Count0) -> + {V,Count1} = new_var(Base, Count0), + {Vs,Count} = new_vars(Vs0, Count1), + {[V|Vs],Count}; +new_vars([], Count) -> {[],Count}. + +new_var({Base,Int}, Count) -> + true = is_integer(Int), %Assertion. + {#b_var{name={Base,Count}},Count+1}; +new_var(Base, Count) -> + {#b_var{name={Base,Count}},Count+1}. diff --git a/lib/compiler/src/beam_ssa_recv.erl b/lib/compiler/src/beam_ssa_recv.erl new file mode 100644 index 0000000000..1e0e1ecac2 --- /dev/null +++ b/lib/compiler/src/beam_ssa_recv.erl @@ -0,0 +1,278 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(beam_ssa_recv). +-export([module/2]). + +%%% +%%% In code such as: +%%% +%%% Ref = make_ref(), %Or erlang:monitor(process, Pid) +%%% . +%%% . +%%% . +%%% receive +%%% {Ref,Reply} -> Reply +%%% end. +%%% +%%% we know that none of the messages that exist in the message queue +%%% before the call to make_ref/0 can be matched out in the receive +%%% statement. Therefore we can avoid going through the entire message +%%% queue if we introduce two new instructions (here written as +%%% BIFs in pseudo-Erlang): +%%% +%%% recv_mark(SomeUniqInteger), +%%% Ref = make_ref(), +%%% . +%%% . +%%% . +%%% recv_set(SomeUniqInteger), +%%% receive +%%% {Ref,Reply} -> Reply +%%% end. +%%% +%%% The recv_mark/1 instruction will save the current position and +%%% SomeUniqInteger in the process context. The recv_set +%%% instruction will verify that SomeUniqInteger is still stored +%%% in the process context. If it is, it will set the current pointer +%%% for the message queue (the next message to be read out) to the +%%% position that was saved by recv_mark/1. +%%% +%%% The remove_message instruction must be modified to invalidate +%%% the information stored by the previous recv_mark/1, in case there +%%% is another receive executed between the calls to recv_mark/1 and +%%% recv_set/1. +%%% +%%% We use a reference to a label (i.e. a position in the loaded code) +%%% as the SomeUniqInteger. +%%% + +-include("beam_ssa.hrl"). +-import(lists, [all/2,reverse/2]). + +-spec module(beam_ssa:b_module(), [compile:option()]) -> + {'ok',beam_ssa:b_module()}. + +module(#b_module{body=Fs0}=Module, _Opts) -> + Fs = [function(F) || F <- Fs0], + {ok,Module#b_module{body=Fs}}. + +%%% +%%% Local functions. +%%% + +function(#b_function{anno=Anno,bs=Blocks0}=F) -> + try + Blocks = opt(Blocks0), + F#b_function{bs=Blocks} + catch + Class:Error:Stack -> + #{func_info:={_,Name,Arity}} = Anno, + io:fwrite("Function: ~w/~w\n", [Name,Arity]), + erlang:raise(Class, Error, Stack) + end. + +opt(Blocks) -> + Linear = beam_ssa:linearize(Blocks), + opt(Linear, Blocks, []). + +opt([{L,#b_blk{is=[#b_set{op=peek_message}|_]}=Blk0}|Bs], Blocks0, Preds) -> + %% Search for a suitable reference creating call in one of the predecessor + %% blocks. Whether we find such a call or not, we always clear the + %% the list of predecessors to ensure that any nested receive can't + %% search above the current receive. + case recv_opt(Preds, L, Blocks0) of + {yes,Blocks1} -> + Blk = beam_ssa:add_anno(recv_set, L, Blk0), + Blocks = Blocks1#{L:=Blk}, + opt(Bs, Blocks, []); + no -> + opt(Bs, Blocks0, []) + end; +opt([{L,_}|Bs], Blocks, Preds) -> + opt(Bs, Blocks, [L|Preds]); +opt([], Blocks, _) -> Blocks. + +recv_opt([L|Ls], RecvLbl, Blocks) -> + #b_blk{is=Is0} = Blk0 = map_get(L, Blocks), + case recv_opt_is(Is0, RecvLbl, Blocks, []) of + {yes,Is} -> + Blk = Blk0#b_blk{is=Is}, + {yes,Blocks#{L:=Blk}}; + no -> + recv_opt(Ls, RecvLbl, Blocks) + end; +recv_opt([], _, _Blocks) -> no. + +recv_opt_is([#b_set{op=call}=I0|Is], RecvLbl, Blocks0, Acc) -> + case makes_ref(I0, Blocks0) of + no -> + recv_opt_is(Is, RecvLbl, Blocks0, [I0|Acc]); + {yes,Ref} -> + case opt_ref_used(RecvLbl, Ref, Blocks0) of + false -> + recv_opt_is(Is, RecvLbl, Blocks0, [I0|Acc]); + true -> + I = beam_ssa:add_anno(recv_mark, RecvLbl, I0), + {yes,reverse(Acc, [I|Is])} + end + end; +recv_opt_is([I|Is], RecvLbl, Blocks, Acc) -> + recv_opt_is(Is, RecvLbl, Blocks, [I|Acc]); +recv_opt_is([], _, _, _) -> no. + +makes_ref(#b_set{dst=Dst,args=[Func0|_]}, Blocks) -> + Func = case Func0 of + #b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=Name},arity=A0} -> + {Name,A0}; + _ -> + none + end, + case Func of + {make_ref,0} -> + {yes,Dst}; + {monitor,2} -> + {yes,Dst}; + {spawn_monitor,A} when A =:= 1; A =:= 3 -> + ref_in_tuple(Dst, Blocks); + _ -> + no + end. + +ref_in_tuple(Tuple, Blocks) -> + F = fun(#b_set{op=get_tuple_element,dst=Ref, + args=[#b_var{}=Tup,#b_literal{val=1}]}, no) + when Tup =:= Tuple -> {yes,Ref}; + (_, A) -> A + end, + beam_ssa:fold_instrs_rpo(F, [0], no, Blocks). + +opt_ref_used(RecvLbl, Ref, Blocks) -> + Vs = #{Ref=>ref,ref=>Ref,ref_matched=>false}, + case opt_ref_used_1(RecvLbl, Vs, Blocks) of + used -> true; + not_used -> false; + done -> false + end. + +opt_ref_used_1(L, Vs0, Blocks) -> + #b_blk{is=Is} = Blk = map_get(L, Blocks), + case opt_ref_used_is(Is, Vs0) of + #{}=Vs -> + opt_ref_used_last(Blk, Vs, Blocks); + Result -> + Result + end. + +opt_ref_used_is([#b_set{op=peek_message,dst=Msg}|Is], Vs0) -> + Vs = Vs0#{Msg=>message}, + opt_ref_used_is(Is, Vs); +opt_ref_used_is([#b_set{op={bif,Bif},args=Args,dst=Dst}=I|Is], + Vs0) -> + S = case Bif of + '=:=' -> true; + '==' -> true; + _ -> none + end, + case S of + none -> + Vs = update_vars(I, Vs0), + opt_ref_used_is(Is, Vs); + Bool when is_boolean(Bool) -> + case is_ref_msg_comparison(Args, Vs0) of + true -> + Vs = Vs0#{Dst=>{is_ref,Bool}}, + opt_ref_used_is(Is, Vs); + false -> + opt_ref_used_is(Is, Vs0) + end + end; +opt_ref_used_is([#b_set{op=remove_message}|_], Vs) -> + case Vs of + #{ref_matched:=true} -> + used; + #{ref_matched:=false} -> + not_used + end; +opt_ref_used_is([#b_set{op=recv_next}|_], _Vs) -> + done; +opt_ref_used_is([#b_set{op=wait_timeout}|_], _Vs) -> + done; +opt_ref_used_is([#b_set{op=wait}|_], _Vs) -> + done; +opt_ref_used_is([#b_set{}=I|Is], Vs0) -> + Vs = update_vars(I, Vs0), + opt_ref_used_is(Is, Vs); +opt_ref_used_is([], Vs) -> Vs. + +opt_ref_used_last(#b_blk{last=Last}=Blk, Vs, Blocks) -> + case Last of + #b_br{bool=#b_var{}=Bool,succ=Succ,fail=Fail} -> + case Vs of + #{Bool:={is_ref,Matched}} -> + ref_used_in([{Succ,Vs#{ref_matched:=Matched}}, + {Fail,Vs#{ref_matched:=not Matched}}], + Blocks); + #{} -> + ref_used_in([{Succ,Vs},{Fail,Vs}], Blocks) + end; + _ -> + SuccVs = [{Succ,Vs} || Succ <- beam_ssa:successors(Blk)], + ref_used_in(SuccVs, Blocks) + end. + +ref_used_in([{L,Vs0}|Ls], Blocks) -> + case opt_ref_used_1(L, Vs0, Blocks) of + not_used -> + not_used; + used -> + case ref_used_in(Ls, Blocks) of + done -> used; + Result -> Result + end; + done -> ref_used_in(Ls, Blocks) + end; +ref_used_in([], _) -> done. + +update_vars(#b_set{args=Args,dst=Dst}, Vs) -> + Vars = [V || #b_var{}=V <- Args], + All = all(fun(Var) -> + case Vs of + #{Var:=message} -> true; + #{} -> false + end + end, Vars), + case All of + true -> Vs#{Dst=>message}; + false -> Vs + end. + +%% is_ref_msg_comparison(Args, Variables) -> true|false. +%% Return 'true' if Args denotes a comparison between the +%% reference and message or part of the message. + +is_ref_msg_comparison([#b_var{}=V1,#b_var{}=V2], Vs) -> + case Vs of + #{V1:=ref,V2:=message} -> true; + #{V1:=message,V2:=ref} -> true; + #{} -> false + end; +is_ref_msg_comparison(_, _) -> false. diff --git a/lib/compiler/src/beam_ssa_share.erl b/lib/compiler/src/beam_ssa_share.erl new file mode 100644 index 0000000000..426efa2cc9 --- /dev/null +++ b/lib/compiler/src/beam_ssa_share.erl @@ -0,0 +1,370 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Share code for semantically equivalent blocks referred to +%% to by `br` and `switch` instructions. +%% +%% A similar optimization is done in beam_jump, but doing it here as +%% well is beneficial as it may enable other optimizations. If there +%% are many semantically equivalent clauses, this optimization can +%% substanstially decrease compilation times. +%% +%% block/2 is called from the liveness optimization pass in +%% beam_ssa_opt, as code sharing helps the liveness pass and vice +%% versa. +%% + +-module(beam_ssa_share). +-export([module/2,block/2]). + +-include("beam_ssa.hrl"). + +-import(lists, [keyfind/3,reverse/1,sort/1]). + +-spec module(beam_ssa:b_module(), [compile:option()]) -> + {'ok',beam_ssa:b_module()}. + +module(#b_module{body=Fs0}=Module, _Opts) -> + Fs = [function(F) || F <- Fs0], + {ok,Module#b_module{body=Fs}}. + +-spec block(Blk0, Blocks0) -> Blk when + Blk0 :: beam_ssa:b_blk(), + Blocks0 :: beam_ssa:block_map(), + Blk :: beam_ssa:b_blk(). + +block(#b_blk{last=Last0}=Blk, Blocks) -> + case share_terminator(Last0, Blocks) of + none -> Blk; + Last -> Blk#b_blk{last=beam_ssa:normalize(Last)} + end. + +%%% +%%% Local functions. +%%% + +function(#b_function{anno=Anno,bs=Blocks0}=F) -> + try + PO = reverse(beam_ssa:rpo(Blocks0)), + {Blocks1,Changed} = blocks(PO, Blocks0, false), + Blocks = case Changed of + true -> + beam_ssa:trim_unreachable(Blocks1); + false -> + Blocks0 + end, + F#b_function{bs=Blocks} + catch + Class:Error:Stack -> + #{func_info:={_,Name,Arity}} = Anno, + io:fwrite("Function: ~w/~w\n", [Name,Arity]), + erlang:raise(Class, Error, Stack) + end. + +blocks([L|Ls], Blocks, Changed) -> + #b_blk{last=Last0} = Blk0 = map_get(L, Blocks), + case block(Blk0, Blocks) of + #b_blk{last=Last0} -> + blocks(Ls, Blocks, Changed); + #b_blk{}=Blk -> + blocks(Ls, Blocks#{L:=Blk}, true) + end; +blocks([], Blocks, Changed) -> + {Blocks,Changed}. + +share_terminator(#b_br{bool=#b_var{},succ=Succ0,fail=Fail0}=Br, Blocks) -> + {Succ,SuccBlk} = shortcut_nonempty_block(Succ0, Blocks), + {Fail,FailBlk} = shortcut_nonempty_block(Fail0, Blocks), + case are_equivalent(Succ, SuccBlk, Fail, FailBlk, Blocks) of + true -> + %% The blocks are semantically equivalent. + Br#b_br{succ=Succ,fail=Succ}; + false -> + if + Succ =:= Succ0, Fail =:= Fail0 -> + %% None of blocks were cut short. + none; + true -> + %% One or both labels were cut short + %% to avoid jumping to an empty block. + Br#b_br{succ=Succ,fail=Fail} + end + end; +share_terminator(#b_switch{}=Sw, Blocks) -> + share_switch(Sw, Blocks); +share_terminator(_Last, _Blocks) -> none. + +%% Test whether the two blocks are semantically equivalent. This +%% function is specially optimized to return `false` as fast as +%% possible if the blocks are not equivalent, as that is the common +%% case. + +are_equivalent(_Succ, _, ?BADARG_BLOCK, _, _Blocks) -> + %% ?BADARG_BLOCK is special. Sharing could be incorrect. + false; +are_equivalent(_Succ, #b_blk{is=Is1,last=#b_ret{arg=RetVal1}=Ret1}, + _Fail, #b_blk{is=Is2,last=#b_ret{arg=RetVal2}=Ret2}, _Blocks) -> + case {RetVal1,RetVal2} of + {#b_literal{},#b_literal{}} -> + case RetVal1 =:= RetVal2 of + true -> + %% The return values are identical literals. We + %% only need to compare the canonicalized bodies. + Can1 = canonical_is(Is1), + Can2 = canonical_is(Is2), + Can1 =:= Can2; + false -> + %% Non-equal literals. + false + end; + {#b_var{},#b_var{}} -> + %% The return values are varibles. We must canonicalize + %% the blocks (including returns) and compare them. + Can1 = canonical_is(Is1 ++ [Ret1]), + Can2 = canonical_is(Is2 ++ [Ret2]), + Can1 =:= Can2; + {_,_} -> + %% One literal and one variable. + false + end; +are_equivalent(Succ, + #b_blk{is=Is1, + last=#b_br{bool=#b_literal{val=true}, + succ=Target}}, + Fail, + #b_blk{is=Is2, + last=#b_br{bool=#b_literal{val=true}, + succ=Target}}, + Blocks) -> + %% Both blocks end with an unconditional branch to the + %% same target block. If the target block has phi nodes, + %% we must pick up the values from the phi nodes and + %% compare them. + #b_blk{is=Is} = map_get(Target, Blocks), + Phis1 = canonical_terminator_phis(Is, Succ), + Phis2 = canonical_terminator_phis(Is, Fail), + case {Phis1,Phis2} of + {[#b_set{args=[#b_literal{}]}|_],_} when Phis1 =/= Phis2 -> + %% Different values are used in the phi nodes. + false; + {_,[#b_set{args=[#b_literal{}]}|_]} when Phis1 =/= Phis2 -> + %% Different values are used in the phi nodes. + false; + {_,_} -> + %% The values in the phi nodes are variables or identical + %% literals. We must canonicalize the blocks and compare + %% them. + Can1 = canonical_is(Is1 ++ Phis1), + Can2 = canonical_is(Is2 ++ Phis2), + Can1 =:= Can2 + end; +are_equivalent(Succ0, #b_blk{is=Is1,last=#b_br{bool=#b_var{},fail=Same}}, + Fail0, #b_blk{is=Is2,last=#b_br{bool=#b_var{},fail=Same}}, + Blocks) -> + %% Two-way branches with identical failure labels. First compare the + %% canonicalized bodies of the blocks. + case canonical_is(Is1) =:= canonical_is(Is2) of + false -> + %% Different bodies. + false; + true -> + %% Bodies were equal. That is fairly uncommon, so to keep + %% the code simple we will rewrite the `br` to a `switch` + %% and let share_switch/2 do the work of following the + %% branches. + Sw = #b_switch{arg=#b_var{name=not_used},fail=Fail0, + list=[{#b_literal{},Succ0}]}, + #b_switch{fail=Fail,list=[{_,Succ}]} = share_switch(Sw, Blocks), + Fail =:= Succ + end; +are_equivalent(_, _, _, _, _) -> false. + +share_switch(#b_switch{fail=Fail0,list=List0}=Sw, Blocks) -> + Prep = share_prepare_sw([{value,Fail0}|List0], Blocks, 0, []), + Res = do_share_switch(Prep, Blocks, []), + [{_,Fail}|List] = [VL || {_,VL} <- sort(Res)], + Sw#b_switch{fail=Fail,list=List}. + +share_prepare_sw([{V,L0}|T], Blocks, N, Acc) -> + {L,_Blk} = shortcut_nonempty_block(L0, Blocks), + share_prepare_sw(T, Blocks, N+1, [{{L,#{}},{N,{V,L}}}|Acc]); +share_prepare_sw([], _, _, Acc) -> Acc. + +do_share_switch(Prep, Blocks, Acc) -> + Map = share_switch_1(Prep, Blocks, #{}), + share_switch_2(maps:values(Map), Blocks, Acc). + +share_switch_1([{Next0,Res}|T], Blocks, Map) -> + {Can,Next} = canonical_block(Next0, Blocks), + case Map of + #{Can:=Ls} -> + share_switch_1(T, Blocks, Map#{Can:=[{Next,Res}|Ls]}); + #{} -> + share_switch_1(T, Blocks, Map#{Can=>[{Next,Res}]}) + end; +share_switch_1([], _Blocks, Map) -> Map. + +share_switch_2([[{_,{N,Res}}]|T], Blocks, Acc) -> + %% This block is not equivalent to any other block. + share_switch_2(T, Blocks, [{N,Res}|Acc]); +share_switch_2([[{done,{_,{_,Common}}}|_]=Eqs|T], Blocks, Acc0) -> + %% Two or more blocks are semantically equivalent, and all blocks + %% are either terminated with a `ret` or a `br` to the same target + %% block. Replace the labels in the `switch` for all of those + %% blocks with the label for the first of the blocks. + Acc = [{N,{V,Common}} || {done,{N,{V,_}}} <- Eqs] ++ Acc0, + share_switch_2(T, Blocks, Acc); +share_switch_2([[{_,_}|_]=Prep|T], Blocks, Acc0) -> + %% Two or more blocks are semantically equivalent, but they have + %% different successful successor blocks. Now we must check + %% recursively whether the successor blocks are equivalent too. + Acc = do_share_switch(Prep, Blocks, Acc0), + share_switch_2(T, Blocks, Acc); +share_switch_2([], _, Acc) -> Acc. + +canonical_block({L,VarMap0}, Blocks) -> + #b_blk{is=Is,last=Last0} = map_get(L, Blocks), + case canonical_terminator(L, Last0, Blocks) of + none -> + %% The block has a terminator that we don't handle. + {{none,L},done}; + {Last,done} -> + %% The block ends with a `ret` or an unconditional `br` to + %% another block. + {Can,_VarMap} = canonical_is(Is ++ Last, VarMap0, []), + {Can,done}; + {Last,Next} -> + %% The block ends with a conditional branch. + {Can,VarMap} = canonical_is(Is ++ Last, VarMap0, []), + {Can,{Next,VarMap}} + end. + +%% Translate a sequence of instructions to a canonical representation. If the +%% canonical representation of two blocks compare equal, the blocks are +%% semantically equivalent. The following translations are done: +%% +%% * Variables defined in the instruction sequence are replaced with +%% {var,0}, {var,1}, and so on. Free variables are not changed. +%% +%% * `location` annotations that would produce a `line` instruction are +%% kept. All other annotations are cleared. +%% +%% * Instructions are repackaged into tuples instead of into the +%% usual records. The main reason is to avoid violating the types for +%% the SSA records. We can simplify things a little by linking the +%% instructions directly instead of putting them into a list. + +canonical_is(Is) -> + {Can,_} = canonical_is(Is, #{}, []), + Can. + +canonical_is([#b_set{op=Op,dst=Dst,args=Args0}=I|Is], VarMap0, Acc) -> + Args = [canonical_arg(Arg, VarMap0) || Arg <-Args0], + Var = {var,map_size(VarMap0)}, + VarMap = VarMap0#{Dst=>Var}, + LineAnno = case Op of + bs_match -> + %% The location annotation for a bs_match instruction + %% is only used in warnings, never to emit a `line` + %% instruction. Therefore, it should not be included. + []; + _ -> + %% The location annotation will be used in a `line` + %% instruction. It must be included. + beam_ssa:get_anno(location, I, none) + end, + canonical_is(Is, VarMap, {Op,LineAnno,Var,Args,Acc}); +canonical_is([#b_ret{arg=Arg}], VarMap, Acc0) -> + Acc1 = case Acc0 of + {call,_Anno,Var,[#b_local{}|_]=Args,PrevAcc} -> + %% This is a tail-recursive call to a local function. + %% There will be no line instruction generated; + %% thus, the annotation is not significant. + {call,[],Var,Args,PrevAcc}; + _ -> + Acc0 + end, + {{ret,canonical_arg(Arg, VarMap),Acc1},VarMap}; +canonical_is([#b_br{bool=#b_var{},fail=Fail}], VarMap, Acc) -> + {{br,succ,Fail,Acc},VarMap}; +canonical_is([#b_br{succ=Succ}], VarMap, Acc) -> + {{br,Succ,Acc},VarMap}; +canonical_is([], VarMap, Acc) -> + {Acc,VarMap}. + +canonical_terminator(_L, #b_ret{}=Ret, _Blocks) -> + {[Ret],done}; +canonical_terminator(L, #b_br{bool=#b_literal{val=true},succ=Succ}=Br, Blocks) -> + #b_blk{is=Is} = map_get(Succ, Blocks), + case canonical_terminator_phis(Is, L) of + [] -> + {[],Succ}; + [_|_]=Phis -> + {Phis ++ [Br],done} + end; +canonical_terminator(_L, #b_br{bool=#b_var{},succ=Succ}=Br, _Blocks) -> + {[Br],Succ}; +canonical_terminator(_, _, _) -> none. + +canonical_terminator_phis([#b_set{op=phi,args=PhiArgs}=Phi|Is], L) -> + {Value,L} = keyfind(L, 2, PhiArgs), + [Phi#b_set{op=copy,args=[Value]}|canonical_terminator_phis(Is, L)]; +canonical_terminator_phis([#b_set{op=peek_message}=I|_], L) -> + %% We could get stuck into an infinite loop if we allowed the + %% comparisons to continue into this block. Force an unequal + %% compare with all other predecessors of this block. + [I#b_set{op=copy,args=[#b_literal{val=L}]}]; +canonical_terminator_phis(_, _) -> []. + +canonical_arg(#b_var{}=Var, VarMap) -> + case VarMap of + #{Var:=CanonicalVar} -> + CanonicalVar; + #{} -> + Var + end; +canonical_arg(#b_remote{mod=Mod,name=Name}, VarMap) -> + {remote,canonical_arg(Mod, VarMap), + canonical_arg(Name, VarMap)}; +canonical_arg(Other, _VarMap) -> Other. + +%% Shortcut branches to empty blocks if safe. + +shortcut_nonempty_block(L, Blocks) -> + case map_get(L, Blocks) of + #b_blk{is=[],last=#b_br{bool=#b_literal{val=true},succ=Succ}}=Blk -> + %% This block is empty. + case is_forbidden(Succ, Blocks) of + false -> + shortcut_nonempty_block(Succ, Blocks); + true -> + {L,Blk} + end; + #b_blk{}=Blk -> + {L,Blk} + end. + +is_forbidden(L, Blocks) -> + case map_get(L, Blocks) of + #b_blk{is=[#b_set{op=phi}|_]} -> true; + #b_blk{is=[#b_set{op=peek_message}|_]} -> true; + #b_blk{} -> false + end. diff --git a/lib/compiler/src/beam_ssa_type.erl b/lib/compiler/src/beam_ssa_type.erl new file mode 100644 index 0000000000..06b42f1928 --- /dev/null +++ b/lib/compiler/src/beam_ssa_type.erl @@ -0,0 +1,1956 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(beam_ssa_type). +-export([opt_start/4, opt_continue/4, opt_finish/3]). + +-include("beam_ssa_opt.hrl"). +-import(lists, [all/2,any/2,droplast/1,foldl/3,last/1,member/2, + keyfind/3,partition/2,reverse/1,reverse/2, + seq/2,sort/1,split/2]). + +-define(UNICODE_INT, #t_integer{elements={0,16#10FFFF}}). + +-record(d, + {ds :: #{beam_ssa:b_var():=beam_ssa:b_set()}, + ls :: #{beam_ssa:label():=type_db()}, + once :: cerl_sets:set(beam_ssa:b_var()), + func_id :: func_id(), + func_db :: func_info_db(), + sub = #{} :: #{beam_ssa:b_var():=beam_ssa:value()}, + ret_type = [] :: [type()]}). + +-define(ATOM_SET_SIZE, 5). + +%% Records that represent type information. +-record(t_atom, {elements=any :: 'any' | [atom()]}). +-record(t_integer, {elements=any :: 'any' | {integer(),integer()}}). +-record(t_bs_match, {type :: type()}). +-record(t_tuple, {size=0 :: integer(), + exact=false :: boolean(), + %% Known element types (1-based index), unknown elements are + %% are assumed to be 'any'. + elements=#{} :: #{ non_neg_integer() => type() }}). + +-type type() :: 'any' | 'none' | + #t_atom{} | #t_integer{} | #t_bs_match{} | #t_tuple{} | + {'binary',pos_integer()} | 'cons' | 'float' | 'list' | 'map' | 'nil' | 'number'. +-type type_db() :: #{beam_ssa:var_name():=type()}. + +-spec opt_start(Linear, Args, Anno, FuncDb) -> {Linear, FuncDb} when + Linear :: [{non_neg_integer(), beam_ssa:b_blk()}], + Args :: [beam_ssa:b_var()], + Anno :: beam_ssa:anno(), + FuncDb :: func_info_db(). +opt_start(Linear, Args, Anno, FuncDb) -> + %% This is the first run through the module, so our arg_types can be + %% incomplete as we may not have visited all call sites at least once. + Ts = maps:from_list([{V,any} || #b_var{}=V <- Args]), + opt_continue_1(Linear, Args, get_func_id(Anno), Ts, FuncDb). + +-spec opt_continue(Linear, Args, Anno, FuncDb) -> {Linear, FuncDb} when + Linear :: [{non_neg_integer(), beam_ssa:b_blk()}], + Args :: [beam_ssa:b_var()], + Anno :: beam_ssa:anno(), + FuncDb :: func_info_db(). +opt_continue(Linear, Args, Anno, FuncDb) -> + Id = get_func_id(Anno), + case FuncDb of + #{ Id := #func_info{exported=false,arg_types=ArgTypes} } -> + %% This is a local function and we're guaranteed to have visited + %% every call site at least once, so we know that the parameter + %% types are at least as narrow as the join of all argument types. + Ts = join_arg_types(Args, ArgTypes, Anno), + opt_continue_1(Linear, Args, Id, Ts, FuncDb); + #{} -> + %% We can't infer the parameter types of exported functions, nor + %% the ones where module-level optimization is disabled, but + %% running the pass again could still help other functions. + Ts = maps:from_list([{V,any} || #b_var{}=V <- Args]), + opt_continue_1(Linear, Args, Id, Ts, FuncDb) + end. + +join_arg_types(Args, ArgTypes, Anno) -> + %% We suppress type optimization for parameters that have already been + %% optimized by another pass, as they may have done things we have no idea + %% how to interpret and running them over could generate incorrect code. + ParamTypes = maps:get(parameter_type_info, Anno, #{}), + Ts0 = join_arg_types_1(Args, ArgTypes, #{}), + maps:fold(fun(Arg, _V, Ts) -> + maps:put(Arg, any, Ts) + end, Ts0, ParamTypes). + +join_arg_types_1([Arg | Args], [TM | TMs], Ts) when map_size(TM) =/= 0 -> + join_arg_types_1(Args, TMs, Ts#{ Arg => join(maps:values(TM))}); +join_arg_types_1([Arg | Args], [_TM | TMs], Ts) -> + join_arg_types_1(Args, TMs, Ts#{ Arg => any }); +join_arg_types_1([], [], Ts) -> + Ts. + +-spec opt_continue_1(Linear, Args, Id, Ts, FuncDb) -> Result when + Linear :: [{non_neg_integer(), beam_ssa:b_blk()}], + Args :: [beam_ssa:b_var()], + Id :: func_id(), + Ts :: type_db(), + FuncDb :: func_info_db(), + Result :: {Linear, FuncDb}. +opt_continue_1(Linear0, Args, Id, Ts, FuncDb0) -> + UsedOnce = used_once(Linear0, Args), + FakeCall = #b_set{op=call,args=[#b_remote{mod=#b_literal{val=unknown}, + name=#b_literal{val=unknown}, + arity=0}]}, + Defs = maps:from_list([{Var,FakeCall#b_set{dst=Var}} || + #b_var{}=Var <- Args]), + + D = #d{ func_db=FuncDb0, + func_id=Id, + ds=Defs, + ls=#{0=>Ts,?BADARG_BLOCK=>#{}}, + once=UsedOnce }, + + {Linear, FuncDb, NewRet} = opt(Linear0, D, []), + + case FuncDb of + #{ Id := Entry0 } -> + Entry = Entry0#func_info{ret_type=NewRet}, + {Linear, FuncDb#{ Id := Entry }}; + #{} -> + %% Module-level optimizations have been turned off for this + %% function. + {Linear, FuncDb} + end. + +-spec opt_finish(Args, Anno, FuncDb) -> {Anno, FuncDb} when + Args :: [beam_ssa:b_var()], + Anno :: beam_ssa:anno(), + FuncDb :: func_info_db(). +opt_finish(Args, Anno, FuncDb) -> + Id = get_func_id(Anno), + case FuncDb of + #{ Id := #func_info{exported=false,arg_types=ArgTypes} } -> + ParamInfo0 = maps:get(parameter_type_info, Anno, #{}), + ParamInfo = opt_finish_1(Args, ArgTypes, ParamInfo0), + {Anno#{ parameter_type_info => ParamInfo }, FuncDb}; + #{} -> + {Anno, FuncDb} + end. + +opt_finish_1([Arg | Args], [TypeMap | TypeMaps], ParamInfo) + when is_map_key(Arg, ParamInfo); %% See join_arg_types/3 + map_size(TypeMap) =:= 0 -> + opt_finish_1(Args, TypeMaps, ParamInfo); +opt_finish_1([Arg | Args], [TypeMap | TypeMaps], ParamInfo0) -> + case join(maps:values(TypeMap)) of + any -> + opt_finish_1(Args, TypeMaps, ParamInfo0); + JoinedType -> + JoinedType = verified_type(JoinedType), + ParamInfo = ParamInfo0#{ Arg => validator_anno(JoinedType) }, + opt_finish_1(Args, TypeMaps, ParamInfo) + end; +opt_finish_1([], [], ParamInfo) -> + ParamInfo. + +validator_anno(#t_tuple{size=Size,exact=Exact,elements=Elements0}) -> + Elements = maps:fold(fun(Index, Type, Acc) -> + Key = beam_validator:type_anno(integer, Index), + Acc#{ Key => validator_anno(Type) } + end, #{}, Elements0), + beam_validator:type_anno(tuple, Size, Exact, Elements); +validator_anno(#t_integer{elements={Same,Same}}) -> + beam_validator:type_anno(integer, Same); +validator_anno(#t_integer{}) -> + beam_validator:type_anno(integer); +validator_anno(float) -> + beam_validator:type_anno(float); +validator_anno(#t_atom{elements=[Val]}) -> + beam_validator:type_anno(atom, Val); +validator_anno(#t_atom{}=A) -> + case t_is_boolean(A) of + true -> beam_validator:type_anno(bool); + false -> beam_validator:type_anno(atom) + end; +validator_anno(T) -> + beam_validator:type_anno(T). + +get_func_id(Anno) -> + #{func_info:={_Mod, Name, Arity}} = Anno, + #b_local{name=#b_literal{val=Name}, arity=Arity}. + +opt([{L,Blk}|Bs], #d{ls=Ls}=D, Acc) -> + case Ls of + #{L:=Ts} -> + opt_1(L, Blk, Bs, Ts, D, Acc); + #{} -> + %% This block is never reached. Discard it. + opt(Bs, D, Acc) + end; +opt([], D, Acc) -> + #d{func_db=FuncDb,ret_type=NewRet} = D, + {reverse(Acc), FuncDb, NewRet}. + +opt_1(L, #b_blk{is=Is0,last=Last0}=Blk0, Bs, Ts0, + #d{ds=Ds0,sub=Sub0,func_db=Fdb0}=D0, Acc) -> + case opt_is(Is0, Ts0, Ds0, Fdb0, D0, Sub0, []) of + {Is,Ts,Ds,Fdb,Sub} -> + D1 = D0#d{ds=Ds,sub=Sub,func_db=Fdb}, + Last1 = simplify_terminator(Last0, Sub, Ts, Ds), + Last = opt_terminator(Last1, Ts, Ds), + D = update_successors(Last, Ts, D1), + Blk = Blk0#b_blk{is=Is,last=Last}, + opt(Bs, D, [{L,Blk}|Acc]); + {no_return,Ret,Is,Ds,Fdb,Sub} -> + %% This call will never reach the successor block. + %% Rewrite the terminator to a 'ret', and remove + %% all type information for this label. That can + %% potentially narrow the type of the phi node + %% in the former successor. + Ls = maps:remove(L, D0#d.ls), + RetType = join([none|D0#d.ret_type]), + D = D0#d{ds=Ds,ls=Ls,sub=Sub, + func_db=Fdb,ret_type=[RetType]}, + Blk = Blk0#b_blk{is=Is,last=Ret}, + opt(Bs, D, [{L,Blk}|Acc]) + end. + +simplify_terminator(#b_br{bool=Bool}=Br, Sub, Ts, _Ds) -> + Br#b_br{bool=simplify_arg(Bool, Sub, Ts)}; +simplify_terminator(#b_switch{arg=Arg}=Sw, Sub, Ts, _Ds) -> + Sw#b_switch{arg=simplify_arg(Arg, Sub, Ts)}; +simplify_terminator(#b_ret{arg=Arg}=Ret, Sub, Ts, Ds) -> + %% Reducing the result of a call to a literal (fairly common for 'ok') + %% breaks tail call optimization. + case Ds of + #{ Arg := #b_set{op=call}} -> Ret; + #{} -> Ret#b_ret{arg=simplify_arg(Arg, Sub, Ts)} + end. + +opt_is([#b_set{op=phi,dst=Dst,args=Args0}=I0|Is], + Ts0, Ds0, Fdb, #d{ls=Ls}=D, Sub0, Acc) -> + %% Simplify the phi node by removing all predecessor blocks that no + %% longer exists or no longer branches to this block. + Args = [{simplify_arg(Arg, Sub0, Ts0),From} || + {Arg,From} <- Args0, maps:is_key(From, Ls)], + case all_same(Args) of + true -> + %% Eliminate the phi node if there is just one source + %% value or if the values are identical. + [{Val,_}|_] = Args, + Sub = Sub0#{Dst=>Val}, + opt_is(Is, Ts0, Ds0, Fdb, D, Sub, Acc); + false -> + I = I0#b_set{args=Args}, + Ts = update_types(I, Ts0, Ds0), + Ds = Ds0#{Dst=>I}, + opt_is(Is, Ts, Ds, Fdb, D, Sub0, [I|Acc]) + end; +opt_is([#b_set{op=call,args=Args0,dst=Dst}=I0|Is], + Ts0, Ds0, Fdb0, D, Sub0, Acc) -> + Args = simplify_args(Args0, Sub0, Ts0), + I1 = beam_ssa:normalize(I0#b_set{args=Args}), + {Ts1,Ds,Fdb,I2} = opt_call(I1, D, Ts0, Ds0, Fdb0), + case {map_get(Dst, Ts1),Is} of + {Type,[#b_set{op=succeeded}]} when Type =/= none -> + %% This call instruction is inside a try/catch + %% block. Don't attempt to simplify it. + opt_is(Is, Ts1, Ds, Fdb, D, Sub0, [I2|Acc]); + {none,[#b_set{op=succeeded}]} -> + %% This call instruction is inside a try/catch + %% block, but we know it will never return and + %% later optimizations may try to exploit that. + %% + %% For example, if we have an expression that + %% either returns this call or a tuple, we know + %% that the expression always returns a tuple + %% and can turn a later element/3 into + %% get_tuple_element. + %% + %% This is sound but difficult to validate in a + %% meaningful way as try/catch currently forces + %% us to maintain the illusion that the success + %% block is reachable even when its not, so we + %% disable the optimization to keep things + %% simple. + Ts = Ts1#{ Dst := any }, + opt_is(Is, Ts, Ds, Fdb, D, Sub0, [I2|Acc]); + {none,_} -> + %% This call never returns. The rest of the + %% instructions will not be executed. + Ret = #b_ret{arg=Dst}, + {no_return,Ret,reverse(Acc, [I2]),Ds,Fdb,Sub0}; + {_,_} -> + case simplify_call(I2) of + #b_set{}=I -> + opt_is(Is, Ts1, Ds, Fdb, D, Sub0, [I|Acc]); + #b_literal{}=Lit -> + Sub = Sub0#{Dst=>Lit}, + Ts = maps:remove(Dst, Ts1), + opt_is(Is, Ts, Ds0, Fdb, D, Sub, Acc); + #b_var{}=Var -> + Ts = maps:remove(Dst, Ts1), + Sub = Sub0#{Dst=>Var}, + opt_is(Is, Ts, Ds0, Fdb, D, Sub, Acc) + end + end; +opt_is([#b_set{op=succeeded,args=[Arg],dst=Dst}=I], + Ts0, Ds0, Fdb, D, Sub0, Acc) -> + case Ds0 of + #{ Arg := #b_set{op=call} } -> + %% The success check of a call is part of exception handling and + %% must not be optimized away. We still have to update its type + %% though. + Ts = update_types(I, Ts0, Ds0), + Ds = Ds0#{Dst=>I}, + + opt_is([], Ts, Ds, Fdb, D, Sub0, [I|Acc]); + #{} -> + Args = simplify_args([Arg], Sub0, Ts0), + Type = type(succeeded, Args, Ts0, Ds0), + case get_literal_from_type(Type) of + #b_literal{}=Lit -> + Sub = Sub0#{Dst=>Lit}, + opt_is([], Ts0, Ds0, Fdb, D, Sub, Acc); + none -> + Ts = Ts0#{Dst=>Type}, + Ds = Ds0#{Dst=>I}, + opt_is([], Ts, Ds, Fdb, D, Sub0, [I|Acc]) + end + end; +opt_is([#b_set{args=Args0,dst=Dst}=I0|Is], + Ts0, Ds0, Fdb, D, Sub0, Acc) -> + Args = simplify_args(Args0, Sub0, Ts0), + I1 = beam_ssa:normalize(I0#b_set{args=Args}), + case simplify(I1, Ts0) of + #b_set{}=I2 -> + I = beam_ssa:normalize(I2), + Ts = update_types(I, Ts0, Ds0), + Ds = Ds0#{Dst=>I}, + opt_is(Is, Ts, Ds, Fdb, D, Sub0, [I|Acc]); + #b_literal{}=Lit -> + Sub = Sub0#{Dst=>Lit}, + opt_is(Is, Ts0, Ds0, Fdb, D, Sub, Acc); + #b_var{}=Var -> + case Is of + [#b_set{op=succeeded,dst=SuccDst,args=[Dst]}] -> + %% We must remove this 'succeeded' instruction. + Sub = Sub0#{Dst=>Var,SuccDst=>#b_literal{val=true}}, + opt_is([], Ts0, Ds0, Fdb, D, Sub, Acc); + _ -> + Sub = Sub0#{Dst=>Var}, + opt_is(Is, Ts0, Ds0, Fdb, D, Sub, Acc) + end + end; +opt_is([], Ts, Ds, Fdb, _D, Sub, Acc) -> + {reverse(Acc), Ts, Ds, Fdb, Sub}. + +simplify_call(#b_set{op=call,args=[#b_remote{}=Rem|Args]}=I) -> + case Rem of + #b_remote{mod=#b_literal{val=Mod}, + name=#b_literal{val=Name}} -> + case erl_bifs:is_pure(Mod, Name, length(Args)) of + true -> + simplify_remote_call(Mod, Name, Args, I); + false -> + I + end; + #b_remote{} -> + I + end; +simplify_call(I) -> I. + +%% Simplify a remote call to a pure BIF. +simplify_remote_call(erlang, '++', [#b_literal{val=[]},Tl], _I) -> + Tl; +simplify_remote_call(erlang, setelement, + [#b_literal{val=Pos}, + #b_literal{val=Tuple}, + #b_var{}=Value], I) + when is_integer(Pos), 1 =< Pos, Pos =< tuple_size(Tuple) -> + %% Position is a literal integer and the shape of the + %% tuple is known. + Els0 = [#b_literal{val=El} || El <- tuple_to_list(Tuple)], + {Bef,[_|Aft]} = split(Pos - 1, Els0), + Els = Bef ++ [Value|Aft], + I#b_set{op=put_tuple,args=Els}; +simplify_remote_call(Mod, Name, Args0, I) -> + case make_literal_list(Args0) of + none -> + I; + Args -> + %% The arguments are literals. Try to evaluate the BIF. + try apply(Mod, Name, Args) of + Val -> + case cerl:is_literal_term(Val) of + true -> + #b_literal{val=Val}; + false -> + %% The value can't be expressed as a literal + %% (e.g. a pid). + I + end + catch + _:_ -> + %% Failed. Don't bother trying to optimize + %% the call. + I + end + end. + +opt_call(#b_set{dst=Dst,args=[#b_local{}=Callee|Args]}=I0, D, Ts0, Ds0, Fdb0) -> + {Ts, Ds, I} = opt_local_call(I0, Ts0, Ds0, Fdb0), + case Fdb0 of + #{ Callee := #func_info{exported=false,arg_types=ArgTypes0}=Info } -> + %% Update the argument types of *this exact call*, the types + %% will be joined later when the callee is optimized. + CallId = {D#d.func_id, Dst}, + ArgTypes = update_arg_types(Args, ArgTypes0, CallId, Ts0), + + Fdb = Fdb0#{ Callee => Info#func_info{arg_types=ArgTypes} }, + {Ts, Ds, Fdb, I}; + #{} -> + %% We can't narrow the argument types of exported functions as they + %% can receive anything as part of an external call. + {Ts, Ds, Fdb0, I} + end; +opt_call(#b_set{dst=Dst}=I, _D, Ts0, Ds0, Fdb) -> + Ts = update_types(I, Ts0, Ds0), + Ds = Ds0#{ Dst => I }, + {Ts, Ds, Fdb, I}. + +opt_local_call(#b_set{dst=Dst,args=[Id|_]}=I0, Ts0, Ds0, Fdb) -> + Type = case Fdb of + #{ Id := #func_info{ret_type=[T]} } -> T; + #{} -> any + end, + I = case Type of + any -> I0; + none -> I0; + _ -> beam_ssa:add_anno(result_type, validator_anno(Type), I0) + end, + Ts = Ts0#{ Dst => Type }, + Ds = Ds0#{ Dst => I }, + {Ts, Ds, I}. + +update_arg_types([Arg | Args], [TypeMap0 | TypeMaps], CallId, Ts) -> + %% Match contexts are treated as bitstrings when optimizing arguments, as + %% we don't yet support removing the "bs_start_match3" instruction. + NewType = case get_type(Arg, Ts) of + #t_bs_match{} -> {binary, 1}; + Type -> Type + end, + TypeMap = TypeMap0#{ CallId => NewType }, + [TypeMap | update_arg_types(Args, TypeMaps, CallId, Ts)]; +update_arg_types([], [], _CallId, _Ts) -> + []. + +simplify(#b_set{op={bif,'and'},args=Args}=I, Ts) -> + case is_safe_bool_op(Args, Ts) of + true -> + case Args of + [_,#b_literal{val=false}=Res] -> Res; + [Res,#b_literal{val=true}] -> Res; + _ -> eval_bif(I, Ts) + end; + false -> + I + end; +simplify(#b_set{op={bif,'or'},args=Args}=I, Ts) -> + case is_safe_bool_op(Args, Ts) of + true -> + case Args of + [Res,#b_literal{val=false}] -> Res; + [_,#b_literal{val=true}=Res] -> Res; + _ -> eval_bif(I, Ts) + end; + false -> + I + end; +simplify(#b_set{op={bif,element},args=[#b_literal{val=Index},Tuple]}=I0, Ts) -> + case t_tuple_size(get_type(Tuple, Ts)) of + {_,Size} when is_integer(Index), 1 =< Index, Index =< Size -> + I = I0#b_set{op=get_tuple_element, + args=[Tuple,#b_literal{val=Index-1}]}, + simplify(I, Ts); + _ -> + eval_bif(I0, Ts) + end; +simplify(#b_set{op={bif,hd},args=[List]}=I, Ts) -> + case get_type(List, Ts) of + cons -> + I#b_set{op=get_hd}; + _ -> + eval_bif(I, Ts) + end; +simplify(#b_set{op={bif,tl},args=[List]}=I, Ts) -> + case get_type(List, Ts) of + cons -> + I#b_set{op=get_tl}; + _ -> + eval_bif(I, Ts) + end; +simplify(#b_set{op={bif,size},args=[Term]}=I, Ts) -> + case get_type(Term, Ts) of + #t_tuple{} -> + simplify(I#b_set{op={bif,tuple_size}}, Ts); + _ -> + eval_bif(I, Ts) + end; +simplify(#b_set{op={bif,tuple_size},args=[Term]}=I, Ts) -> + case get_type(Term, Ts) of + #t_tuple{size=Size,exact=true} -> + #b_literal{val=Size}; + _ -> + I + end; +simplify(#b_set{op={bif,'=='},args=Args}=I, Ts) -> + Types = get_types(Args, Ts), + EqEq = case {meet(Types),join(Types)} of + {none,any} -> true; + {#t_integer{},#t_integer{}} -> true; + {float,float} -> true; + {{binary,_},_} -> true; + {#t_atom{},_} -> true; + {_,_} -> false + end, + case EqEq of + true -> + simplify(I#b_set{op={bif,'=:='}}, Ts); + false -> + eval_bif(I, Ts) + end; +simplify(#b_set{op={bif,'=:='},args=[Same,Same]}, _Ts) -> + #b_literal{val=true}; +simplify(#b_set{op={bif,'=:='},args=[A1,_A2]=Args}=I, Ts) -> + [T1,T2] = get_types(Args, Ts), + case meet(T1, T2) of + none -> + #b_literal{val=false}; + _ -> + case {t_is_boolean(T1),T2} of + {true,#t_atom{elements=[true]}} -> + %% Bool =:= true ==> Bool + A1; + {_,_} -> + eval_bif(I, Ts) + end + end; +simplify(#b_set{op={bif,Op},args=Args}=I, Ts) -> + Types = get_types(Args, Ts), + case is_float_op(Op, Types) of + false -> + eval_bif(I, Ts); + true -> + AnnoArgs = [anno_float_arg(A) || A <- Types], + eval_bif(beam_ssa:add_anno(float_op, AnnoArgs, I), Ts) + end; +simplify(#b_set{op=get_tuple_element,args=[Tuple,#b_literal{val=N}]}=I, Ts) -> + case get_type(Tuple, Ts) of + #t_tuple{size=Size,elements=Es} when Size > N -> + ElemType = get_element_type(N + 1, Es), + case get_literal_from_type(ElemType) of + #b_literal{}=Lit -> Lit; + none -> I + end; + none -> + %% Will never be executed because of type conflict. + %% #b_literal{val=ignored}; + I + end; +simplify(#b_set{op=is_nonempty_list,args=[Src]}=I, Ts) -> + case get_type(Src, Ts) of + any -> I; + list -> I; + cons -> #b_literal{val=true}; + _ -> #b_literal{val=false} + end; +simplify(#b_set{op=is_tagged_tuple, + args=[Src,#b_literal{val=Size},#b_literal{}=Tag]}=I, Ts) -> + simplify_is_record(I, get_type(Src, Ts), Size, Tag, Ts); +simplify(#b_set{op=put_list,args=[#b_literal{val=H}, + #b_literal{val=T}]}, _Ts) -> + #b_literal{val=[H|T]}; +simplify(#b_set{op=put_tuple,args=Args}=I, _Ts) -> + case make_literal_list(Args) of + none -> I; + List -> #b_literal{val=list_to_tuple(List)} + end; +simplify(#b_set{op=wait_timeout,args=[#b_literal{val=0}]}, _Ts) -> + #b_literal{val=true}; +simplify(#b_set{op=wait_timeout,args=[#b_literal{val=infinity}]}=I, _Ts) -> + I#b_set{op=wait,args=[]}; +simplify(I, _Ts) -> I. + +make_literal_list(Args) -> + make_literal_list(Args, []). + +make_literal_list([#b_literal{val=H}|T], Acc) -> + make_literal_list(T, [H|Acc]); +make_literal_list([_|_], _) -> + none; +make_literal_list([], Acc) -> + reverse(Acc). + +is_safe_bool_op(Args, Ts) -> + [T1,T2] = get_types(Args, Ts), + t_is_boolean(T1) andalso t_is_boolean(T2). + +all_same([{H,_}|T]) -> + all(fun({E,_}) -> E =:= H end, T). + +eval_bif(#b_set{op={bif,Bif},args=Args}=I, Ts) -> + Arity = length(Args), + case erl_bifs:is_pure(erlang, Bif, Arity) of + false -> + I; + true -> + case make_literal_list(Args) of + none -> + case get_types(Args, Ts) of + [any] -> + I; + [Type] -> + case will_succeed(Bif, Type) of + yes -> + #b_literal{val=true}; + no -> + #b_literal{val=false}; + maybe -> + I + end; + _ -> + I + end; + LitArgs -> + try apply(erlang, Bif, LitArgs) of + Val -> #b_literal{val=Val} + catch + error:_ -> I + end + + end + end. + +simplify_args(Args, Sub, Ts) -> + [simplify_arg(Arg, Sub, Ts) || Arg <- Args]. + +simplify_arg(#b_var{}=Arg0, Sub, Ts) -> + case sub_arg(Arg0, Sub) of + #b_literal{}=LitArg -> + LitArg; + #b_var{}=Arg -> + Type = get_type(Arg, Ts), + case get_literal_from_type(Type) of + none -> Arg; + #b_literal{}=Lit -> Lit + end + end; +simplify_arg(#b_remote{mod=Mod,name=Name}=Rem, Sub, Ts) -> + Rem#b_remote{mod=simplify_arg(Mod, Sub, Ts), + name=simplify_arg(Name, Sub, Ts)}; +simplify_arg(Arg, _Sub, _Ts) -> Arg. + +sub_arg(#b_var{}=Old, Sub) -> + case Sub of + #{Old:=New} -> New; + #{} -> Old + end. + +is_float_op('-', [float]) -> + true; +is_float_op('/', [_,_]) -> + true; +is_float_op(Op, [float,_Other]) -> + is_float_op_1(Op); +is_float_op(Op, [_Other,float]) -> + is_float_op_1(Op); +is_float_op(_, _) -> false. + +is_float_op_1('+') -> true; +is_float_op_1('-') -> true; +is_float_op_1('*') -> true; +is_float_op_1(_) -> false. + +anno_float_arg(float) -> float; +anno_float_arg(_) -> convert. + +opt_terminator(#b_br{bool=#b_literal{}}=Br, _Ts, _Ds) -> + beam_ssa:normalize(Br); +opt_terminator(#b_br{bool=#b_var{}}=Br, Ts, Ds) -> + simplify_not(Br, Ts, Ds); +opt_terminator(#b_switch{arg=#b_literal{}}=Sw, _Ts, _Ds) -> + beam_ssa:normalize(Sw); +opt_terminator(#b_switch{arg=#b_var{}=V}=Sw, Ts, Ds) -> + case get_type(V, Ts) of + any -> + beam_ssa:normalize(Sw); + Type -> + beam_ssa:normalize(opt_switch(Sw, Type, Ts, Ds)) + end; +opt_terminator(#b_ret{}=Ret, _Ts, _Ds) -> Ret. + + +opt_switch(#b_switch{fail=Fail,list=List0}=Sw0, Type, Ts, Ds) -> + List = prune_switch_list(List0, Fail, Type, Ts), + Sw1 = Sw0#b_switch{list=List}, + case Type of + #t_integer{elements={_,_}=Range} -> + simplify_switch_int(Sw1, Range); + #t_atom{elements=[_|_]} -> + case t_is_boolean(Type) of + true -> + #b_br{} = Br = simplify_switch_bool(Sw1, Ts, Ds), + opt_terminator(Br, Ts, Ds); + false -> + simplify_switch_atom(Type, Sw1) + end; + _ -> + Sw1 + end. + +prune_switch_list([{_,Fail}|T], Fail, Type, Ts) -> + prune_switch_list(T, Fail, Type, Ts); +prune_switch_list([{Arg,_}=Pair|T], Fail, Type, Ts) -> + case meet(get_type(Arg, Ts), Type) of + none -> + %% Different types. This value can never match. + prune_switch_list(T, Fail, Type, Ts); + _ -> + [Pair|prune_switch_list(T, Fail, Type, Ts)] + end; +prune_switch_list([], _, _, _) -> []. + +update_successors(#b_br{bool=#b_literal{val=true},succ=S}, Ts, D) -> + update_successor(S, Ts, D); +update_successors(#b_br{bool=#b_var{}=Bool,succ=Succ,fail=Fail}, Ts0, D0) -> + case cerl_sets:is_element(Bool, D0#d.once) of + true -> + %% This variable is defined in this block and is only + %% referenced by this br terminator. Therefore, there is + %% no need to include it in the type database passed on to + %% the successors of this block. + Ts = maps:remove(Bool, Ts0), + {SuccTs,FailTs} = infer_types_br(Bool, Ts, D0), + D = update_successor(Fail, FailTs, D0), + update_successor(Succ, SuccTs, D); + false -> + {SuccTs,FailTs} = infer_types_br(Bool, Ts0, D0), + D = update_successor_bool(Bool, false, Fail, FailTs, D0), + update_successor_bool(Bool, true, Succ, SuccTs, D) + end; +update_successors(#b_switch{arg=#b_var{}=V,fail=Fail,list=List}, Ts, D0) -> + case cerl_sets:is_element(V, D0#d.once) of + true -> + %% This variable is defined in this block and is only + %% referenced by this switch terminator. Therefore, there is + %% no need to include it in the type database passed on to + %% the successors of this block. + D = update_successor(Fail, Ts, D0), + F = fun({Val,S}, A) -> + SuccTs0 = infer_types_switch(V, Val, Ts, D), + SuccTs = maps:remove(V, SuccTs0), + update_successor(S, SuccTs, A) + end, + foldl(F, D, List); + false -> + %% V can not be equal to any of the values in List at the fail + %% block. + FailTs = subtract_sw_list(V, List, Ts), + D = update_successor(Fail, FailTs, D0), + F = fun({Val,S}, A) -> + SuccTs = infer_types_switch(V, Val, Ts, D), + update_successor(S, SuccTs, A) + end, + foldl(F, D, List) + end; +update_successors(#b_ret{arg=Arg}, Ts, D) -> + FuncId = D#d.func_id, + case D#d.ds of + #{ Arg := #b_set{op=call,args=[FuncId | _]} } -> + %% Returning a call to ourselves doesn't affect our own return + %% type. + D; + #{} -> + RetType = join([get_type(Arg, Ts) | D#d.ret_type]), + D#d{ret_type=[RetType]} + end. + +subtract_sw_list(V, List, Ts) -> + Ts#{ V := sub_sw_list_1(get_type(V, Ts), List, Ts) }. + +sub_sw_list_1(Type, [{Val,_}|T], Ts) -> + ValType = get_type(Val, Ts), + sub_sw_list_1(subtract(Type, ValType), T, Ts); +sub_sw_list_1(Type, [], _Ts) -> + Type. + +update_successor_bool(#b_var{}=Var, BoolValue, S, Ts, D) -> + case t_is_boolean(get_type(Var, Ts)) of + true -> + update_successor(S, Ts#{Var:=t_atom(BoolValue)}, D); + false -> + %% The `br` terminator is preceeded by an instruction that + %% does not produce a boolean value, such a `new_try_tag`. + update_successor(S, Ts, D) + end. + +update_successor(?BADARG_BLOCK, _Ts, #d{}=D) -> + %% We KNOW that no variables are used in the ?BADARG_BLOCK, + %% so there is no need to update the type information. That + %% can be a huge timesaver for huge functions. + D; +update_successor(S, Ts0, #d{ls=Ls}=D) -> + case Ls of + #{S:=Ts1} -> + Ts = join_types(Ts0, Ts1), + D#d{ls=Ls#{S:=Ts}}; + #{} -> + D#d{ls=Ls#{S=>Ts0}} + end. + +update_types(#b_set{op=Op,dst=Dst,args=Args}, Ts, Ds) -> + T = type(Op, Args, Ts, Ds), + Ts#{Dst=>T}. + +type(phi, Args, Ts, _Ds) -> + Types = [get_type(A, Ts) || {A,_} <- Args], + join(Types); +type({bif,'band'}, Args, Ts, _Ds) -> + band_type(Args, Ts); +type({bif,Bif}, Args, Ts, _Ds) -> + case bif_type(Bif, Args) of + number -> + arith_op_type(Args, Ts); + Type -> + Type + end; +type(bs_init, [#b_literal{val=Type}|Args], _Ts, _Ds) -> + case {Type,Args} of + {new,[_,#b_literal{val=Unit}]} -> + {binary,Unit}; + {append,[_,_,#b_literal{val=Unit}]} -> + {binary,Unit}; + {private_append,[_,_,#b_literal{val=Unit}]} -> + {binary,Unit} + end; +type(bs_extract, [Ctx], Ts, _Ds) -> + #t_bs_match{type=Type} = get_type(Ctx, Ts), + Type; +type(bs_match, Args, _Ts, _Ds) -> + #t_bs_match{type=bs_match_type(Args)}; +type(bs_get_tail, _Args, _Ts, _Ds) -> + {binary, 1}; +type(call, [#b_remote{mod=#b_literal{val=Mod}, + name=#b_literal{val=Name}}|Args], Ts, _Ds) -> + case {Mod,Name,Args} of + {erlang,setelement,[Pos,Tuple,Arg]} -> + case {get_type(Pos, Ts),get_type(Tuple, Ts)} of + {#t_integer{elements={Index,Index}}, + #t_tuple{elements=Es0,size=Size}=T} -> + %% This is an exact index, update the type of said element + %% or return 'none' if it's known to be out of bounds. + Es = set_element_type(Index, get_type(Arg, Ts), Es0), + case T#t_tuple.exact of + false -> + T#t_tuple{size=max(Index, Size),elements=Es}; + true when Index =< Size -> + T#t_tuple{elements=Es}; + true -> + none + end; + {#t_integer{elements={Min,Max}}, + #t_tuple{elements=Es0,size=Size}=T} -> + %% We know this will land between Min and Max, so kill the + %% types for those indexes. + Es = maps:without(seq(Min, Max), Es0), + case T#t_tuple.exact of + false -> + T#t_tuple{elements=Es,size=max(Min, Size)}; + true when Min =< Size -> + T#t_tuple{elements=Es,size=Size}; + true -> + none + end; + {_,#t_tuple{}=T} -> + %% Position unknown, so we have to discard all element + %% information. + T#t_tuple{elements=#{}}; + {#t_integer{elements={Min,_Max}},_} -> + #t_tuple{size=Min}; + {_,_} -> + #t_tuple{} + end; + {erlang,'++',[List1,List2]} -> + case get_type(List1, Ts) =:= cons orelse + get_type(List2, Ts) =:= cons of + true -> cons; + false -> list + end; + {erlang,'--',[_,_]} -> + list; + {lists,F,Args} -> + Types = get_types(Args, Ts), + lists_function_type(F, Types); + {math,_,_} -> + case is_math_bif(Name, length(Args)) of + false -> any; + true -> float + end; + {_,_,_} -> + case erl_bifs:is_exit_bif(Mod, Name, length(Args)) of + true -> none; + false -> any + end + end; +type(get_tuple_element, [Tuple, Offset], Ts, _Ds) -> + #t_tuple{size=Size,elements=Es} = get_type(Tuple, Ts), + #b_literal{val=N} = Offset, + true = Size > N, %Assertion. + get_element_type(N + 1, Es); +type(is_nonempty_list, [_], _Ts, _Ds) -> + t_boolean(); +type(is_tagged_tuple, [_,#b_literal{},#b_literal{}], _Ts, _Ds) -> + t_boolean(); +type(put_map, _Args, _Ts, _Ds) -> + map; +type(put_list, _Args, _Ts, _Ds) -> + cons; +type(put_tuple, Args, Ts, _Ds) -> + {Es, _} = foldl(fun(Arg, {Es0, Index}) -> + Type = get_type(Arg, Ts), + Es = set_element_type(Index, Type, Es0), + {Es, Index + 1} + end, {#{}, 1}, Args), + #t_tuple{exact=true,size=length(Args),elements=Es}; +type(succeeded, [#b_var{}=Src], Ts, Ds) -> + case maps:get(Src, Ds) of + #b_set{op={bif,Bif},args=BifArgs} -> + Types = get_types(BifArgs, Ts), + case {Bif,Types} of + {BoolOp,[T1,T2]} when BoolOp =:= 'and'; BoolOp =:= 'or' -> + case t_is_boolean(T1) andalso t_is_boolean(T2) of + true -> t_atom(true); + false -> t_boolean() + end; + {byte_size,[{binary,_}]} -> + t_atom(true); + {bit_size,[{binary,_}]} -> + t_atom(true); + {map_size,[map]} -> + t_atom(true); + {'not',[Type]} -> + case t_is_boolean(Type) of + true -> t_atom(true); + false -> t_boolean() + end; + {size,[{binary,_}]} -> + t_atom(true); + {tuple_size,[#t_tuple{}]} -> + t_atom(true); + {_,_} -> + t_boolean() + end; + #b_set{op=get_hd} -> + t_atom(true); + #b_set{op=get_tl} -> + t_atom(true); + #b_set{op=get_tuple_element} -> + t_atom(true); + #b_set{op=wait} -> + t_atom(false); + #b_set{} -> + t_boolean() + end; +type(succeeded, [#b_literal{}], _Ts, _Ds) -> + t_atom(true); +type(_, _, _, _) -> any. + +arith_op_type(Args, Ts) -> + Types = get_types(Args, Ts), + foldl(fun(#t_integer{}, unknown) -> t_integer(); + (#t_integer{}, number) -> number; + (#t_integer{}, float) -> float; + (#t_integer{}, #t_integer{}) -> t_integer(); + (float, unknown) -> float; + (float, #t_integer{}) -> float; + (float, number) -> float; + (number, unknown) -> number; + (number, #t_integer{}) -> number; + (number, float) -> float; + (any, _) -> number; + (Same, Same) -> Same; + (_, _) -> none + end, unknown, Types). + +lists_function_type(F, Types) -> + case {F,Types} of + %% Functions that return booleans. + {all,[_,_]} -> + t_boolean(); + {any,[_,_]} -> + t_boolean(); + {keymember,[_,_,_]} -> + t_boolean(); + {member,[_,_]} -> + t_boolean(); + {prefix,[_,_]} -> + t_boolean(); + {suffix,[_,_]} -> + t_boolean(); + + %% Functions that return lists. + {dropwhile,[_,_]} -> + list; + {duplicate,[_,_]} -> + list; + {filter,[_,_]} -> + list; + {flatten,[_]} -> + list; + {map,[_Fun,List]} -> + same_length_type(List); + {MapFold,[_Fun,_Acc,List]} when MapFold =:= mapfoldl; + MapFold =:= mapfoldr -> + #t_tuple{size=2,exact=true, + elements=#{1=>same_length_type(List)}}; + {partition,[_,_]} -> + t_two_tuple(list, list); + {reverse,[List]} -> + same_length_type(List); + {sort,[List]} -> + same_length_type(List); + {splitwith,[_,_]} -> + t_two_tuple(list, list); + {takewhile,[_,_]} -> + list; + {unzip,[List]} -> + ListType = same_length_type(List), + t_two_tuple(ListType, ListType); + {usort,[List]} -> + same_length_type(List); + {zip,[_,_]} -> + list; + {zipwith,[_,_,_]} -> + list; + {_,_} -> + any + end. + +%% For a lists function that return a list of the same +%% length as the input list, return the type of the list. +same_length_type(cons) -> cons; +same_length_type(nil) -> nil; +same_length_type(_) -> list. + +t_two_tuple(Type1, Type2) -> + #t_tuple{size=2,exact=true, + elements=#{1=>Type1,2=>Type2}}. + +%% will_succeed(TestOperation, Type) -> yes|no|maybe. +%% Test whether TestOperation applied to an argument of type Type +%% will succeed. Return yes, no, or maybe. +%% +%% Type is a type as described in the comment for verified_type/1 at +%% the very end of this file, but it will *never* be 'any'. + +will_succeed(is_atom, Type) -> + case Type of + #t_atom{} -> yes; + _ -> no + end; +will_succeed(is_binary, Type) -> + case Type of + {binary,U} when U rem 8 =:= 0 -> yes; + {binary,_} -> maybe; + _ -> no + end; +will_succeed(is_bitstring, Type) -> + case Type of + {binary,_} -> yes; + _ -> no + end; +will_succeed(is_boolean, Type) -> + case Type of + #t_atom{elements=any} -> + maybe; + #t_atom{elements=Es} -> + case t_is_boolean(Type) of + true -> + yes; + false -> + case any(fun is_boolean/1, Es) of + true -> maybe; + false -> no + end + end; + _ -> + no + end; +will_succeed(is_float, Type) -> + case Type of + float -> yes; + number -> maybe; + _ -> no + end; +will_succeed(is_integer, Type) -> + case Type of + #t_integer{} -> yes; + number -> maybe; + _ -> no + end; +will_succeed(is_list, Type) -> + case Type of + list -> yes; + cons -> yes; + _ -> no + end; +will_succeed(is_map, Type) -> + case Type of + map -> yes; + _ -> no + end; +will_succeed(is_number, Type) -> + case Type of + float -> yes; + #t_integer{} -> yes; + number -> yes; + _ -> no + end; +will_succeed(is_tuple, Type) -> + case Type of + #t_tuple{} -> yes; + _ -> no + end; +will_succeed(_, _) -> maybe. + + +band_type([Other,#b_literal{val=Int}], Ts) when is_integer(Int) -> + band_type_1(Int, Other, Ts); +band_type([_,_], _) -> t_integer(). + +band_type_1(Int, OtherSrc, Ts) -> + Type = band_type_2(Int, 0), + OtherType = get_type(OtherSrc, Ts), + meet(Type, OtherType). + +band_type_2(N, Bits) when Bits < 64 -> + case 1 bsl Bits of + P when P =:= N + 1 -> + t_integer(0, N); + P when P > N + 1 -> + t_integer(); + _ -> + band_type_2(N, Bits+1) + end; +band_type_2(_, _) -> + %% Negative or large positive number. Give up. + t_integer(). + +bs_match_type([#b_literal{val=Type}|Args]) -> + bs_match_type(Type, Args). + +bs_match_type(binary, Args) -> + [_,_,_,#b_literal{val=U}] = Args, + {binary,U}; +bs_match_type(float, _) -> + float; +bs_match_type(integer, Args) -> + case Args of + [_, + #b_literal{val=Flags}, + #b_literal{val=Size}, + #b_literal{val=Unit}] when Size * Unit < 64 -> + NumBits = Size * Unit, + case member(unsigned, Flags) of + true -> + t_integer(0, (1 bsl NumBits)-1); + false -> + %% Signed integer. Don't bother. + t_integer() + end; + [_|_] -> + t_integer() + end; +bs_match_type(skip, _) -> + any; +bs_match_type(string, _) -> + any; +bs_match_type(utf8, _) -> + ?UNICODE_INT; +bs_match_type(utf16, _) -> + ?UNICODE_INT; +bs_match_type(utf32, _) -> + ?UNICODE_INT. + +simplify_switch_atom(#t_atom{elements=Atoms}, #b_switch{list=List0}=Sw) -> + case sort([A || {#b_literal{val=A},_} <- List0]) of + Atoms -> + %% All possible atoms are included in the list. The + %% failure label will never be used. + [{_,Fail}|List] = List0, + Sw#b_switch{fail=Fail,list=List}; + _ -> + Sw + end. + +simplify_switch_int(#b_switch{list=List0}=Sw, {Min,Max}) -> + List1 = sort(List0), + Vs = [V || {#b_literal{val=V},_} <- List1], + case eq_ranges(Vs, Min, Max) of + true -> + {_,LastL} = last(List1), + List = droplast(List1), + Sw#b_switch{fail=LastL,list=List}; + false -> + Sw + end. + +eq_ranges([H], H, H) -> true; +eq_ranges([H|T], H, Max) -> eq_ranges(T, H+1, Max); +eq_ranges(_, _, _) -> false. + +simplify_is_record(I, #t_tuple{exact=Exact, + size=Size, + elements=Es}, + RecSize, RecTag, Ts) -> + TagType = maps:get(1, Es, any), + TagMatch = case get_literal_from_type(TagType) of + #b_literal{}=RecTag -> yes; + #b_literal{} -> no; + none -> + %% Is it at all possible for the tag to match? + case meet(get_type(RecTag, Ts), TagType) of + none -> no; + _ -> maybe + end + end, + if + Size =/= RecSize, Exact; Size > RecSize; TagMatch =:= no -> + #b_literal{val=false}; + Size =:= RecSize, Exact, TagMatch =:= yes -> + #b_literal{val=true}; + true -> + I + end; +simplify_is_record(I, any, _Size, _Tag, _Ts) -> + I; +simplify_is_record(_I, _Type, _Size, _Tag, _Ts) -> + #b_literal{val=false}. + +simplify_switch_bool(#b_switch{arg=B,fail=Fail,list=List0}, Ts, Ds) -> + FalseVal = #b_literal{val=false}, + TrueVal = #b_literal{val=true}, + List1 = List0 ++ [{FalseVal,Fail},{TrueVal,Fail}], + {_,FalseLbl} = keyfind(FalseVal, 1, List1), + {_,TrueLbl} = keyfind(TrueVal, 1, List1), + Br = beam_ssa:normalize(#b_br{bool=B,succ=TrueLbl,fail=FalseLbl}), + simplify_not(Br, Ts, Ds). + +simplify_not(#b_br{bool=#b_var{}=V,succ=Succ,fail=Fail}=Br0, Ts, Ds) -> + case Ds of + #{V:=#b_set{op={bif,'not'},args=[Bool]}} -> + case t_is_boolean(get_type(Bool, Ts)) of + true -> + Br = Br0#b_br{bool=Bool,succ=Fail,fail=Succ}, + beam_ssa:normalize(Br); + false -> + Br0 + end; + #{} -> + Br0 + end; +simplify_not(#b_br{bool=#b_literal{}}=Br, _Ts, _Ds) -> Br. + +%%% +%%% Calculate the set of variables that are only used once in the +%%% terminator of the block that defines them. That will allow us to +%%% discard type information for variables that will never be +%%% referenced by the successor blocks, potentially improving +%%% compilation times. +%%% + +used_once(Linear, Args) -> + Map0 = used_once_1(reverse(Linear), #{}), + Map = maps:without(Args, Map0), + cerl_sets:from_list(maps:keys(Map)). + +used_once_1([{L,#b_blk{is=Is,last=Last}}|Bs], Uses0) -> + Uses1 = used_once_last_uses(beam_ssa:used(Last), L, Uses0), + Uses = used_once_2(reverse(Is), L, Uses1), + used_once_1(Bs, Uses); +used_once_1([], Uses) -> Uses. + +used_once_2([#b_set{dst=Dst}=I|Is], L, Uses0) -> + Uses = used_once_uses(beam_ssa:used(I), L, Uses0), + case Uses of + #{Dst:=[L]} -> + used_once_2(Is, L, Uses); + #{} -> + %% Used more than once or used once in + %% in another block. + used_once_2(Is, L, maps:remove(Dst, Uses)) + end; +used_once_2([], _, Uses) -> Uses. + +used_once_uses([V|Vs], L, Uses) -> + case Uses of + #{V:=more_than_once} -> + used_once_uses(Vs, L, Uses); + #{} -> + %% Already used or first use is not in + %% a terminator. + used_once_uses(Vs, L, Uses#{V=>more_than_once}) + end; +used_once_uses([], _, Uses) -> Uses. + +used_once_last_uses([V|Vs], L, Uses) -> + case Uses of + #{V:=[_]} -> + %% Second time this variable is used. + used_once_last_uses(Vs, L, Uses#{V:=more_than_once}); + #{V:=more_than_once} -> + %% Used at least twice before. + used_once_last_uses(Vs, L, Uses); + #{} -> + %% First time this variable is used. + used_once_last_uses(Vs, L, Uses#{V=>[L]}) + end; +used_once_last_uses([], _, Uses) -> Uses. + + +get_types(Values, Ts) -> + [get_type(Val, Ts) || Val <- Values]. +-spec get_type(beam_ssa:value(), type_db()) -> type(). + +get_type(#b_var{}=V, Ts) -> + #{V:=T} = Ts, + T; +get_type(#b_literal{val=Val}, _Ts) -> + if + is_atom(Val) -> + t_atom(Val); + is_float(Val) -> + float; + is_integer(Val) -> + t_integer(Val); + is_list(Val), Val =/= [] -> + cons; + is_map(Val) -> + map; + Val =:= {} -> + #t_tuple{exact=true}; + is_tuple(Val) -> + {Es, _} = foldl(fun(E, {Es0, Index}) -> + Type = get_type(#b_literal{val=E}, #{}), + Es = set_element_type(Index, Type, Es0), + {Es, Index + 1} + end, {#{}, 1}, tuple_to_list(Val)), + #t_tuple{exact=true,size=tuple_size(Val),elements=Es}; + Val =:= [] -> + nil; + true -> + any + end. + +%% infer_types(Var, Types, #d{}) -> {SuccTypes,FailTypes} +%% Looking at the expression that defines the variable Var, infer +%% the types for the variables in the arguments. Return the updated +%% type database for the case that the expression evaluates to +%% true, and and for the case that it evaluates to false. +%% +%% Here is an example. The variable being asked about is +%% the variable Bool, which is defined like this: +%% +%% Bool = is_nonempty_list L +%% +%% If 'is_nonempty_list L' evaluates to 'true', L must +%% must be cons. The meet of the previously known type of L and 'cons' +%% will be added to SuccTypes. +%% +%% On the other hand, if 'is_nonempty_list L' evaluates to false, L +%% is not cons and cons can be subtracted from the previously known +%% type for L. For example, if L was known to be 'list', subtracting +%% 'cons' would give 'nil' as the only possible type. The result of the +%% subtraction for L will be added to FailTypes. +%% +%% Here is another example, asking about the variable Bool: +%% +%% Head = bif:hd L +%% Bool = succeeded Head +%% +%% 'succeeded Head' will evaluate to 'true' if the instrution that +%% defined Head succeeded. In this case, it is the 'bif:hd L' +%% instruction, which will succeed if L is 'cons'. Thus, the meet of +%% the previous type for L and 'cons' will be added to SuccTypes. +%% +%% If 'succeeded Head' evaluates to 'false', it means that 'bif:hd L' +%% failed and that L is not 'cons'. 'cons' can be subtracted from the +%% previously known type for L and the result put in FailTypes. + +infer_types_br(#b_var{}=V, Ts, #d{ds=Ds}) -> + #{V:=#b_set{op=Op,args=Args}} = Ds, + Types0 = infer_type(Op, Args, Ds), + + %% We must be careful with types inferred from '=:='. + %% + %% If we have seen L =:= [a], we know that L is 'cons' if the + %% comparison succeeds. However, if the comparison fails, L could + %% still be 'cons'. Therefore, we must not subtract 'cons' from the + %% previous type of L. + %% + %% However, it is safe to subtract a type inferred from '=:=' if + %% it is single-valued, e.g. if it is [] or the atom 'true'. + EqTypes0 = infer_eq_type(Op, Args, Ts, Ds), + {Types1,EqTypes} = partition(fun({_,T}) -> + is_singleton_type(T) + end, EqTypes0), + + Types = Types1 ++ Types0, + {meet_types(EqTypes++Types, Ts),subtract_types(Types, Ts)}. + +infer_types_switch(V, Lit, Ts, #d{ds=Ds}) -> + Types = infer_eq_type({bif,'=:='}, [V, Lit], Ts, Ds), + meet_types(Types, Ts). + +infer_eq_type({bif,'=:='}, [#b_var{}=Src,#b_literal{}=Lit], Ts, Ds) -> + Def = maps:get(Src, Ds), + Type = get_type(Lit, Ts), + [{Src,Type} | infer_eq_lit(Def, Lit)]; +infer_eq_type({bif,'=:='}, [#b_var{}=Arg0,#b_var{}=Arg1], Ts, _Ds) -> + %% As an example, assume that L1 is known to be 'list', and L2 is + %% known to be 'cons'. Then if 'L1 =:= L2' evaluates to 'true', it can + %% be inferred that L1 is 'cons' (the meet of 'cons' and 'list'). + Type0 = get_type(Arg0, Ts), + Type1 = get_type(Arg1, Ts), + Type = meet(Type0, Type1), + [{V,MeetType} || + {V,OrigType,MeetType} <- + [{Arg0,Type0,Type},{Arg1,Type1,Type}], + OrigType =/= MeetType]; +infer_eq_type(_Op, _Args, _Ts, _Ds) -> + []. + +infer_eq_lit(#b_set{op={bif,tuple_size},args=[#b_var{}=Tuple]}, + #b_literal{val=Size}) when is_integer(Size) -> + [{Tuple,#t_tuple{exact=true,size=Size}}]; +infer_eq_lit(#b_set{op=get_tuple_element, + args=[#b_var{}=Tuple,#b_literal{val=N}]}, + #b_literal{}=Lit) -> + Index = N + 1, + Es = set_element_type(Index, get_type(Lit, #{}), #{}), + [{Tuple,#t_tuple{size=Index,elements=Es}}]; +infer_eq_lit(_, _) -> []. + +infer_type({bif,element}, [#b_literal{val=Pos},#b_var{}=Tuple], _Ds) -> + if + is_integer(Pos), 1 =< Pos -> + [{Tuple,#t_tuple{size=Pos}}]; + true -> + [] + end; +infer_type({bif,element}, [#b_var{}=Position,#b_var{}=Tuple], _Ds) -> + [{Position,t_integer()},{Tuple,#t_tuple{}}]; +infer_type({bif,Bif}, [#b_var{}=Src]=Args, _Ds) -> + case inferred_bif_type(Bif, Args) of + any -> []; + T -> [{Src,T}] + end; +infer_type({bif,binary_part}, [#b_var{}=Src,_], _Ds) -> + [{Src,{binary,8}}]; +infer_type({bif,is_map_key}, [_,#b_var{}=Src], _Ds) -> + [{Src,map}]; +infer_type({bif,map_get}, [_,#b_var{}=Src], _Ds) -> + [{Src,map}]; +infer_type({bif,Bif}, [_,_]=Args, _Ds) -> + case inferred_bif_type(Bif, Args) of + any -> []; + T -> [{A,T} || #b_var{}=A <- Args] + end; +infer_type({bif,binary_part}, [#b_var{}=Src,Pos,Len], _Ds) -> + [{Src,{binary,8}}| + [{V,t_integer()} || #b_var{}=V <- [Pos,Len]]]; +infer_type(bs_start_match, [#b_var{}=Bin], _Ds) -> + [{Bin,{binary,1}}]; +infer_type(is_nonempty_list, [#b_var{}=Src], _Ds) -> + [{Src,cons}]; +infer_type(is_tagged_tuple, [#b_var{}=Src,#b_literal{val=Size}, + #b_literal{}=Tag], _Ds) -> + Es = set_element_type(1, get_type(Tag, #{}), #{}), + [{Src,#t_tuple{exact=true,size=Size,elements=Es}}]; +infer_type(succeeded, [#b_var{}=Src], Ds) -> + #b_set{op=Op,args=Args} = maps:get(Src, Ds), + infer_type(Op, Args, Ds); +infer_type(_Op, _Args, _Ds) -> + []. + +%% bif_type(Name, Args) -> Type +%% Return the return type for the guard BIF or operator Name with +%% arguments Args. +%% +%% Note that that the following BIFs are handle elsewhere: +%% +%% band/2 + +bif_type(abs, [_]) -> number; +bif_type(bit_size, [_]) -> t_integer(); +bif_type(byte_size, [_]) -> t_integer(); +bif_type(ceil, [_]) -> t_integer(); +bif_type(float, [_]) -> float; +bif_type(floor, [_]) -> t_integer(); +bif_type(is_map_key, [_,_]) -> t_boolean(); +bif_type(length, [_]) -> t_integer(); +bif_type(map_size, [_]) -> t_integer(); +bif_type(node, []) -> #t_atom{}; +bif_type(node, [_]) -> #t_atom{}; +bif_type(round, [_]) -> t_integer(); +bif_type(size, [_]) -> t_integer(); +bif_type(trunc, [_]) -> t_integer(); +bif_type(tuple_size, [_]) -> t_integer(); +bif_type('bnot', [_]) -> t_integer(); +bif_type('bor', [_,_]) -> t_integer(); +bif_type('bsl', [_,_]) -> t_integer(); +bif_type('bsr', [_,_]) -> t_integer(); +bif_type('bxor', [_,_]) -> t_integer(); +bif_type('div', [_,_]) -> t_integer(); +bif_type('rem', [_,_]) -> t_integer(); +bif_type('/', [_,_]) -> float; +bif_type(Name, Args) -> + Arity = length(Args), + case erl_internal:new_type_test(Name, Arity) orelse + erl_internal:bool_op(Name, Arity) orelse + erl_internal:comp_op(Name, Arity) of + true -> + t_boolean(); + false -> + case erl_internal:arith_op(Name, Arity) of + true -> number; + false -> any + end + end. + +inferred_bif_type(is_atom, [_]) -> t_atom(); +inferred_bif_type(is_binary, [_]) -> {binary,8}; +inferred_bif_type(is_bitstring, [_]) -> {binary,1}; +inferred_bif_type(is_boolean, [_]) -> t_boolean(); +inferred_bif_type(is_float, [_]) -> float; +inferred_bif_type(is_integer, [_]) -> t_integer(); +inferred_bif_type(is_list, [_]) -> list; +inferred_bif_type(is_map, [_]) -> map; +inferred_bif_type(is_number, [_]) -> number; +inferred_bif_type(is_tuple, [_]) -> #t_tuple{}; +inferred_bif_type(abs, [_]) -> number; +inferred_bif_type(bit_size, [_]) -> {binary,1}; +inferred_bif_type('bnot', [_]) -> t_integer(); +inferred_bif_type(byte_size, [_]) -> {binary,1}; +inferred_bif_type(ceil, [_]) -> number; +inferred_bif_type(float, [_]) -> number; +inferred_bif_type(floor, [_]) -> number; +inferred_bif_type(hd, [_]) -> cons; +inferred_bif_type(length, [_]) -> list; +inferred_bif_type(map_size, [_]) -> map; +inferred_bif_type('not', [_]) -> t_boolean(); +inferred_bif_type(round, [_]) -> number; +inferred_bif_type(trunc, [_]) -> number; +inferred_bif_type(tl, [_]) -> cons; +inferred_bif_type(tuple_size, [_]) -> #t_tuple{}; +inferred_bif_type('and', [_,_]) -> t_boolean(); +inferred_bif_type('or', [_,_]) -> t_boolean(); +inferred_bif_type('xor', [_,_]) -> t_boolean(); +inferred_bif_type('band', [_,_]) -> t_integer(); +inferred_bif_type('bor', [_,_]) -> t_integer(); +inferred_bif_type('bsl', [_,_]) -> t_integer(); +inferred_bif_type('bsr', [_,_]) -> t_integer(); +inferred_bif_type('bxor', [_,_]) -> t_integer(); +inferred_bif_type('div', [_,_]) -> t_integer(); +inferred_bif_type('rem', [_,_]) -> t_integer(); +inferred_bif_type('+', [_,_]) -> number; +inferred_bif_type('-', [_,_]) -> number; +inferred_bif_type('*', [_,_]) -> number; +inferred_bif_type('/', [_,_]) -> number; +inferred_bif_type(_, _) -> any. + +is_math_bif(cos, 1) -> true; +is_math_bif(cosh, 1) -> true; +is_math_bif(sin, 1) -> true; +is_math_bif(sinh, 1) -> true; +is_math_bif(tan, 1) -> true; +is_math_bif(tanh, 1) -> true; +is_math_bif(acos, 1) -> true; +is_math_bif(acosh, 1) -> true; +is_math_bif(asin, 1) -> true; +is_math_bif(asinh, 1) -> true; +is_math_bif(atan, 1) -> true; +is_math_bif(atanh, 1) -> true; +is_math_bif(erf, 1) -> true; +is_math_bif(erfc, 1) -> true; +is_math_bif(exp, 1) -> true; +is_math_bif(log, 1) -> true; +is_math_bif(log2, 1) -> true; +is_math_bif(log10, 1) -> true; +is_math_bif(sqrt, 1) -> true; +is_math_bif(atan2, 2) -> true; +is_math_bif(pow, 2) -> true; +is_math_bif(ceil, 1) -> true; +is_math_bif(floor, 1) -> true; +is_math_bif(fmod, 2) -> true; +is_math_bif(pi, 0) -> true; +is_math_bif(_, _) -> false. + +join_types(Ts0, Ts1) -> + if + map_size(Ts0) < map_size(Ts1) -> + join_types_1(maps:keys(Ts0), Ts1, Ts0); + true -> + join_types_1(maps:keys(Ts1), Ts0, Ts1) + end. + +join_types_1([V|Vs], Ts0, Ts1) -> + case {Ts0,Ts1} of + {#{V:=Same},#{V:=Same}} -> + join_types_1(Vs, Ts0, Ts1); + {#{V:=T0},#{V:=T1}} -> + case join(T0, T1) of + T1 -> + join_types_1(Vs, Ts0, Ts1); + T -> + join_types_1(Vs, Ts0, Ts1#{V:=T}) + end; + {#{},#{V:=_}} -> + join_types_1(Vs, Ts0, Ts1) + end; +join_types_1([], Ts0, Ts1) -> + maps:merge(Ts0, Ts1). + +join([T1,T2|Ts]) -> + join([join(T1, T2)|Ts]); +join([T]) -> T. + +get_literal_from_type(#t_atom{elements=[Atom]}) -> + #b_literal{val=Atom}; +get_literal_from_type(#t_integer{elements={Int,Int}}) -> + #b_literal{val=Int}; +get_literal_from_type(nil) -> + #b_literal{val=[]}; +get_literal_from_type(_) -> none. + +t_atom() -> + #t_atom{elements=any}. + +t_atom(Atom) when is_atom(Atom) -> + #t_atom{elements=[Atom]}. + +t_boolean() -> + #t_atom{elements=[false,true]}. + +t_integer() -> + #t_integer{elements=any}. + +t_integer(Int) when is_integer(Int) -> + #t_integer{elements={Int,Int}}. + +t_integer(Min, Max) when is_integer(Min), is_integer(Max) -> + #t_integer{elements={Min,Max}}. + +t_is_boolean(#t_atom{elements=[F,T]}) -> + F =:= false andalso T =:= true; +t_is_boolean(#t_atom{elements=[B]}) -> + is_boolean(B); +t_is_boolean(_) -> false. + +t_tuple_size(#t_tuple{size=Size,exact=false}) -> + {at_least,Size}; +t_tuple_size(#t_tuple{size=Size,exact=true}) -> + {exact,Size}; +t_tuple_size(_) -> + none. + +is_singleton_type(Type) -> + get_literal_from_type(Type) =/= none. + +get_element_type(Index, Es) -> + case Es of + #{ Index := T } -> T; + #{} -> any + end. + +set_element_type(_Key, none, Es) -> + Es; +set_element_type(Key, any, Es) -> + maps:remove(Key, Es); +set_element_type(Key, Type, Es) -> + Es#{ Key => Type }. + +%% join(Type1, Type2) -> Type +%% Return the "join" of Type1 and Type2. The join is a more general +%% type than Type1 and Type2. For example: +%% +%% join(#t_integer{elements=any}, #t_integer=elements={0,3}}) -> +%% #t_integer{} +%% +%% The join for two different types result in 'any', which is +%% the top element for our type lattice: +%% +%% join(#t_integer{}, map) -> any + +-spec join(type(), type()) -> type(). + +join(T, T) -> + verified_type(T); +join(none, T) -> + verified_type(T); +join(T, none) -> + verified_type(T); +join(any, _) -> any; +join(_, any) -> any; +join(#t_atom{elements=[_|_]=Set1}, #t_atom{elements=[_|_]=Set2}) -> + Set = ordsets:union(Set1, Set2), + case ordsets:size(Set) of + Size when Size =< ?ATOM_SET_SIZE -> + #t_atom{elements=Set}; + _Size -> + #t_atom{elements=any} + end; +join(#t_atom{elements=any}=T, #t_atom{elements=[_|_]}) -> T; +join(#t_atom{elements=[_|_]}, #t_atom{elements=any}=T) -> T; +join({binary,U1}, {binary,U2}) -> + {binary,gcd(U1, U2)}; +join(#t_integer{}, #t_integer{}) -> t_integer(); +join(list, cons) -> list; +join(cons, list) -> list; +join(nil, cons) -> list; +join(cons, nil) -> list; +join(nil, list) -> list; +join(list, nil) -> list; +join(#t_integer{}, float) -> number; +join(float, #t_integer{}) -> number; +join(#t_integer{}, number) -> number; +join(number, #t_integer{}) -> number; +join(float, number) -> number; +join(number, float) -> number; +join(#t_tuple{size=Sz,exact=ExactA,elements=EsA}, + #t_tuple{size=Sz,exact=ExactB,elements=EsB}) -> + Exact = ExactA and ExactB, + Es = join_tuple_elements(Sz, EsA, EsB), + #t_tuple{size=Sz,exact=Exact,elements=Es}; +join(#t_tuple{size=SzA,elements=EsA}, #t_tuple{size=SzB,elements=EsB}) -> + Sz = min(SzA, SzB), + Es = join_tuple_elements(Sz, EsA, EsB), + #t_tuple{size=Sz,elements=Es}; +join(_T1, _T2) -> + %%io:format("~p ~p\n", [_T1,_T2]), + any. + +join_tuple_elements(MinSize, EsA, EsB) -> + Es0 = join_elements(EsA, EsB), + maps:filter(fun(Index, _Type) -> Index =< MinSize end, Es0). + +join_elements(Es1, Es2) -> + Keys = if + map_size(Es1) =< map_size(Es2) -> maps:keys(Es1); + map_size(Es1) > map_size(Es2) -> maps:keys(Es2) + end, + join_elements_1(Keys, Es1, Es2, #{}). + +join_elements_1([Key | Keys], Es1, Es2, Acc0) -> + case {Es1, Es2} of + {#{ Key := Type1 }, #{ Key := Type2 }} -> + Acc = set_element_type(Key, join(Type1, Type2), Acc0), + join_elements_1(Keys, Es1, Es2, Acc); + {#{}, #{}} -> + join_elements_1(Keys, Es1, Es2, Acc0) + end; +join_elements_1([], _Es1, _Es2, Acc) -> + Acc. + +gcd(A, B) -> + case A rem B of + 0 -> B; + X -> gcd(B, X) + end. + +meet_types([{V,T0}|Vs], Ts) -> + #{V:=T1} = Ts, + case meet(T0, T1) of + T1 -> meet_types(Vs, Ts); + T -> meet_types(Vs, Ts#{V:=T}) + end; +meet_types([], Ts) -> Ts. + +meet([T1,T2|Ts]) -> + meet([meet(T1, T2)|Ts]); +meet([T]) -> T. + +subtract_types([{V,T0}|Vs], Ts) -> + #{V:=T1} = Ts, + case subtract(T1, T0) of + T1 -> subtract_types(Vs, Ts); + T -> subtract_types(Vs, Ts#{V:=T}) + end; +subtract_types([], Ts) -> Ts. + +%% subtract(Type1, Type2) -> Type. +%% Subtract Type2 from Type1. Example: +%% +%% subtract(list, cons) -> nil + +subtract(#t_atom{elements=[_|_]=Set0}, #t_atom{elements=[_|_]=Set1}) -> + case ordsets:subtract(Set0, Set1) of + [] -> none; + [_|_]=Set -> #t_atom{elements=Set} + end; +subtract(number, float) -> #t_integer{}; +subtract(number, #t_integer{elements=any}) -> float; +subtract(list, cons) -> nil; +subtract(list, nil) -> cons; +subtract(T, _) -> T. + +%% meet(Type1, Type2) -> Type +%% Return the "meet" of Type1 and Type2. The meet is a narrower +%% type than Type1 and Type2. For example: +%% +%% meet(#t_integer{elements=any}, #t_integer{elements={0,3}}) -> +%% #t_integer{elements={0,3}} +%% +%% The meet for two different types result in 'none', which is +%% the bottom element for our type lattice: +%% +%% meet(#t_integer{}, map) -> none + +-spec meet(type(), type()) -> type(). + +meet(T, T) -> + verified_type(T); +meet(#t_atom{elements=[_|_]=Set1}, #t_atom{elements=[_|_]=Set2}) -> + case ordsets:intersection(Set1, Set2) of + [] -> + none; + [_|_]=Set -> + #t_atom{elements=Set} + end; +meet(#t_atom{elements=[_|_]}=T, #t_atom{elements=any}) -> + T; +meet(#t_atom{elements=any}, #t_atom{elements=[_|_]}=T) -> + T; +meet(#t_integer{elements={_,_}}=T, #t_integer{elements=any}) -> + T; +meet(#t_integer{elements=any}, #t_integer{elements={_,_}}=T) -> + T; +meet(#t_integer{elements={Min1,Max1}}, + #t_integer{elements={Min2,Max2}}) -> + #t_integer{elements={max(Min1, Min2),min(Max1, Max2)}}; +meet(#t_integer{}=T, number) -> T; +meet(float=T, number) -> T; +meet(number, #t_integer{}=T) -> T; +meet(number, float=T) -> T; +meet(list, cons) -> cons; +meet(list, nil) -> nil; +meet(cons, list) -> cons; +meet(nil, list) -> nil; +meet(#t_tuple{}=T1, #t_tuple{}=T2) -> + meet_tuples(T1, T2); +meet({binary,U1}, {binary,U2}) -> + {binary,max(U1, U2)}; +meet(any, T) -> + verified_type(T); +meet(T, any) -> + verified_type(T); +meet(_, _) -> + %% Inconsistent types. There will be an exception at runtime. + none. + +meet_tuples(#t_tuple{size=Sz1,exact=true}, + #t_tuple{size=Sz2,exact=true}) when Sz1 =/= Sz2 -> + none; +meet_tuples(#t_tuple{size=Sz1,exact=Ex1,elements=Es1}, + #t_tuple{size=Sz2,exact=Ex2,elements=Es2}) -> + Size = max(Sz1, Sz2), + Exact = Ex1 or Ex2, + case meet_elements(Es1, Es2) of + none -> + none; + Es -> + #t_tuple{size=Size,exact=Exact,elements=Es} + end. + +meet_elements(Es1, Es2) -> + Keys = maps:keys(Es1) ++ maps:keys(Es2), + meet_elements_1(Keys, Es1, Es2, #{}). + +meet_elements_1([Key | Keys], Es1, Es2, Acc) -> + case {Es1, Es2} of + {#{ Key := Type1 }, #{ Key := Type2 }} -> + case meet(Type1, Type2) of + none -> none; + Type -> meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type }) + end; + {#{ Key := Type1 }, _} -> + meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type1 }); + {_, #{ Key := Type2 }} -> + meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type2 }) + end; +meet_elements_1([], _Es1, _Es2, Acc) -> + Acc. + +%% verified_type(Type) -> Type +%% Returns the passed in type if it is one of the defined types. +%% Crashes if there is anything wrong with the type. +%% +%% Here are all possible types: +%% +%% any Any Erlang term (top element for the type lattice). +%% +%% #t_atom{} Any atom or some specific atoms. +%% {binary,Unit} Binary/bitstring aligned to unit Unit. +%% float Floating point number. +%% #t_integer{} Integer +%% list Empty or nonempty list. +%% map Map. +%% nil Empty list. +%% cons Cons (nonempty list). +%% number A number (float or integer). +%% #t_tuple{} Tuple. +%% +%% none No type (bottom element for the type lattice). + +-spec verified_type(T) -> T when + T :: type(). + +verified_type(any=T) -> T; +verified_type(none=T) -> T; +verified_type(#t_atom{elements=any}=T) -> T; +verified_type(#t_atom{elements=[_|_]}=T) -> T; +verified_type({binary,U}=T) when is_integer(U) -> T; +verified_type(#t_integer{elements=any}=T) -> T; +verified_type(#t_integer{elements={Min,Max}}=T) + when is_integer(Min), is_integer(Max) -> T; +verified_type(list=T) -> T; +verified_type(map=T) -> T; +verified_type(nil=T) -> T; +verified_type(cons=T) -> T; +verified_type(number=T) -> T; +verified_type(#t_tuple{size=Size,elements=Es}=T) -> + %% All known elements must have a valid index and type. 'any' is prohibited + %% since it's implicit and should never be present in the map. + maps:fold(fun(Index, Element, _) when is_integer(Index), + 1 =< Index, Index =< Size, + Element =/= any, Element =/= none -> + verified_type(Element) + end, [], Es), + T; +verified_type(float=T) -> T. diff --git a/lib/compiler/src/beam_trim.erl b/lib/compiler/src/beam_trim.erl index 4da0985085..acf3838da4 100644 --- a/lib/compiler/src/beam_trim.erl +++ b/lib/compiler/src/beam_trim.erl @@ -21,12 +21,11 @@ -module(beam_trim). -export([module/2]). --import(lists, [reverse/1,reverse/2,splitwith/2,sort/1]). +-import(lists, [any/2,member/2,reverse/1,reverse/2,splitwith/2,sort/1]). -record(st, - {safe :: gb_sets:set(beam_asm:label()), %Safe labels. - lbl :: beam_utils:code_index() %Code at each label. - }). + {safe :: cerl_sets:set(beam_asm:label()) %Safe labels. + }). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. @@ -36,10 +35,15 @@ module({Mod,Exp,Attr,Fs0,Lc}, _Opts) -> {ok,{Mod,Exp,Attr,Fs,Lc}}. function({function,Name,Arity,CLabel,Is0}) -> - %%ok = io:fwrite("~w: ~p\n", [?LINE,{Name,Arity}]), - St = #st{safe=safe_labels(Is0, []),lbl=beam_utils:index_labels(Is0)}, - Is = trim(Is0, St, []), - {function,Name,Arity,CLabel,Is}. + try + St = #st{safe=safe_labels(Is0, [])}, + Is = trim(Is0, St, []), + {function,Name,Arity,CLabel,Is} + catch + Class:Error:Stack -> + io:fwrite("Function: ~w/~w\n", [Name,Arity]), + erlang:raise(Class, Error, Stack) + end. trim([{kill,_}|_]=Is0, St, Acc) -> {Kills0,Is1} = splitwith(fun({kill,_}) -> true; @@ -47,14 +51,33 @@ trim([{kill,_}|_]=Is0, St, Acc) -> end, Is0), Kills = sort(Kills0), try - {FrameSize,Layout} = frame_layout(Is1, Kills, St), - Configs = trim_instructions(Layout), - try_remap(Configs, Is1, FrameSize) - of + %% Find out the size and layout of the stack frame. + %% Example of a layout: + %% + %% [{kill,{y,0}},{dead,{y,1},{live,{y,2}},{kill,{y,3}}] + %% + %% That means that y0 and y3 are to be killed, that y1 + %% has been killed previously, and that y2 is live. + {FrameSize,Layout} = frame_layout(Is1, Kills, St), + + %% Calculate all recipes that are not worse in terms + %% of estimated execution time. The recipes are ordered + %% in descending order from how much they trim. + Recipes = trim_recipes(Layout), + + %% Try the recipes in order. A recipe may not work out because + %% a register that was previously killed may be + %% resurrected. If that happens, the next recipe, which trims + %% less, will be tried. + try_remap(Recipes, Is1, FrameSize) + of {Is,TrimInstr} -> + %% One of the recipes was applied. trim(Is, St, reverse(TrimInstr)++Acc) catch not_possible -> + %% No recipe worked out. Use the original kill + %% instructions. trim(Is1, St, reverse(Kills, Acc)) end; trim([I|Is], St, Acc) -> @@ -62,34 +85,42 @@ trim([I|Is], St, Acc) -> trim([], _, Acc) -> reverse(Acc). -%% trim_instructions([{kill,R}|{live,R}|{dead,R}]) -> {[Instruction],MapFun} -%% Figure out the sequence of moves and trim to use. +%% trim_recipes([{kill,R}|{live,R}|{dead,R}]) -> [Recipe]. +%% Recipe = {Kills,NumberToTrim,Moves} +%% Kills = [{kill,Y}] +%% Moves = [{move,SrcY,DstY}] +%% +%% Calculate how to best trim the stack and kill the correct +%% Y registers. Return a list of possible recipes. The best +%% recipe (the one that trims the most) is first in the list. +%% All of the recipes are no worse in estimated execution time +%% than the original sequences of kill instructions. -trim_instructions(Layout) -> +trim_recipes(Layout) -> Cost = length([I || {kill,_}=I <- Layout]), - trim_instructions_1(Layout, 0, [], {Cost,[]}). + trim_recipes_1(Layout, 0, [], {Cost,[]}). -trim_instructions_1([{kill,{y,Trim0}}|Ks], Trim0, Moves, Config0) -> +trim_recipes_1([{kill,{y,Trim0}}|Ks], Trim0, Moves, Recipes0) -> Trim = Trim0 + 1, - Config = save_config(Ks, Trim, Moves, Config0), - trim_instructions_1(Ks, Trim, Moves, Config); -trim_instructions_1([{dead,{y,Trim0}}|Ks], Trim0, Moves, Config0) -> + Recipes = save_recipe(Ks, Trim, Moves, Recipes0), + trim_recipes_1(Ks, Trim, Moves, Recipes); +trim_recipes_1([{dead,{y,Trim0}}|Ks], Trim0, Moves, Recipes0) -> Trim = Trim0 + 1, - Config = save_config(Ks, Trim, Moves, Config0), - trim_instructions_1(Ks, Trim, Moves, Config); -trim_instructions_1([{live,{y,Trim0}=Src}|Ks0], Trim0, Moves0, Config0) -> + Recipes = save_recipe(Ks, Trim, Moves, Recipes0), + trim_recipes_1(Ks, Trim, Moves, Recipes); +trim_recipes_1([{live,{y,Trim0}=Src}|Ks0], Trim0, Moves0, Recipes0) -> case take_last_dead(Ks0) of none -> - {_,ConfigList} = Config0, - ConfigList; + {_,RecipesList} = Recipes0, + RecipesList; {Dst,Ks} -> Trim = Trim0 + 1, Moves = [{move,Src,Dst}|Moves0], - Config = save_config(Ks, Trim, Moves, Config0), - trim_instructions_1(Ks, Trim, Moves, Config) + Recipes = save_recipe(Ks, Trim, Moves, Recipes0), + trim_recipes_1(Ks, Trim, Moves, Recipes) end; -trim_instructions_1([], _, _, {_,ConfigList}) -> - ConfigList. +trim_recipes_1([], _, _, {_,RecipesList}) -> + RecipesList. take_last_dead(L) -> take_last_dead_1(reverse(L)). @@ -100,28 +131,48 @@ take_last_dead_1([{dead,Reg}|Is]) -> {Reg,reverse(Is)}; take_last_dead_1(_) -> none. -save_config(Ks, Trim, Moves, {MaxCost,Acc}=Config) -> - case config_cost(Ks, Moves) of - Cost when Cost =< MaxCost -> - {MaxCost,[{Ks,Trim,Moves}|Acc]}; +save_recipe(Ks, Trim, Moves, {MaxCost,Acc}=Recipes) -> + case recipe_cost(Ks, Moves) of + Cost when Cost =< MaxCost -> + %% The price is right. + {MaxCost,[{Ks,Trim,Moves}|Acc]}; _Cost -> - Config + %% Too expensive. + Recipes end. -config_cost(Ks, Moves) -> +recipe_cost(Ks, Moves) -> %% We estimate that a {move,{y,_},{y,_}} instruction is roughly twice as %% expensive as a {kill,{y,_}} instruction. A {trim,_} instruction is %% roughly as expensive as a {kill,{y,_}} instruction. - config_cost_1(Ks, 1+2*length(Moves)). + recipe_cost_1(Ks, 1+2*length(Moves)). -config_cost_1([{kill,_}|Ks], Cost) -> - config_cost_1(Ks, Cost+1); -config_cost_1([_|Ks], Cost) -> - config_cost_1(Ks, Cost); -config_cost_1([], Cost) -> Cost. +recipe_cost_1([{kill,_}|Ks], Cost) -> + recipe_cost_1(Ks, Cost+1); +recipe_cost_1([_|Ks], Cost) -> + recipe_cost_1(Ks, Cost); +recipe_cost_1([], Cost) -> Cost. -expand_config({Layout,Trim,Moves}, FrameSize) -> +%% try_remap([Recipe], [Instruction], FrameSize) -> +%% {[Instruction],[TrimInstruction]}. +%% Try to renumber Y registers in the instruction stream. The +%% first rececipe that works will be used. +%% +%% This function will issue a `not_possible` exception if none +%% of the recipes were possible to apply. + +try_remap([R|Rs], Is, FrameSize) -> + {TrimInstr,Map} = expand_recipe(R, FrameSize), + try + {remap(Is, Map, []),TrimInstr} + catch + throw:not_possible -> + try_remap(Rs, Is, FrameSize) + end; +try_remap([], _, _) -> throw(not_possible). + +expand_recipe({Layout,Trim,Moves}, FrameSize) -> Kills = [Kill || {kill,_}=Kill <- Layout], {Kills++reverse(Moves, [{trim,Trim,FrameSize-Trim}]),create_map(Trim, Moves)}. @@ -132,16 +183,16 @@ create_map(Trim, []) -> (Any) -> Any end; create_map(Trim, Moves) -> - GbTree0 = [{Src,Dst-Trim} || {move,{y,Src},{y,Dst}} <- Moves], - GbTree = gb_trees:from_orddict(sort(GbTree0)), - IllegalTargets = gb_sets:from_list([Dst || {move,_,{y,Dst}} <- Moves]), + Map0 = [{Src,Dst-Trim} || {move,{y,Src},{y,Dst}} <- Moves], + Map = maps:from_list(Map0), + IllegalTargets = cerl_sets:from_list([Dst || {move,_,{y,Dst}} <- Moves]), fun({y,Y0}) when Y0 < Trim -> - case gb_trees:lookup(Y0, GbTree) of - {value,Y} -> {y,Y}; - none -> throw(not_possible) - end; + case Map of + #{Y0:=Y} -> {y,Y}; + #{} -> throw(not_possible) + end; ({y,Y}) -> - case gb_sets:is_element(Y, IllegalTargets) of + case cerl_sets:is_element(Y, IllegalTargets) of true -> throw(not_possible); false -> {y,Y-Trim} end; @@ -149,19 +200,17 @@ create_map(Trim, Moves) -> (Any) -> Any end. -try_remap([C|Cs], Is, FrameSize) -> - {TrimInstr,Map} = expand_config(C, FrameSize), - try - {remap(Is, Map, []),TrimInstr} - catch - throw:not_possible -> - try_remap(Cs, Is, FrameSize) - end; -try_remap([], _, _) -> throw(not_possible). - +remap([{'%',_}=I|Is], Map, Acc) -> + remap(Is, Map, [I|Acc]); remap([{block,Bl0}|Is], Map, Acc) -> Bl = remap_block(Bl0, Map, []), remap(Is, Map, [{block,Bl}|Acc]); +remap([{bs_get_tail,Src,Dst,Live}|Is], Map, Acc) -> + I = {bs_get_tail,Map(Src),Map(Dst),Live}, + remap(Is, Map, [I|Acc]); +remap([{bs_set_position,Src1,Src2}|Is], Map, Acc) -> + I = {bs_set_position,Map(Src1),Map(Src2)}, + remap(Is, Map, [I|Acc]); remap([{call_fun,_}=I|Is], Map, Acc) -> remap(Is, Map, [I|Acc]); remap([{call,_,_}=I|Is], Map, Acc) -> @@ -205,35 +254,68 @@ remap([return|_]=Is, _, Acc) -> reverse(Acc, Is); remap([{line,_}=I|Is], Map, Acc) -> remap(Is, Map, [I|Acc]). - + remap_block([{set,Ds0,Ss0,Info}|Is], Map, Acc) -> Ds = [Map(D) || D <- Ds0], Ss = [Map(S) || S <- Ss0], remap_block(Is, Map, [{set,Ds,Ss,Info}|Acc]); remap_block([], _, Acc) -> reverse(Acc). - -safe_labels([{label,L},{line,_},{badmatch,{Tag,_}}|Is], Acc) when Tag =/= y -> - safe_labels(Is, [L|Acc]); -safe_labels([{label,L},{line,_},{case_end,{Tag,_}}|Is], Acc) when Tag =/= y -> - safe_labels(Is, [L|Acc]); -safe_labels([{label,L},{line,_},if_end|Is], Acc) -> - safe_labels(Is, [L|Acc]); -safe_labels([{label,L}, - {block,[{set,[{x,0}],[{Tag,_}],move}]}, - {line,_}, - {call_ext,1,{extfunc,erlang,error,1}}|Is], Acc) when Tag =/= y -> - safe_labels(Is, [L|Acc]); + +%% safe_labels([Instruction], Accumulator) -> gb_set() +%% Build a gb_set of safe labels. The code at a safe +%% label does not depend on the values in a specific +%% Y register, only that all Y registers are initialized +%% so that it safe to scan the stack when an exception +%% is generated. +%% +%% In other words, code at a safe label will continue +%% to work if Y registers have been renumbered and +%% the size of the stack frame has changed. + +safe_labels([{label,L}|Is], Acc) -> + case is_safe_label(Is) of + true -> safe_labels(Is, [L|Acc]); + false -> safe_labels(Is, Acc) + end; safe_labels([_|Is], Acc) -> safe_labels(Is, Acc); -safe_labels([], Acc) -> gb_sets:from_list(Acc). +safe_labels([], Acc) -> cerl_sets:from_list(Acc). + +is_safe_label([{'%',_}|Is]) -> + is_safe_label(Is); +is_safe_label([{line,_}|Is]) -> + is_safe_label(Is); +is_safe_label([{badmatch,{Tag,_}}|_]) -> + Tag =/= y; +is_safe_label([{case_end,{Tag,_}}|_]) -> + Tag =/= y; +is_safe_label([{try_case_end,{Tag,_}}|_]) -> + Tag =/= y; +is_safe_label([if_end|_]) -> + true; +is_safe_label([{block,Bl}|Is]) -> + is_safe_label_block(Bl) andalso is_safe_label(Is); +is_safe_label([{call_ext,_,{extfunc,M,F,A}}|_]) -> + erl_bifs:is_exit_bif(M, F, A); +is_safe_label(_) -> false. + +is_safe_label_block([{set,Ds,Ss,_}|Is]) -> + IsYreg = fun({y,_}) -> true; + (_) -> false + end, + %% This instruction is safe if the instruction + %% neither reads or writes Y registers. + not (any(IsYreg, Ss) orelse any(IsYreg, Ds)) andalso + is_safe_label_block(Is); +is_safe_label_block([]) -> true. %% frame_layout([Instruction], [{kill,_}], St) -> %% [{kill,Reg} | {live,Reg} | {dead,Reg}] %% Figure out the layout of the stack frame. -frame_layout(Is, Kills, #st{safe=Safe,lbl=D}) -> +frame_layout(Is, Kills, #st{safe=Safe}) -> N = frame_size(Is, Safe), - IsKilled = fun(R) -> beam_utils:is_not_used(R, Is, D) end, + IsKilled = fun(R) -> is_not_used(R, Is) end, {N,frame_layout_1(Kills, 0, N, IsKilled, [])}. frame_layout_1([{kill,{y,Y}}=I|Ks], Y, N, IsKilled, Acc) -> @@ -253,7 +335,14 @@ frame_layout_2(Is) -> reverse(Is). %% frame_size([Instruction], SafeLabels) -> FrameSize %% Find out the frame size by looking at the code that follows. +%% +%% Implicitly, also check that the instructions are a straight +%% sequence of code that ends in a return. Any branches are +%% to safe labels (i.e., the code at those labels don't depend +%% on the contents of any Y register). +frame_size([{'%',_}|Is], Safe) -> + frame_size(Is, Safe); frame_size([{block,_}|Is], Safe) -> frame_size(Is, Safe); frame_size([{call_fun,_}|Is], Safe) -> @@ -285,15 +374,94 @@ frame_size([{make_fun2,_,_,_,_}|Is], Safe) -> frame_size(Is, Safe); frame_size([{get_map_elements,{f,L},_,_}|Is], Safe) -> frame_size_branch(L, Is, Safe); -frame_size([{deallocate,N}|_], _) -> N; +frame_size([{deallocate,N}|_], _) -> + N; frame_size([{line,_}|Is], Safe) -> frame_size(Is, Safe); -frame_size([_|_], _) -> throw(not_possible). +frame_size([{bs_set_position,_,_}|Is], Safe) -> + frame_size(Is, Safe); +frame_size([{bs_get_tail,_,_,_}|Is], Safe) -> + frame_size(Is, Safe); +frame_size(_, _) -> throw(not_possible). frame_size_branch(0, Is, Safe) -> frame_size(Is, Safe); frame_size_branch(L, Is, Safe) -> - case gb_sets:is_member(L, Safe) of + case cerl_sets:is_element(L, Safe) of false -> throw(not_possible); true -> frame_size(Is, Safe) end. + +%% is_not_used(Y, [Instruction]) -> true|false. +%% Test whether the value of Y is unused in the instruction sequence. +%% Return true if the value of Y is not used, and false if it is used. +%% +%% This function handles the same instructions as frame_size/2. It +%% assumes that any labels in the instructions are safe labels. + +is_not_used(Y, [{'%',_}|Is]) -> + is_not_used(Y, Is); +is_not_used(Y, [{apply,_}|Is]) -> + is_not_used(Y, Is); +is_not_used(Y, [{bif,_,{f,_},Ss,Dst}|Is]) -> + is_not_used_ss_dst(Y, Ss, Dst, Is); +is_not_used(Y, [{block,Bl}|Is]) -> + case is_not_used_block(Y, Bl) of + used -> false; + killed -> true; + transparent -> is_not_used(Y, Is) + end; +is_not_used(Y, [{bs_get_tail,Src,Dst,_}|Is]) -> + is_not_used_ss_dst(Y, [Src], Dst, Is); +is_not_used(Y, [{bs_init,_,_,_,Ss,Dst}|Is]) -> + is_not_used_ss_dst(Y, Ss, Dst, Is); +is_not_used(Y, [{bs_put,{f,_},_,Ss}|Is]) -> + not member(Y, Ss) andalso is_not_used(Y, Is); +is_not_used(Y, [{bs_set_position,Src1,Src2}|Is]) -> + Y =/= Src1 andalso Y =/= Src2 andalso + is_not_used(Y, Is); +is_not_used(Y, [{call,_,_}|Is]) -> + is_not_used(Y, Is); +is_not_used(Y, [{call_ext,_,_}=I|Is]) -> + beam_jump:is_exit_instruction(I) orelse is_not_used(Y, Is); +is_not_used(Y, [{call_fun,_}|Is]) -> + is_not_used(Y, Is); +is_not_used(_Y, [{deallocate,_}|_]) -> + true; +is_not_used(Y, [{gc_bif,_,{f,_},_Live,Ss,Dst}|Is]) -> + is_not_used_ss_dst(Y, Ss, Dst, Is); +is_not_used(Y, [{get_map_elements,{f,_},S,{list,List}}|Is]) -> + {Ss,Ds} = beam_utils:split_even(List), + case member(Y, [S|Ss]) of + true -> + false; + false -> + member(Y, Ds) orelse is_not_used(Y, Is) + end; +is_not_used(Y, [{kill,Yreg}|Is]) -> + Y =:= Yreg orelse is_not_used(Y, Is); +is_not_used(Y, [{line,_}|Is]) -> + is_not_used(Y, Is); +is_not_used(Y, [{make_fun2,_,_,_,_}|Is]) -> + is_not_used(Y, Is); +is_not_used(Y, [{test,_,_,Ss}|Is]) -> + not member(Y, Ss) andalso is_not_used(Y, Is); +is_not_used(Y, [{test,_Op,{f,_},_Live,Ss,Dst}|Is]) -> + is_not_used_ss_dst(Y, Ss, Dst, Is). + +is_not_used_block(Y, [{set,Ds,Ss,_}|Is]) -> + case member(Y, Ss) of + true -> + used; + false -> + case member(Y, Ds) of + true -> + killed; + false -> + is_not_used_block(Y, Is) + end + end; +is_not_used_block(_Y, []) -> transparent. + +is_not_used_ss_dst(Y, Ss, Dst, Is) -> + not member(Y, Ss) andalso (Y =:= Dst orelse is_not_used(Y, Is)). diff --git a/lib/compiler/src/beam_type.erl b/lib/compiler/src/beam_type.erl deleted file mode 100644 index a1e9eff8f3..0000000000 --- a/lib/compiler/src/beam_type.erl +++ /dev/null @@ -1,1118 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1999-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% -%% Purpose: Type-based optimisations. See the comment for verified_type/1 -%% the very end of this file for a description of the types in the -%% type database. - --module(beam_type). - --export([module/2]). - --import(lists, [foldl/3,member/2,reverse/1,reverse/2,sort/1]). - --define(UNICODE_INT, {integer,{0,16#10FFFF}}). - --spec module(beam_utils:module_code(), [compile:option()]) -> - {'ok',beam_utils:module_code()}. - -module({Mod,Exp,Attr,Fs0,Lc}, _Opts) -> - Fs = [function(F) || F <- Fs0], - {ok,{Mod,Exp,Attr,Fs,Lc}}. - -function({function,Name,Arity,CLabel,Asm0}) -> - try - Asm1 = beam_utils:live_opt(Asm0), - Asm2 = opt(Asm1, [], tdb_new()), - Asm3 = beam_utils:live_opt(Asm2), - Asm = beam_utils:delete_annos(Asm3), - {function,Name,Arity,CLabel,Asm} - catch - Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [Name,Arity]), - erlang:raise(Class, Error, Stack) - end. - -%% opt([Instruction], Accumulator, TypeDb) -> {[Instruction'],TypeDb'} -%% Keep track of type information; try to simplify. - -opt([{block,Body1}|Is], [{block,Body0}|Acc], Ts0) -> - {Body2,Ts} = simplify(Body1, Ts0), - Body = merge_blocks(Body0, Body2), - opt(Is, [{block,Body}|Acc], Ts); -opt([{block,Body0}|Is], Acc, Ts0) -> - {Body,Ts} = simplify(Body0, Ts0), - opt(Is, [{block,Body}|Acc], Ts); -opt([I0|Is], Acc, Ts0) -> - case simplify_basic([I0], Ts0) of - {[],Ts} -> opt(Is, Acc, Ts); - {[I],Ts} -> opt(Is, [I|Acc], Ts) - end; -opt([], Acc, _) -> reverse(Acc). - -%% simplify(Instruction, TypeDb) -> NewInstruction -%% Simplify an instruction using type information (this is -%% technically a "strength reduction"). - -simplify(Is0, TypeDb0) -> - {Is,_} = BasicRes = simplify_basic(Is0, TypeDb0), - case simplify_float(Is, TypeDb0) of - not_possible -> BasicRes; - {_,_}=Res -> Res - end. - -%% simplify_basic([Instruction], TypeDatabase) -> {[Instruction],TypeDatabase'} -%% Basic simplification, mostly tuples, no floating point optimizations. - -simplify_basic(Is, Ts) -> - simplify_basic(Is, Ts, []). - -simplify_basic([I0|Is], Ts0, Acc) -> - case simplify_instr(I0, Ts0) of - [] -> - simplify_basic(Is, Ts0, Acc); - [I] -> - Ts = update(I, Ts0), - simplify_basic(Is, Ts, [I|Acc]) - end; -simplify_basic([], Ts, Acc) -> - {reverse(Acc),Ts}. - -%% simplify_instr(Instruction, Ts) -> [Instruction]. - -%% Simplify a simple instruction using type information. Return an -%% empty list if the instruction should be removed, or a list with -%% the original or modified instruction. - -simplify_instr({set,[D],[{integer,Index},Reg],{bif,element,_}}=I, Ts) -> - case max_tuple_size(Reg, Ts) of - Sz when 0 < Index, Index =< Sz -> - [{set,[D],[Reg],{get_tuple_element,Index-1}}]; - _ -> [I] - end; -simplify_instr({test,Test,Fail,[R]}=I, Ts) -> - case tdb_find(R, Ts) of - any -> - [I]; - Type -> - case will_succeed(Test, Type) of - yes -> []; - no -> [{jump,Fail}]; - maybe -> [I] - end - end; -simplify_instr({set,[D],[TupleReg],{get_tuple_element,0}}=I, Ts) -> - case tdb_find(TupleReg, Ts) of - {tuple,_,_,[Contents]} -> - [{set,[D],[Contents],move}]; - _ -> - [I] - end; -simplify_instr({test,test_arity,_,[R,Arity]}=I, Ts) -> - case tdb_find(R, Ts) of - {tuple,exact_size,Arity,_} -> []; - _ -> [I] - end; -simplify_instr({test,is_eq_exact,Fail,[R,{atom,A}=Atom]}=I, Ts) -> - case tdb_find(R, Ts) of - {atom,_}=Atom -> []; - boolean when is_boolean(A) -> [I]; - any -> [I]; - _ -> [{jump,Fail}] - end; -simplify_instr({test,is_record,_,[R,{atom,_}=Tag,{integer,Arity}]}=I, Ts) -> - case tdb_find(R, Ts) of - {tuple,exact_size,Arity,[Tag]} -> []; - _ -> [I] - end; -simplify_instr({select,select_val,Reg,_,_}=I, Ts) -> - [case tdb_find(Reg, Ts) of - {integer,Range} -> - simplify_select_val_int(I, Range); - boolean -> - simplify_select_val_bool(I); - _ -> - I - end]; -simplify_instr({test,bs_test_unit,_,[Src,Unit]}=I, Ts) -> - case tdb_find(Src, Ts) of - {binary,U} when U rem Unit =:= 0 -> []; - _ -> [I] - end; -simplify_instr(I, _) -> [I]. - -simplify_select_val_int({select,select_val,R,_,L0}=I, {Min,Max}) -> - Vs = sort([V || {integer,V} <- L0]), - case eq_ranges(Vs, Min, Max) of - false -> I; - true -> simplify_select_val_1(L0, {integer,Max}, R, []) - end. - -simplify_select_val_bool({select,select_val,R,_,L}=I) -> - Vs = sort([V || {atom,V} <- L]), - case Vs of - [false,true] -> - simplify_select_val_1(L, {atom,false}, R, []); - _ -> - I - end. - -simplify_select_val_1([Val,F|T], Val, R, Acc) -> - L = reverse(Acc, T), - {select,select_val,R,F,L}; -simplify_select_val_1([V,F|T], Val, R, Acc) -> - simplify_select_val_1(T, Val, R, [F,V|Acc]). - -eq_ranges([H], H, H) -> true; -eq_ranges([H|T], H, Max) -> eq_ranges(T, H+1, Max); -eq_ranges(_, _, _) -> false. - -%% will_succeed(TestOperation, Type) -> yes|no|maybe. -%% Test whether TestOperation applied to an argument of type Type -%% will succeed. Return yes, no, or maybe. -%% -%% Type is a type as described in the comment for verified_type/1 at -%% the very end of this file, but it will *never* be 'any'. - -will_succeed(is_atom, Type) -> - case Type of - {atom,_} -> yes; - boolean -> yes; - _ -> no - end; -will_succeed(is_binary, Type) -> - case Type of - {binary,U} when U rem 8 =:= 0 -> yes; - {binary,_} -> maybe; - _ -> no - end; -will_succeed(is_bitstr, Type) -> - case Type of - {binary,_} -> yes; - _ -> no - end; -will_succeed(is_integer, Type) -> - case Type of - integer -> yes; - {integer,_} -> yes; - _ -> no - end; -will_succeed(is_map, Type) -> - case Type of - map -> yes; - _ -> no - end; -will_succeed(is_nonempty_list, Type) -> - case Type of - nonempty_list -> yes; - _ -> no - end; -will_succeed(is_tuple, Type) -> - case Type of - {tuple,_,_,_} -> yes; - _ -> no - end; -will_succeed(_, _) -> maybe. - -%% simplify_float([Instruction], TypeDatabase) -> -%% {[Instruction],TypeDatabase'} | not_possible -%% Simplify floating point operations in blocks. -%% -simplify_float(Is0, Ts0) -> - {Is1,Ts} = simplify_float_1(Is0, Ts0, [], []), - Is2 = opt_fmoves(Is1, []), - Is3 = flt_need_heap(Is2), - try - {flt_liveness(Is3),Ts} - catch - throw:not_possible -> not_possible - end. - -simplify_float_1([{set,[],[],fclearerror}|Is], Ts, Rs, Acc) -> - simplify_float_1(Is, Ts, Rs, clearerror(Acc)); -simplify_float_1([{set,[],[],fcheckerror}|Is], Ts, Rs, Acc) -> - simplify_float_1(Is, Ts, Rs, checkerror(Acc)); -simplify_float_1([{set,[{fr,_}],_,_}=I|Is], Ts, Rs, Acc) -> - simplify_float_1(Is, Ts, Rs, [I|Acc]); -simplify_float_1([{set,[D0],[A0],{alloc,_,{gc_bif,'-',{f,0}}}}=I|Is]=Is0, - Ts0, Rs0, Acc0) -> - case tdb_find(A0, Ts0) of - float -> - A = coerce_to_float(A0), - {Rs1,Acc1} = load_reg(A, Ts0, Rs0, Acc0), - {D,Rs} = find_dest(D0, Rs1), - Areg = fetch_reg(A, Rs), - Acc = [{set,[D],[Areg],{bif,fnegate,{f,0}}}|clearerror(Acc1)], - Ts = tdb_store(D0, float, Ts0), - simplify_float_1(Is, Ts, Rs, Acc); - _Other -> - Ts = update(I, Ts0), - {Rs,Acc} = flush(Rs0, Is0, Acc0), - simplify_float_1(Is, Ts, Rs, [I|checkerror(Acc)]) - end; -simplify_float_1([{set,[D0],[A0,B0],{alloc,_,{gc_bif,Op0,{f,0}}}}=I|Is]=Is0, - Ts0, Rs0, Acc0) -> - case float_op(Op0, A0, B0, Ts0) of - no -> - Ts = update(I, Ts0), - {Rs,Acc} = flush(Rs0, Is0, Acc0), - simplify_float_1(Is, Ts, Rs, [I|checkerror(Acc)]); - {yes,Op} -> - A = coerce_to_float(A0), - B = coerce_to_float(B0), - {Rs1,Acc1} = load_reg(A, Ts0, Rs0, Acc0), - {Rs2,Acc2} = load_reg(B, Ts0, Rs1, Acc1), - {D,Rs} = find_dest(D0, Rs2), - Areg = fetch_reg(A, Rs), - Breg = fetch_reg(B, Rs), - Acc = [{set,[D],[Areg,Breg],{bif,Op,{f,0}}}|clearerror(Acc2)], - Ts = tdb_store(D0, float, Ts0), - simplify_float_1(Is, Ts, Rs, Acc) - end; -simplify_float_1([{set,_,_,{try_catch,_,_}}=I|Is]=Is0, _Ts, Rs0, Acc0) -> - Acc = flush_all(Rs0, Is0, Acc0), - simplify_float_1(Is, tdb_new(), Rs0, [I|Acc]); -simplify_float_1([{set,_,_,{line,_}}=I|Is], Ts, Rs, Acc) -> - simplify_float_1(Is, Ts, Rs, [I|Acc]); -simplify_float_1([I|Is], Ts0, [], Acc) -> - Ts = update(I, Ts0), - simplify_float_1(Is, Ts, [], [I|Acc]); -simplify_float_1([I|Is]=Is0, Ts0, Rs0, Acc0) -> - Ts = update(I, Ts0), - {Rs,Acc} = flush(Rs0, Is0, Acc0), - simplify_float_1(Is, Ts, Rs, [I|checkerror(Acc)]); -simplify_float_1([], Ts, [], Acc) -> - Is = reverse(Acc), - {Is,Ts}. - -coerce_to_float({integer,I}=Int) -> - try float(I) of - F -> - {float,F} - catch _:_ -> - %% Let the overflow happen at run-time. - Int - end; -coerce_to_float(Other) -> Other. - -opt_fmoves([{set,[{x,_}=R],[{fr,_}]=Src,fmove}=I1, - {set,[_]=Dst,[{x,_}=R],move}=I2|Is], Acc) -> - case beam_utils:is_killed_block(R, Is) of - false -> opt_fmoves(Is, [I2,I1|Acc]); - true -> opt_fmoves(Is, [{set,Dst,Src,fmove}|Acc]) - end; -opt_fmoves([I|Is], Acc) -> - opt_fmoves(Is, [I|Acc]); -opt_fmoves([], Acc) -> reverse(Acc). - -clearerror(Is) -> - clearerror(Is, Is). - -clearerror([{set,[],[],fclearerror}|_], OrigIs) -> OrigIs; -clearerror([{set,[],[],fcheckerror}|_], OrigIs) -> [{set,[],[],fclearerror}|OrigIs]; -clearerror([_|Is], OrigIs) -> clearerror(Is, OrigIs); -clearerror([], OrigIs) -> [{set,[],[],fclearerror}|OrigIs]. - -%% merge_blocks(Block1, Block2) -> Block. -%% Combine two blocks and eliminate any move instructions that assign -%% to registers that are killed later in the block. -%% -merge_blocks(B1, [{'%anno',_}|B2]) -> - merge_blocks_1(B1++[{set,[],[],stop_here}|B2]). - -merge_blocks_1([{set,[],_,stop_here}|Is]) -> Is; -merge_blocks_1([{set,[D],_,move}=I|Is]) -> - case beam_utils:is_killed_block(D, Is) of - true -> merge_blocks_1(Is); - false -> [I|merge_blocks_1(Is)] - end; -merge_blocks_1([I|Is]) -> [I|merge_blocks_1(Is)]. - -%% flt_need_heap([Instruction]) -> [Instruction] -%% Insert need heap allocation instructions in the instruction stream -%% to properly account for both inserted floating point operations and -%% normal term build operations (such as put_list/3). -%% -%% Ignore old heap allocation instructions (except if they allocate a stack -%% frame too), as they may be in the wrong place (because gc_bif instructions -%% could have been converted to floating point operations). - -flt_need_heap(Is) -> - flt_need_heap_1(reverse(Is), 0, 0, []). - -flt_need_heap_1([{set,[],[],{alloc,_,Alloc}}|Is], H, Fl, Acc) -> - case Alloc of - {_,nostack,_,_} -> - %% Remove any existing test_heap/2 instruction. - flt_need_heap_1(Is, H, Fl, Acc); - {Z,Stk,_,Inits} when is_integer(Stk) -> - %% Keep any allocate*/2 instruction and recalculate heap need. - I = {set,[],[],{alloc,regs,{Z,Stk,build_alloc(H, Fl),Inits}}}, - flt_need_heap_1(Is, 0, 0, [I|Acc]) - end; -flt_need_heap_1([I|Is], H0, Fl0, Acc) -> - {Ns,H1,Fl1} = flt_need_heap_2(I, H0, Fl0), - flt_need_heap_1(Is, H1, Fl1, [I|Ns]++Acc); -flt_need_heap_1([], H, Fl, Acc) -> - flt_alloc(H, Fl) ++ Acc. - -%% First come all instructions that build. We pass through, while we -%% add to the need for heap words and floats on the heap. -flt_need_heap_2({set,[_],[{fr,_}],fmove}, H, Fl) -> - {[],H,Fl+1}; -flt_need_heap_2({set,_,_,put_list}, H, Fl) -> - {[],H+2,Fl}; -flt_need_heap_2({set,_,_,{put_tuple,_}}, H, Fl) -> - {[],H+1,Fl}; -flt_need_heap_2({set,_,_,put}, H, Fl) -> - {[],H+1,Fl}; -%% The following instructions cause the insertion of an allocation -%% instruction if needed. -flt_need_heap_2({set,_,_,{alloc,_,_}}, H, Fl) -> - {flt_alloc(H, Fl),0,0}; -flt_need_heap_2({set,_,_,{set_tuple_element,_}}, H, Fl) -> - {flt_alloc(H, Fl),0,0}; -flt_need_heap_2({'%anno',_}, H, Fl) -> - {flt_alloc(H, Fl),0,0}; -%% All other instructions are "neutral". We just pass them. -flt_need_heap_2(_, H, Fl) -> - {[],H,Fl}. - -flt_alloc(0, 0) -> - []; -flt_alloc(H, 0) -> - [{set,[],[],{alloc,regs,{nozero,nostack,H,[]}}}]; -flt_alloc(H, F) -> - [{set,[],[],{alloc,regs,{nozero,nostack, - build_alloc(H, F),[]}}}]. - -build_alloc(Words, 0) -> Words; -build_alloc(Words, Floats) -> {alloc,[{words,Words},{floats,Floats}]}. - - -%% flt_liveness([Instruction]) -> [Instruction] -%% (Re)calculate the number of live registers for each heap allocation -%% function. We base liveness of the number of register map at the -%% beginning of the instruction sequence. -%% -%% A 'not_possible' term will be thrown if the set of live registers -%% is not continous at an allocation function (e.g. if {x,0} and {x,2} -%% are live, but not {x,1}). - -flt_liveness([{'%anno',{used,Regs}}=LiveInstr|Is]) -> - flt_liveness_1(Is, Regs, [LiveInstr]). - -flt_liveness_1([{set,Ds,Ss,{alloc,Live0,Alloc}}|Is], Regs0, Acc) -> - Live = min(Live0, live_regs(Regs0)), - I = {set,Ds,Ss,{alloc,Live,Alloc}}, - Regs1 = init_regs(Live), - Regs = x_live(Ds, Regs1), - flt_liveness_1(Is, Regs, [I|Acc]); -flt_liveness_1([{set,Ds,_,_}=I|Is], Regs0, Acc) -> - Regs = x_live(Ds, Regs0), - flt_liveness_1(Is, Regs, [I|Acc]); -flt_liveness_1([{'%anno',_}], _Regs, Acc) -> - reverse(Acc). - -init_regs(Live) -> - (1 bsl Live) - 1. - -live_regs(Regs) -> - live_regs_1(Regs, 0). - -live_regs_1(0, N) -> N; -live_regs_1(R, N) -> - case R band 1 of - 0 -> throw(not_possible); - 1 -> live_regs_1(R bsr 1, N+1) - end. - -x_live([{x,N}|Rs], Regs) -> x_live(Rs, Regs bor (1 bsl N)); -x_live([_|Rs], Regs) -> x_live(Rs, Regs); -x_live([], Regs) -> Regs. - -%% update(Instruction, TypeDb) -> NewTypeDb -%% Update the type database to account for executing an instruction. -%% -%% First the cases for instructions inside basic blocks. -update({'%anno',_}, Ts) -> - Ts; -update({set,[D],[S],move}, Ts) -> - tdb_copy(S, D, Ts); -update({set,[D],[Index,Reg],{bif,element,_}}, Ts0) -> - MinSize = case Index of - {integer,I} -> I; - _ -> 0 - end, - Ts = tdb_meet(Reg, {tuple,min_size,MinSize,[]}, Ts0), - tdb_store(D, any, Ts); -update({set,[D],[_Key,Map],{bif,map_get,_}}, Ts0) -> - Ts = tdb_meet(Map, map, Ts0), - tdb_store(D, any, Ts); -update({set,[D],Args,{bif,N,_}}, Ts) -> - Ar = length(Args), - BoolOp = erl_internal:new_type_test(N, Ar) orelse - erl_internal:comp_op(N, Ar) orelse - erl_internal:bool_op(N, Ar), - Type = case BoolOp of - true -> boolean; - false -> unary_op_type(N) - end, - tdb_store(D, Type, Ts); -update({set,[D],[S],{get_tuple_element,0}}, Ts0) -> - if - D =:= S -> - tdb_store(D, any, Ts0); - true -> - Ts = tdb_store(D, {tuple_element,S,0}, Ts0), - tdb_store(S, {tuple,min_size,1,[]}, Ts) - end; -update({set,[D],[S],{alloc,_,{gc_bif,float,{f,0}}}}, Ts0) -> - %% Make sure we reject non-numeric literal argument. - case possibly_numeric(S) of - true -> tdb_store(D, float, Ts0); - false -> Ts0 - end; -update({set,[D],[S1,S2],{alloc,_,{gc_bif,'band',{f,0}}}}, Ts) -> - Type = band_type(S1, S2, Ts), - tdb_store(D, Type, Ts); -update({set,[D],[S1,S2],{alloc,_,{gc_bif,'/',{f,0}}}}, Ts) -> - %% Make sure we reject non-numeric literals. - case possibly_numeric(S1) andalso possibly_numeric(S2) of - true -> tdb_store(D, float, Ts); - false -> Ts - end; -update({set,[D],[S1,S2],{alloc,_,{gc_bif,Op,{f,0}}}}, Ts0) -> - case op_type(Op) of - integer -> - tdb_store(D, integer, Ts0); - {float,_} -> - case {tdb_find(S1, Ts0),tdb_find(S2, Ts0)} of - {float,_} -> tdb_store(D, float, Ts0); - {_,float} -> tdb_store(D, float, Ts0); - {_,_} -> tdb_store(D, any, Ts0) - end; - Type -> - tdb_store(D, Type, Ts0) - end; -update({set,[D],[_],{alloc,_,{gc_bif,Op,{f,0}}}}, Ts) -> - tdb_store(D, unary_op_type(Op), Ts); -update({set,[],_Src,_Op}, Ts) -> - Ts; -update({set,[D],_Src,_Op}, Ts) -> - tdb_store(D, any, Ts); -update({kill,D}, Ts) -> - tdb_store(D, any, Ts); - -%% Instructions outside of blocks. -update({test,test_arity,_Fail,[Src,Arity]}, Ts) -> - tdb_meet(Src, {tuple,exact_size,Arity,[]}, Ts); -update({get_map_elements,_,Src,{list,Elems0}}, Ts0) -> - Ts1 = tdb_meet(Src, map, Ts0), - {_Ss,Ds} = beam_utils:split_even(Elems0), - foldl(fun(Dst, A) -> tdb_store(Dst, any, A) end, Ts1, Ds); -update({test,is_eq_exact,_,[Reg,{atom,_}=Atom]}, Ts0) -> - Ts = case tdb_find_source_tuple(Reg, Ts0) of - {source_tuple,TupleReg} -> - tdb_meet(TupleReg, {tuple,min_size,1,[Atom]}, Ts0); - none -> - Ts0 - end, - tdb_meet(Reg, Atom, Ts); -update({test,is_record,_Fail,[Src,Tag,{integer,Arity}]}, Ts) -> - tdb_meet(Src, {tuple,exact_size,Arity,[Tag]}, Ts); - -%% Binaries and binary matching. - -update({test,bs_get_integer2,_,_,Args,Dst}, Ts) -> - tdb_store(Dst, get_bs_integer_type(Args), Ts); -update({test,bs_get_utf8,_,_,_,Dst}, Ts) -> - tdb_store(Dst, ?UNICODE_INT, Ts); -update({test,bs_get_utf16,_,_,_,Dst}, Ts) -> - tdb_store(Dst, ?UNICODE_INT, Ts); -update({test,bs_get_utf32,_,_,_,Dst}, Ts) -> - tdb_store(Dst, ?UNICODE_INT, Ts); -update({bs_init,_,{bs_init2,_,_},_,_,Dst}, Ts) -> - tdb_store(Dst, {binary,8}, Ts); -update({bs_init,_,_,_,_,Dst}, Ts) -> - tdb_store(Dst, {binary,1}, Ts); -update({bs_put,_,_,_}, Ts) -> - Ts; -update({bs_save2,_,_}, Ts) -> - Ts; -update({bs_restore2,_,_}, Ts) -> - Ts; -update({bs_context_to_binary,Dst}, Ts) -> - tdb_store(Dst, any, Ts); -update({test,bs_start_match2,_,_,[Src,_],Dst}, Ts0) -> - Ts = tdb_meet(Src, {binary,1}, Ts0), - tdb_copy(Src, Dst, Ts); -update({test,bs_get_binary2,_,_,[_,_,Unit,_],Dst}, Ts) -> - true = is_integer(Unit), %Assertion. - tdb_store(Dst, {binary,Unit}, Ts); -update({test,bs_get_float2,_,_,_,Dst}, Ts) -> - tdb_store(Dst, float, Ts); -update({test,bs_test_unit,_,[Src,Unit]}, Ts) -> - tdb_meet(Src, {binary,Unit}, Ts); - -%% Other test instructions -update({test,Test,_Fail,[Src]}, Ts) -> - Type = case Test of - is_binary -> {binary,8}; - is_bitstr -> {binary,1}; - is_boolean -> boolean; - is_float -> float; - is_integer -> integer; - is_map -> map; - is_nonempty_list -> nonempty_list; - _ -> any - end, - tdb_meet(Src, Type, Ts); -update({test,_Test,_Fail,_Other}, Ts) -> - Ts; - -%% Calls - -update({call_ext,Ar,{extfunc,math,Math,Ar}}, Ts) -> - case is_math_bif(Math, Ar) of - true -> tdb_store({x,0}, float, Ts); - false -> tdb_kill_xregs(Ts) - end; -update({call_ext,3,{extfunc,erlang,setelement,3}}, Ts0) -> - Ts = tdb_kill_xregs(Ts0), - case tdb_find({x,1}, Ts0) of - {tuple,SzKind,Sz,_}=T0 -> - T = case tdb_find({x,0}, Ts0) of - {integer,{I,I}} when I > 1 -> - %% First element is not changed. The result - %% will have the same type. - T0; - _ -> - %% Position is 1 or unknown. May change the - %% first element of the tuple. - {tuple,SzKind,Sz,[]} - end, - tdb_store({x,0}, T, Ts); - _ -> - Ts - end; -update({call,_Arity,_Func}, Ts) -> tdb_kill_xregs(Ts); -update({call_ext,_Arity,_Func}, Ts) -> tdb_kill_xregs(Ts); -update({make_fun2,_,_,_,_}, Ts) -> tdb_kill_xregs(Ts); -update({call_fun, _}, Ts) -> tdb_kill_xregs(Ts); -update({apply, _}, Ts) -> tdb_kill_xregs(Ts); - -update({line,_}, Ts) -> Ts; -update({'%',_}, Ts) -> Ts; - -%% The instruction is unknown. Kill all information. -update(_I, _Ts) -> tdb_new(). - -band_type({integer,Int}, Other, Ts) -> - band_type_1(Int, Other, Ts); -band_type(Other, {integer,Int}, Ts) -> - band_type_1(Int, Other, Ts); -band_type(_, _, _) -> integer. - -band_type_1(Int, OtherSrc, Ts) -> - Type = band_type_2(Int, 0), - OtherType = tdb_find(OtherSrc, Ts), - meet(Type, OtherType). - -band_type_2(N, Bits) when Bits < 64 -> - case 1 bsl Bits of - P when P =:= N + 1 -> - {integer,{0,N}}; - P when P > N + 1 -> - integer; - _ -> - band_type_2(N, Bits+1) - end; -band_type_2(_, _) -> - %% Negative or large positive number. Give up. - integer. - -get_bs_integer_type([_,{integer,N},U,{field_flags,Fl}]) - when N*U < 64 -> - NumBits = N*U, - case member(unsigned, Fl) of - true -> - {integer,{0,(1 bsl NumBits)-1}}; - false -> - %% Signed integer. Don't bother. - integer - end; -get_bs_integer_type(_) -> - %% Avoid creating ranges with a huge upper limit. - integer. - -is_math_bif(cos, 1) -> true; -is_math_bif(cosh, 1) -> true; -is_math_bif(sin, 1) -> true; -is_math_bif(sinh, 1) -> true; -is_math_bif(tan, 1) -> true; -is_math_bif(tanh, 1) -> true; -is_math_bif(acos, 1) -> true; -is_math_bif(acosh, 1) -> true; -is_math_bif(asin, 1) -> true; -is_math_bif(asinh, 1) -> true; -is_math_bif(atan, 1) -> true; -is_math_bif(atanh, 1) -> true; -is_math_bif(erf, 1) -> true; -is_math_bif(erfc, 1) -> true; -is_math_bif(exp, 1) -> true; -is_math_bif(log, 1) -> true; -is_math_bif(log2, 1) -> true; -is_math_bif(log10, 1) -> true; -is_math_bif(sqrt, 1) -> true; -is_math_bif(atan2, 2) -> true; -is_math_bif(pow, 2) -> true; -is_math_bif(ceil, 1) -> true; -is_math_bif(floor, 1) -> true; -is_math_bif(fmod, 2) -> true; -is_math_bif(pi, 0) -> true; -is_math_bif(_, _) -> false. - -%% Reject non-numeric literals. -possibly_numeric({x,_}) -> true; -possibly_numeric({y,_}) -> true; -possibly_numeric({integer,_}) -> true; -possibly_numeric({float,_}) -> true; -possibly_numeric(_) -> false. - -max_tuple_size(Reg, Ts) -> - case tdb_find(Reg, Ts) of - {tuple,_,Sz,_} -> Sz; - _Other -> 0 - end. - -float_op('/', A, B, _) -> - case possibly_numeric(A) andalso possibly_numeric(B) of - true -> {yes,fdiv}; - false -> no - end; -float_op(Op, {float,_}, B, _) -> - case possibly_numeric(B) of - true -> arith_op(Op); - false -> no - end; -float_op(Op, A, {float,_}, _) -> - case possibly_numeric(A) of - true -> arith_op(Op); - false -> no - end; -float_op(Op, A, B, Ts) -> - case {tdb_find(A, Ts),tdb_find(B, Ts)} of - {float,_} -> arith_op(Op); - {_,float} -> arith_op(Op); - {_,_} -> no - end. - -find_dest(V, Rs0) -> - case find_reg(V, Rs0) of - {ok,FR} -> - {FR,mark(V, Rs0, dirty)}; - error -> - Rs = put_reg(V, Rs0, dirty), - {ok,FR} = find_reg(V, Rs), - {FR,Rs} - end. - -load_reg({float,_}=F, _, Rs0, Is0) -> - Rs = put_reg(F, Rs0, clean), - {ok,FR} = find_reg(F, Rs), - Is = [{set,[FR],[F],fmove}|Is0], - {Rs,Is}; -load_reg(V, Ts, Rs0, Is0) -> - case find_reg(V, Rs0) of - {ok,_FR} -> {Rs0,Is0}; - error -> - Rs = put_reg(V, Rs0, clean), - {ok,FR} = find_reg(V, Rs), - Op = case tdb_find(V, Ts) of - float -> fmove; - _ -> fconv - end, - Is = [{set,[FR],[V],Op}|Is0], - {Rs,Is} - end. - -arith_op(Op) -> - case op_type(Op) of - {float,Instr} -> {yes,Instr}; - _ -> no - end. - -op_type('+') -> {float,fadd}; -op_type('-') -> {float,fsub}; -op_type('*') -> {float,fmul}; -%% '/' and 'band' are specially handled. -op_type('bor') -> integer; -op_type('bxor') -> integer; -op_type('bsl') -> integer; -op_type('bsr') -> integer; -op_type('div') -> integer; -op_type(_) -> any. - -unary_op_type(bit_size) -> integer; -unary_op_type(byte_size) -> integer; -unary_op_type(length) -> integer; -unary_op_type(map_size) -> integer; -unary_op_type(size) -> integer; -unary_op_type(tuple_size) -> integer; -unary_op_type(_) -> any. - -flush(Rs, [{set,[_],[_,_,_],{bif,is_record,_}}|_]=Is0, Acc0) -> - Acc = flush_all(Rs, Is0, Acc0), - {[],Acc}; -flush(Rs, [{set,[_],[],{put_tuple,_}}|_]=Is0, Acc0) -> - Acc = flush_all(Rs, Is0, Acc0), - {[],Acc}; -flush(Rs0, [{set,Ds,Ss,_Op}|_], Acc0) -> - Save = cerl_sets:from_list(Ss), - Acc = save_regs(Rs0, Save, Acc0), - Rs1 = foldl(fun(S, A) -> mark(S, A, clean) end, Rs0, Ss), - Kill = cerl_sets:from_list(Ds), - Rs = kill_regs(Rs1, Kill), - {Rs,Acc}; -flush(Rs0, Is, Acc0) -> - Acc = flush_all(Rs0, Is, Acc0), - {[],Acc}. - -flush_all([{_,{float,_},_}|Rs], Is, Acc) -> - flush_all(Rs, Is, Acc); -flush_all([{I,V,dirty}|Rs], Is, Acc0) -> - Acc = checkerror(Acc0), - case beam_utils:is_killed_block(V, Is) of - true -> flush_all(Rs, Is, Acc); - false -> flush_all(Rs, Is, [{set,[V],[{fr,I}],fmove}|Acc]) - end; -flush_all([{_,_,clean}|Rs], Is, Acc) -> flush_all(Rs, Is, Acc); -flush_all([free|Rs], Is, Acc) -> flush_all(Rs, Is, Acc); -flush_all([], _, Acc) -> Acc. - -save_regs(Rs, Save, Acc) -> - foldl(fun(R, A) -> save_reg(R, Save, A) end, Acc, Rs). - -save_reg({I,V,dirty}, Save, Acc) -> - case cerl_sets:is_element(V, Save) of - true -> [{set,[V],[{fr,I}],fmove}|checkerror(Acc)]; - false -> Acc - end; -save_reg(_, _, Acc) -> Acc. - -kill_regs(Rs, Kill) -> - [kill_reg(R, Kill) || R <- Rs]. - -kill_reg({_,V,_}=R, Kill) -> - case cerl_sets:is_element(V, Kill) of - true -> free; - false -> R - end; -kill_reg(R, _) -> R. - -mark(V, [{I,V,_}|Rs], Mark) -> [{I,V,Mark}|Rs]; -mark(V, [R|Rs], Mark) -> [R|mark(V, Rs, Mark)]; -mark(_, [], _) -> []. - -fetch_reg(V, [{I,V,_}|_]) -> {fr,I}; -fetch_reg(V, [_|SRs]) -> fetch_reg(V, SRs). - -find_reg(V, [{I,V,_}|_]) -> {ok,{fr,I}}; -find_reg(V, [_|SRs]) -> find_reg(V, SRs); -find_reg(_, []) -> error. - -put_reg(V, Rs, Dirty) -> put_reg_1(V, Rs, Dirty, 0). - -put_reg_1(V, [free|Rs], Dirty, I) -> [{I,V,Dirty}|Rs]; -put_reg_1(V, [R|Rs], Dirty, I) -> [R|put_reg_1(V, Rs, Dirty, I+1)]; -put_reg_1(V, [], Dirty, I) -> [{I,V,Dirty}]. - -checkerror(Is) -> - checkerror_1(Is, Is). - -checkerror_1([{set,[],[],fcheckerror}|_], OrigIs) -> OrigIs; -checkerror_1([{set,_,_,{bif,fadd,_}}|_], OrigIs) -> checkerror_2(OrigIs); -checkerror_1([{set,_,_,{bif,fsub,_}}|_], OrigIs) -> checkerror_2(OrigIs); -checkerror_1([{set,_,_,{bif,fmul,_}}|_], OrigIs) -> checkerror_2(OrigIs); -checkerror_1([{set,_,_,{bif,fdiv,_}}|_], OrigIs) -> checkerror_2(OrigIs); -checkerror_1([{set,_,_,{bif,fnegate,_}}|_], OrigIs) -> checkerror_2(OrigIs); -checkerror_1([_|Is], OrigIs) -> checkerror_1(Is, OrigIs); -checkerror_1([], OrigIs) -> OrigIs. - -checkerror_2(OrigIs) -> [{set,[],[],fcheckerror}|OrigIs]. - - -%%% Routines for maintaining a type database. The type database -%%% associates type information with registers. -%%% -%%% See the comment for verified_type/1 at the end of module for -%%% a description of the possible types. - -%% tdb_new() -> EmptyDataBase -%% Creates a new, empty type database. - -tdb_new() -> []. - -%% tdb_find(Register, Db) -> Type -%% Returns type information or the atom error if there is no type -%% information available for Register. -%% -%% See the comment for verified_type/1 at the end of module for -%% a description of the possible types. - -tdb_find(Reg, Ts) -> - case tdb_find_raw(Reg, Ts) of - {tuple_element,_,_} -> any; - Type -> Type - end. - -%% tdb_find_source_tuple(Register, Ts) -> {source_tuple,Register} | 'none'. -%% Find the tuple whose first element was fetched to the register Register. - -tdb_find_source_tuple(Reg, Ts) -> - case tdb_find_raw(Reg, Ts) of - {tuple_element,Src,0} -> - {source_tuple,Src}; - _ -> - none - end. - -%% tdb_copy(Source, Dest, Db) -> Db' -%% Update the type information for Dest to have the same type -%% as the Source. - -tdb_copy({Tag,_}=S, D, Ts) when Tag =:= x; Tag =:= y -> - case tdb_find_raw(S, Ts) of - any -> orddict:erase(D, Ts); - Type -> orddict:store(D, Type, Ts) - end; -tdb_copy(Literal, D, Ts) -> - Type = case Literal of - {atom,_} -> Literal; - {float,_} -> float; - {integer,Int} -> {integer,{Int,Int}}; - {literal,[_|_]} -> nonempty_list; - {literal,#{}} -> map; - {literal,Tuple} when tuple_size(Tuple) >= 1 -> - Lit = tag_literal(element(1, Tuple)), - {tuple,exact_size,tuple_size(Tuple),[Lit]}; - _ -> any - end, - tdb_store(D, verified_type(Type), Ts). - -%% tdb_store(Register, Type, Ts0) -> Ts. -%% Store a new type for register Register. Return the update type -%% database. Use this function when a new value is assigned to -%% a register. -%% -%% See the comment for verified_type/1 at the end of module for -%% a description of the possible types. - -tdb_store(Reg, any, Ts) -> - erase(Reg, Ts); -tdb_store(Reg, Type, Ts) -> - store(Reg, verified_type(Type), Ts). - -store(Key, New, [{K,_}|_]=Dict) when Key < K -> - [{Key,New}|Dict]; -store(Key, New, [{K,Val}=E|Dict]) when Key > K -> - case Val of - {tuple_element,Key,_} -> store(Key, New, Dict); - _ -> [E|store(Key, New, Dict)] - end; -store(Key, New, [{_K,Old}|Dict]) -> %Key == K - case Old of - {tuple,_,_,_} -> - [{Key,New}|erase_tuple_element(Key, Dict)]; - _ -> - [{Key,New}|Dict] - end; -store(Key, New, []) -> [{Key,New}]. - -erase(Key, [{K,_}=E|Dict]) when Key < K -> - [E|Dict]; -erase(Key, [{K,Val}=E|Dict]) when Key > K -> - case Val of - {tuple_element,Key,_} -> erase(Key, Dict); - _ -> [E|erase(Key, Dict)] - end; -erase(Key, [{_K,Val}|Dict]) -> %Key == K - case Val of - {tuple,_,_,_} -> erase_tuple_element(Key, Dict); - _ -> Dict - end; -erase(_, []) -> []. - -erase_tuple_element(Key, [{_,{tuple_element,Key,_}}|Dict]) -> - erase_tuple_element(Key, Dict); -erase_tuple_element(Key, [E|Dict]) -> - [E|erase_tuple_element(Key, Dict)]; -erase_tuple_element(_Key, []) -> []. - -%% tdb_meet(Register, Type, Ts0) -> Ts. -%% Update information of a register that is used as the source for an -%% instruction. The type Type will be combined using the meet operation -%% with the previous type information for the register, resulting in -%% narrower (more specific) type. -%% -%% For example, if the previous type is {tuple,min_size,2,[]} and the -%% the new type is {tuple,exact_size,5,[]}, the meet of the types will -%% be {tuple,exact_size,5,[]}. -%% -%% See the comment for verified_type/1 at the end of module for -%% a description of the possible types. - -tdb_meet(Reg, NewType, Ts) -> - Update = fun(Type0) -> meet(Type0, NewType) end, - orddict:update(Reg, Update, NewType, Ts). - -%%% -%%% Here follows internal helper functions for accessing and -%%% updating the type database. -%%% - -tdb_find_raw({x,_}=K, Ts) -> tdb_find_raw_1(K, Ts); -tdb_find_raw({y,_}=K, Ts) -> tdb_find_raw_1(K, Ts); -tdb_find_raw(_, _) -> any. - -tdb_find_raw_1(K, Ts) -> - case orddict:find(K, Ts) of - {ok,Val} -> Val; - error -> any - end. - -tag_literal(A) when is_atom(A) -> {atom,A}; -tag_literal(F) when is_float(F) -> {float,F}; -tag_literal(I) when is_integer(I) -> {integer,I}; -tag_literal([]) -> nil; -tag_literal(Lit) -> {literal,Lit}. - -%% tdb_kill_xregs(Db) -> NewDb -%% Kill all information about x registers. Also kill all tuple_element -%% dependencies from y registers to x registers. - -tdb_kill_xregs([{{x,_},_Type}|Db]) -> tdb_kill_xregs(Db); -tdb_kill_xregs([{{y,_},{tuple_element,{x,_},_}}|Db]) -> tdb_kill_xregs(Db); -tdb_kill_xregs([Any|Db]) -> [Any|tdb_kill_xregs(Db)]; -tdb_kill_xregs([]) -> []. - -%% meet(Type1, Type2) -> Type -%% Returns the "meet" of Type1 and Type2. The meet is a narrower -%% type than Type1 and Type2. For example: -%% -%% meet(integer, {integer,{0,3}}) -> {integer,{0,3}} -%% -%% The meet for two different types result in 'none', which is -%% the bottom element for our type lattice: -%% -%% meet(integer, map) -> none - -meet(T, T) -> - T; -meet({integer,_}=T, integer) -> - T; -meet(integer, {integer,_}=T) -> - T; -meet({integer,{Min1,Max1}}, {integer,{Min2,Max2}}) -> - {integer,{max(Min1, Min2),min(Max1, Max2)}}; -meet({tuple,min_size,Sz1,Same}, {tuple,min_size,Sz2,Same}=Max) when Sz1 < Sz2 -> - Max; -meet({tuple,min_size,Sz1,Same}=Max, {tuple,min_size,Sz2,Same}) when Sz1 > Sz2 -> - Max; -meet({tuple,exact_size,_,Same}=Exact, {tuple,_,_,Same}) -> - Exact; -meet({tuple,_,_,Same},{tuple,exact_size,_,Same}=Exact) -> - Exact; -meet({tuple,SzKind1,Sz1,[]}, {tuple,_SzKind2,_Sz2,First}=Tuple2) -> - meet({tuple,SzKind1,Sz1,First}, Tuple2); -meet({tuple,_SzKind1,_Sz1,First}=Tuple1, {tuple,SzKind2,Sz2,_}) -> - meet(Tuple1, {tuple,SzKind2,Sz2,First}); -meet({binary,U1}, {binary,U2}) -> - {binary,max(U1, U2)}; -meet(T1, T2) -> - case is_any(T1) of - true -> - verified_type(T2); - false -> - case is_any(T2) of - true -> - verified_type(T1); - false -> - none %The bottom element. - end - end. - -is_any(any) -> true; -is_any({tuple_element,_,_}) -> true; -is_any(_) -> false. - -%% verified_type(Type) -> Type -%% Returns the passed in type if it is one of the defined types. -%% Crashes if there is anything wrong with the type. -%% -%% Here are all possible types: -%% -%% any Any Erlang term (top element for the type lattice). -%% -%% {atom,Atom} The specific atom Atom. -%% {binary,Unit} Binary/bitstring aligned to unit Unit. -%% boolean 'true' | 'false' -%% float Floating point number. -%% integer Integer. -%% {integer,{Min,Max}} Integer in the inclusive range Min through Max. -%% map Map. -%% nonempty_list Nonempty list. -%% {tuple,_,_,_} Tuple (see below). -%% -%% none No type (bottom element for the type lattice). -%% -%% {tuple,min_size,Size,First} means that the corresponding register -%% contains a tuple with *at least* Size elements (conversely, -%% {tuple,exact_size,Size,First} means that it contains a tuple with -%% *exactly* Size elements). An tuple with unknown size is -%% represented as {tuple,min_size,0,[]}. First is either [] (meaning -%% that the tuple's first element is unknown) or [FirstElement] (the -%% contents of the first element). -%% -%% There is also a pseudo-type called {tuple_element,_,_}: -%% -%% {tuple_element,SrcTuple,ElementNumber} -%% -%% that does not provide any information about the type of the -%% register itself, but provides a link back to the source tuple that -%% the register got its value from. -%% -%% Note that {tuple_element,_,_} will *never* be returned by tdb_find/2. -%% Use tdb_find_source_tuple/2 to locate the source tuple for a register. - -verified_type(any=T) -> T; -verified_type({atom,_}=T) -> T; -verified_type({binary,U}=T) when is_integer(U) -> T; -verified_type(boolean=T) -> T; -verified_type(integer=T) -> T; -verified_type({integer,{Min,Max}}=T) - when is_integer(Min), is_integer(Max) -> T; -verified_type(map=T) -> T; -verified_type(nonempty_list=T) -> T; -verified_type({tuple,_,Sz,[]}=T) when is_integer(Sz) -> T; -verified_type({tuple,_,Sz,[_]}=T) when is_integer(Sz) -> T; -verified_type({tuple_element,_,_}=T) -> T; -verified_type(float=T) -> T; -verified_type(none=T) -> T. diff --git a/lib/compiler/src/beam_utils.erl b/lib/compiler/src/beam_utils.erl index 6b2ab5a2a4..6e6574c0b3 100644 --- a/lib/compiler/src/beam_utils.erl +++ b/lib/compiler/src/beam_utils.erl @@ -18,29 +18,16 @@ %% %CopyrightEnd% %% %% Purpose : Common utilities used by several optimization passes. -%% +%% -module(beam_utils). --export([is_killed_block/2,is_killed/3,is_killed_at/3, - is_not_used/3,usage/3, - empty_label_index/0,index_label/3,index_labels/1,replace_labels/4, - code_at/2,bif_to_test/3,is_pure_test/1, - live_opt/1,delete_annos/1,combine_heap_needs/2, - anno_defs/1, - split_even/1 - ]). +-export([replace_labels/4,is_pure_test/1,split_even/1]). -export_type([code_index/0,module_code/0,instruction/0]). --import(lists, [flatmap/2,map/2,member/2,sort/1,reverse/1,splitwith/2]). - --define(is_const(Val), (Val =:= nil orelse - element(1, Val) =:= integer orelse - element(1, Val) =:= float orelse - element(1, Val) =:= atom orelse - element(1, Val) =:= literal)). +-import(lists, [map/2,reverse/1]). -%% instruction() describes all instructions that are used during optimzation +%% instruction() describes all instructions that are used during optimization %% (from beam_a to beam_z). -type instruction() :: atom() | tuple(). @@ -56,137 +43,6 @@ -type fail() :: beam_asm:fail() | 'fail'. -type test() :: {'test',atom(),fail(),[beam_asm:src()]} | {'test',atom(),fail(),integer(),list(),beam_asm:reg()}. --type result_cache() :: gb_trees:tree(beam_asm:label(), 'killed' | 'used'). - --record(live, - {lbl :: code_index(), %Label to code index. - res :: result_cache()}). %Result cache for each label. - -%% usage(Register, [Instruction], State) -> killed|not_used|used. -%% Determine the usage of Register in the instruction sequence. -%% The return value is one of: -%% -%% killed - The register is not used in any way. -%% not_used - The register is referenced only by an allocating instruction -%% (the actual value does not matter). -%% used - The register is used (its value do matter). - --spec usage(beam_asm:reg(), [instruction()], code_index()) -> - 'killed' | 'not_used' | 'used'. - -usage(R, Is, D) -> - St = #live{lbl=D,res=gb_trees:empty()}, - {Usage,_} = check_liveness(R, Is, St), - Usage. - - -%% is_killed_block(Register, [Instruction]) -> true|false -%% Determine whether a register is killed by the instruction sequence inside -%% a block. -%% -%% If true is returned, it means that the register will not be -%% referenced in ANY way (not even indirectly by an allocate instruction); -%% i.e. it is OK to enter the instruction sequence with Register -%% containing garbage. - --spec is_killed_block(beam_asm:reg(), [instruction()]) -> boolean(). - -is_killed_block({x,X}, [{set,_,_,{alloc,Live,_}}|_]) -> - X >= Live; -is_killed_block(R, [{set,Ds,Ss,_Op}|Is]) -> - not member(R, Ss) andalso (member(R, Ds) orelse is_killed_block(R, Is)); -is_killed_block(R, [{'%anno',{used,Regs}}|Is]) -> - case R of - {x,X} when (Regs bsr X) band 1 =:= 0 -> true; - _ -> is_killed_block(R, Is) - end; -is_killed_block(_, []) -> false. - -%% is_killed(Register, [Instruction], State) -> true|false -%% Determine whether a register is killed by the instruction sequence. -%% If true is returned, it means that the register will not be -%% referenced in ANY way (not even indirectly by an allocate instruction); -%% i.e. it is OK to enter the instruction sequence with Register -%% containing garbage. -%% -%% The state (constructed by index_instructions/1) is used to allow us -%% to determine the kill state across branches. - --spec is_killed(beam_asm:reg(), [instruction()], code_index()) -> boolean(). - -is_killed(R, Is, D) -> - St = #live{lbl=D,res=gb_trees:empty()}, - case check_liveness(R, Is, St) of - {killed,_} -> true; - {exit_not_used,_} -> false; - {_,_} -> false - end. - -%% is_killed_at(Reg, Lbl, State) -> true|false -%% Determine whether Reg is killed at label Lbl. - --spec is_killed_at(beam_asm:reg(), beam_asm:label(), code_index()) -> boolean(). - -is_killed_at(R, Lbl, D) when is_integer(Lbl) -> - St0 = #live{lbl=D,res=gb_trees:empty()}, - case check_liveness_at(R, Lbl, St0) of - {killed,_} -> true; - {exit_not_used,_} -> false; - {_,_} -> false - end. - -%% is_not_used(Register, [Instruction], State) -> true|false -%% Determine whether a register is never used in the instruction sequence -%% (it could still be referenced by an allocate instruction, meaning that -%% it MUST be initialized, but that its value does not matter). -%% The state is used to allow us to determine the usage state -%% across branches. - --spec is_not_used(beam_asm:reg(), [instruction()], code_index()) -> boolean(). - -is_not_used(R, Is, D) -> - St = #live{lbl=D,res=gb_trees:empty()}, - case check_liveness(R, Is, St) of - {used,_} -> false; - {exit_not_used,_} -> true; - {_,_} -> true - end. - -%% index_labels(FunctionIs) -> State -%% Index the instruction sequence so that we can quickly -%% look up the instruction following a specific label. - --spec index_labels([instruction()]) -> code_index(). - -index_labels(Is) -> - index_labels_1(Is, []). - -%% empty_label_index() -> State -%% Create an empty label index. - --spec empty_label_index() -> code_index(). - -empty_label_index() -> - gb_trees:empty(). - -%% index_label(Label, [Instruction], State) -> State -%% Add an index for a label. - --spec index_label(beam_asm:label(), [instruction()], code_index()) -> - code_index(). - -index_label(Lbl, Is0, Acc) -> - Is = drop_labels(Is0), - gb_trees:enter(Lbl, Is, Acc). - - -%% code_at(Label, State) -> [I]. -%% Retrieve the code at the given label. - --spec code_at(beam_asm:label(), code_index()) -> [instruction()]. - -code_at(L, Ll) -> - gb_trees:get(L, Ll). %% replace_labels(FunctionIs, Tail, ReplaceDb, Fallback) -> FunctionIs. %% Replace all labels in instructions according to the ReplaceDb. @@ -200,49 +56,6 @@ code_at(L, Ll) -> replace_labels(Is, Acc, D, Fb) -> replace_labels_1(Is, Acc, D, Fb). -%% bif_to_test(Bif, [Op], Fail) -> {test,Test,Fail,[Op]} -%% Convert a BIF to a test. Fail if not possible. - --spec bif_to_test(atom(), list(), fail()) -> test(). - -bif_to_test(is_atom, [_]=Ops, Fail) -> {test,is_atom,Fail,Ops}; -bif_to_test(is_boolean, [_]=Ops, Fail) -> {test,is_boolean,Fail,Ops}; -bif_to_test(is_binary, [_]=Ops, Fail) -> {test,is_binary,Fail,Ops}; -bif_to_test(is_bitstring,[_]=Ops, Fail) -> {test,is_bitstr,Fail,Ops}; -bif_to_test(is_float, [_]=Ops, Fail) -> {test,is_float,Fail,Ops}; -bif_to_test(is_function, [_]=Ops, Fail) -> {test,is_function,Fail,Ops}; -bif_to_test(is_function, [_,_]=Ops, Fail) -> {test,is_function2,Fail,Ops}; -bif_to_test(is_integer, [_]=Ops, Fail) -> {test,is_integer,Fail,Ops}; -bif_to_test(is_list, [_]=Ops, Fail) -> {test,is_list,Fail,Ops}; -bif_to_test(is_map, [_]=Ops, Fail) -> {test,is_map,Fail,Ops}; -bif_to_test(is_number, [_]=Ops, Fail) -> {test,is_number,Fail,Ops}; -bif_to_test(is_pid, [_]=Ops, Fail) -> {test,is_pid,Fail,Ops}; -bif_to_test(is_port, [_]=Ops, Fail) -> {test,is_port,Fail,Ops}; -bif_to_test(is_reference, [_]=Ops, Fail) -> {test,is_reference,Fail,Ops}; -bif_to_test(is_tuple, [_]=Ops, Fail) -> {test,is_tuple,Fail,Ops}; -bif_to_test('=<', [A,B], Fail) -> {test,is_ge,Fail,[B,A]}; -bif_to_test('>', [A,B], Fail) -> {test,is_lt,Fail,[B,A]}; -bif_to_test('<', [_,_]=Ops, Fail) -> {test,is_lt,Fail,Ops}; -bif_to_test('>=', [_,_]=Ops, Fail) -> {test,is_ge,Fail,Ops}; -bif_to_test('==', [A,nil], Fail) -> {test,is_nil,Fail,[A]}; -bif_to_test('==', [nil,A], Fail) -> {test,is_nil,Fail,[A]}; -bif_to_test('==', [C,A], Fail) when ?is_const(C) -> - {test,is_eq,Fail,[A,C]}; -bif_to_test('==', [_,_]=Ops, Fail) -> {test,is_eq,Fail,Ops}; -bif_to_test('/=', [C,A], Fail) when ?is_const(C) -> - {test,is_ne,Fail,[A,C]}; -bif_to_test('/=', [_,_]=Ops, Fail) -> {test,is_ne,Fail,Ops}; -bif_to_test('=:=', [A,nil], Fail) -> {test,is_nil,Fail,[A]}; -bif_to_test('=:=', [nil,A], Fail) -> {test,is_nil,Fail,[A]}; -bif_to_test('=:=', [C,A], Fail) when ?is_const(C) -> - {test,is_eq_exact,Fail,[A,C]}; -bif_to_test('=:=', [_,_]=Ops, Fail) -> {test,is_eq_exact,Fail,Ops}; -bif_to_test('=/=', [C,A], Fail) when ?is_const(C) -> - {test,is_ne_exact,Fail,[A,C]}; -bif_to_test('=/=', [_,_]=Ops, Fail) -> {test,is_ne_exact,Fail,Ops}; -bif_to_test(is_record, [_,_,_]=Ops, Fail) -> {test,is_record,Fail,Ops}. - - %% is_pure_test({test,Op,Fail,Ops}) -> true|false. %% Return 'true' if the test instruction does not modify any %% registers and/or bit syntax matching state. @@ -256,82 +69,15 @@ is_pure_test({test,is_eq_exact,_,[_,_]}) -> true; is_pure_test({test,is_ne_exact,_,[_,_]}) -> true; is_pure_test({test,is_ge,_,[_,_]}) -> true; is_pure_test({test,is_lt,_,[_,_]}) -> true; -is_pure_test({test,is_nil,_,[_]}) -> true; is_pure_test({test,is_nonempty_list,_,[_]}) -> true; +is_pure_test({test,is_tagged_tuple,_,[_,_,_]}) -> true; is_pure_test({test,test_arity,_,[_,_]}) -> true; is_pure_test({test,has_map_fields,_,[_|_]}) -> true; is_pure_test({test,is_bitstr,_,[_]}) -> true; is_pure_test({test,is_function2,_,[_,_]}) -> true; -is_pure_test({test,Op,_,Ops}) -> +is_pure_test({test,Op,_,Ops}) -> erl_internal:new_type_test(Op, length(Ops)). - -%% live_opt([Instruction]) -> [Instruction]. -%% Go through the instruction sequence in reverse execution -%% order, keep track of liveness and remove 'move' instructions -%% whose destination is a register that will not be used. -%% Also insert {used,Regs} annotations at the beginning -%% and end of each block. - --spec live_opt([instruction()]) -> [instruction()]. - -live_opt(Is0) -> - {[{label,Fail}|_]=Bef,[Fi|Is]} = - splitwith(fun({func_info,_,_,_}) -> false; - (_) -> true - end, Is0), - {func_info,_,_,Live} = Fi, - D = gb_trees:insert(Fail, live_call(Live), gb_trees:empty()), - Bef ++ [Fi|live_opt(reverse(Is), 0, D, [])]. - - -%% delete_annos([Instruction]) -> [Instruction]. -%% Delete all annotations. - --spec delete_annos([instruction()]) -> [instruction()]. - -delete_annos([{block,Bl0}|Is]) -> - case delete_annos(Bl0) of - [] -> delete_annos(Is); - [_|_]=Bl -> [{block,Bl}|delete_annos(Is)] - end; -delete_annos([{'%anno',_}|Is]) -> - delete_annos(Is); -delete_annos([I|Is]) -> - [I|delete_annos(Is)]; -delete_annos([]) -> []. - -%% combine_heap_needs(HeapNeed1, HeapNeed2) -> HeapNeed -%% Combine the heap need for two allocation instructions. - --type heap_need_tag() :: 'floats' | 'words'. --type heap_need() :: non_neg_integer() | - {'alloc',[{heap_need_tag(),non_neg_integer()}]}. --spec combine_heap_needs(heap_need(), heap_need()) -> heap_need(). - -combine_heap_needs(H1, H2) when is_integer(H1), is_integer(H2) -> - H1 + H2; -combine_heap_needs(H1, H2) -> - {alloc,combine_alloc_lists([H1,H2])}. - - -%% anno_defs(Instructions) -> Instructions' -%% Add {def,RegisterBitmap} annotations to the beginning of -%% each block. Iff bit X is set in the the bitmap, it means -%% that {x,X} is defined when the block is entered. - --spec anno_defs([instruction()]) -> [instruction()]. - -anno_defs(Is0) -> - {Bef,[Fi|Is1]} = - splitwith(fun({func_info,_,_,_}) -> false; - (_) -> true - end, Is0), - {func_info,_,_,Arity} = Fi, - Regs = init_def_regs(Arity), - Is = defs(Is1, Regs, #{}), - Bef ++ [Fi|Is]. - %% split_even/1 %% [1,2,3,4,5,6] -> {[1,3,5],[2,4,6]} @@ -343,446 +89,6 @@ split_even(Rs) -> split_even(Rs, [], []). %%% Local functions. %%% - -%% check_liveness(Reg, [Instruction], #live{}) -> -%% {killed | not_used | used, #live{}} -%% Find out whether Reg is used or killed in instruction sequence. -%% -%% killed - Reg is assigned or killed by an allocation instruction. -%% not_used - the value of Reg is not used, but Reg must not be garbage -%% exit_not_used - the value of Reg is not used, but must not be garbage -%% because the stack will be scanned because an -%% exit BIF will raise an exception -%% used - Reg is used - -check_liveness({fr,_}, _, St) -> - %% Conservatively always consider the floating point register used. - {used,St}; -check_liveness(R, [{block,Blk}|Is], St0) -> - case check_liveness_block(R, Blk, St0) of - {transparent,St1} -> - check_liveness(R, Is, St1); - {alloc_used,St1} -> - %% Used by an allocating instruction, but value not referenced. - %% Must check the rest of the instructions. - not_used(check_liveness(R, Is, St1)); - {Other,_}=Res when is_atom(Other) -> - Res - end; -check_liveness(R, [{label,_}|Is], St) -> - check_liveness(R, Is, St); -check_liveness(R, [{test,_,{f,Fail},As}|Is], St0) -> - case member(R, As) of - true -> - {used,St0}; - false -> - case check_liveness_at(R, Fail, St0) of - {killed,St1} -> - check_liveness(R, Is, St1); - {exit_not_used,St1} -> - not_used(check_liveness(R, Is, St1)); - {not_used,St1} -> - not_used(check_liveness(R, Is, St1)); - {used,_}=Used -> - Used - end - end; -check_liveness(R, [{test,Op,Fail,Live,Ss,Dst}|Is], St) -> - %% Check this instruction as a block to get a less conservative - %% result if the caller is is_not_used/3. - Block = [{set,[Dst],Ss,{alloc,Live,{bif,Op,Fail}}}], - check_liveness(R, [{block,Block}|Is], St); -check_liveness(R, [{select,_,R,_,_}|_], St) -> - {used,St}; -check_liveness(R, [{select,_,_,Fail,Branches}|_], St) -> - check_liveness_everywhere(R, [Fail|Branches], St); -check_liveness(R, [{jump,{f,F}}|_], St) -> - check_liveness_at(R, F, St); -check_liveness(R, [{case_end,Used}|_], St) -> - check_liveness_exit(R, Used, St); -check_liveness(R, [{try_case_end,Used}|_], St) -> - check_liveness_exit(R, Used, St); -check_liveness(R, [{badmatch,Used}|_], St) -> - check_liveness_exit(R, Used, St); -check_liveness(R, [if_end|_], St) -> - check_liveness_exit(R, ignore, St); -check_liveness(R, [{func_info,_,_,Ar}|_], St) -> - case R of - {x,X} when X < Ar -> {used,St}; - _ -> {killed,St} - end; -check_liveness(R, [{kill,R}|_], St) -> - {killed,St}; -check_liveness(R, [{kill,_}|Is], St) -> - check_liveness(R, Is, St); -check_liveness(R, [{bs_init,_,_,none,Ss,Dst}|Is], St) -> - case member(R, Ss) of - true -> - {used,St}; - false -> - if - R =:= Dst -> {killed,St}; - true -> check_liveness(R, Is, St) - end - end; -check_liveness(R, [{bs_init,_,_,Live,Ss,Dst}|Is], St) -> - case R of - {x,X} -> - case member(R, Ss) of - true -> - {used,St}; - false -> - if - X < Live -> - not_used(check_liveness(R, Is, St)); - true -> - {killed,St} - end - end; - {y,_} -> - case member(R, Ss) of - true -> {used,St}; - false -> - %% If the exception is taken, the stack may - %% be scanned. Therefore the register is not - %% guaranteed to be killed. - if - R =:= Dst -> {not_used,St}; - true -> not_used(check_liveness(R, Is, St)) - end - end - end; -check_liveness(R, [{deallocate,_}|Is], St) -> - case R of - {y,_} -> {killed,St}; - _ -> check_liveness(R, Is, St) - end; -check_liveness({x,_}=R, [return|_], St) -> - case R of - {x,0} -> {used,St}; - {x,_} -> {killed,St} - end; -check_liveness(R, [{call,Live,_}|Is], St) -> - case R of - {x,X} when X < Live -> {used,St}; - {x,_} -> {killed,St}; - {y,_} -> not_used(check_liveness(R, Is, St)) - end; -check_liveness(R, [{call_ext,Live,_}=I|Is], St) -> - case R of - {x,X} when X < Live -> - {used,St}; - {x,_} -> - {killed,St}; - {y,_} -> - case beam_jump:is_exit_instruction(I) of - false -> - not_used(check_liveness(R, Is, St)); - true -> - %% We must make sure we don't check beyond this - %% instruction or we will fall through into random - %% unrelated code and get stuck in a loop. - {exit_not_used,St} - end - end; -check_liveness(R, [{call_fun,Live}|Is], St) -> - case R of - {x,X} when X =< Live -> {used,St}; - {x,_} -> {killed,St}; - {y,_} -> not_used(check_liveness(R, Is, St)) - end; -check_liveness(R, [{apply,Args}|Is], St) -> - case R of - {x,X} when X < Args+2 -> {used,St}; - {x,_} -> {killed,St}; - {y,_} -> not_used(check_liveness(R, Is, St)) - end; -check_liveness(R, [{bif,Op,Fail,Ss,D}|Is], St) -> - Set = {set,[D],Ss,{bif,Op,Fail}}, - check_liveness(R, [{block,[Set]}|Is], St); -check_liveness(R, [{gc_bif,Op,{f,Fail},Live,Ss,D}|Is], St) -> - Set = {set,[D],Ss,{alloc,Live,{gc_bif,Op,Fail}}}, - check_liveness(R, [{block,[Set]}|Is], St); -check_liveness(R, [{bs_put,{f,0},_,Ss}|Is], St) -> - case member(R, Ss) of - true -> {used,St}; - false -> check_liveness(R, Is, St) - end; -check_liveness(R, [{bs_restore2,S,_}|Is], St) -> - case R of - S -> {used,St}; - _ -> check_liveness(R, Is, St) - end; -check_liveness(R, [{bs_save2,S,_}|Is], St) -> - case R of - S -> {used,St}; - _ -> check_liveness(R, Is, St) - end; -check_liveness(R, [{move,S,D}|Is], St) -> - case R of - S -> {used,St}; - D -> {killed,St}; - _ -> check_liveness(R, Is, St) - end; -check_liveness(R, [{make_fun2,_,_,_,NumFree}|Is], St) -> - case R of - {x,X} when X < NumFree -> {used,St}; - {x,_} -> {killed,St}; - {y,_} -> not_used(check_liveness(R, Is, St)) - end; -check_liveness(R, [{'catch'=Op,Y,Fail}|Is], St) -> - Set = {set,[Y],[],{try_catch,Op,Fail}}, - check_liveness(R, [{block,[Set]}|Is], St); -check_liveness(R, [{'try'=Op,Y,Fail}|Is], St) -> - Set = {set,[Y],[],{try_catch,Op,Fail}}, - check_liveness(R, [{block,[Set]}|Is], St); -check_liveness(R, [{try_end,Y}|Is], St) -> - case R of - Y -> - {killed,St}; - {y,_} -> - %% y registers will be used if an exception occurs and - %% control transfers to the label given in the previous - %% try/2 instruction. - {used,St}; - _ -> - check_liveness(R, Is, St) - end; -check_liveness(R, [{catch_end,Y}|Is], St) -> - case R of - Y -> {killed,St}; - _ -> check_liveness(R, Is, St) - end; -check_liveness(R, [{get_tuple_element,S,_,D}|Is], St) -> - case R of - S -> {used,St}; - D -> {killed,St}; - _ -> check_liveness(R, Is, St) - end; -check_liveness(R, [{bs_context_to_binary,S}|Is], St) -> - case R of - S -> {used,St}; - _ -> check_liveness(R, Is, St) - end; -check_liveness(R, [{loop_rec,{f,_},{x,0}}|_], St) -> - case R of - {x,_} -> - {killed,St}; - _ -> - %% y register. Rarely happens. Be very conversative and - %% assume it's used. - {used,St} - end; -check_liveness(R, [{loop_rec_end,{f,Fail}}|_], St) -> - check_liveness_at(R, Fail, St); -check_liveness(R, [{line,_}|Is], St) -> - check_liveness(R, Is, St); -check_liveness(R, [{get_map_elements,{f,Fail},S,{list,L}}|Is], St0) -> - {Ss,Ds} = split_even(L), - case member(R, [S|Ss]) of - true -> - {used,St0}; - false -> - case check_liveness_at(R, Fail, St0) of - {killed,St}=Killed -> - case member(R, Ds) of - true -> Killed; - false -> check_liveness(R, Is, St) - end; - Other -> - Other - end - end; -check_liveness(R, [{put_map,F,Op,S,D,Live,{list,Puts}}|Is], St) -> - Set = {set,[D],[S|Puts],{alloc,Live,{put_map,Op,F}}}, - check_liveness(R, [{block,[Set]}||Is], St); -check_liveness(R, [{put_tuple,Ar,D}|Is], St) -> - Set = {set,[D],[],{put_tuple,Ar}}, - check_liveness(R, [{block,[Set]}||Is], St); -check_liveness(R, [{put_list,S1,S2,D}|Is], St) -> - Set = {set,[D],[S1,S2],put_list}, - check_liveness(R, [{block,[Set]}||Is], St); -check_liveness(R, [{test_heap,N,Live}|Is], St) -> - I = {block,[{set,[],[],{alloc,Live,{nozero,nostack,N,[]}}}]}, - check_liveness(R, [I|Is], St); -check_liveness(R, [{allocate_zero,N,Live}|Is], St) -> - I = {block,[{set,[],[],{alloc,Live,{zero,N,0,[]}}}]}, - check_liveness(R, [I|Is], St); -check_liveness(R, [{get_hd,S,D}|Is], St) -> - I = {block,[{set,[D],[S],get_hd}]}, - check_liveness(R, [I|Is], St); -check_liveness(R, [{get_tl,S,D}|Is], St) -> - I = {block,[{set,[D],[S],get_tl}]}, - check_liveness(R, [I|Is], St); -check_liveness(R, [remove_message|Is], St) -> - check_liveness(R, Is, St); -check_liveness({x,X}, [build_stacktrace|_], St) when X > 0 -> - {killed,St}; -check_liveness(R, [{recv_mark,_}|Is], St) -> - check_liveness(R, Is, St); -check_liveness(R, [{recv_set,_}|Is], St) -> - check_liveness(R, Is, St); -check_liveness(R, [{'%',_}|Is], St) -> - check_liveness(R, Is, St); -check_liveness(_R, Is, St) when is_list(Is) -> - %% Not implemented. Conservatively assume that the register is used. - {used,St}. - -check_liveness_everywhere(R, Lbls, St0) -> - check_liveness_everywhere_1(R, Lbls, killed, St0). - -check_liveness_everywhere_1(R, [{f,Lbl}|T], Res0, St0) -> - {Res1,St} = check_liveness_at(R, Lbl, St0), - Res = case Res1 of - killed -> Res0; - _ -> Res1 - end, - case Res of - used -> {used,St}; - _ -> check_liveness_everywhere_1(R, T, Res, St) - end; -check_liveness_everywhere_1(R, [_|T], Res, St) -> - check_liveness_everywhere_1(R, T, Res, St); -check_liveness_everywhere_1(_, [], Res, St) -> - {Res,St}. - -check_liveness_at(R, Lbl, #live{lbl=Ll,res=ResMemorized}=St0) -> - case gb_trees:lookup(Lbl, ResMemorized) of - {value,Res} -> - {Res,St0}; - none -> - {Res,St} = case gb_trees:lookup(Lbl, Ll) of - {value,Is} -> check_liveness(R, Is, St0); - none -> {used,St0} - end, - {Res,St#live{res=gb_trees:insert(Lbl, Res, St#live.res)}} - end. - -not_used({used,_}=Res) -> Res; -not_used({_,St}) -> {not_used,St}. - -check_liveness_exit(R, R, St) -> {used,St}; -check_liveness_exit({x,_}, _, St) -> {killed,St}; -check_liveness_exit({y,_}, _, St) -> {exit_not_used,St}. - -%% check_liveness_block(Reg, [Instruction], State) -> -%% {killed | not_used | used | alloc_used | transparent,State'} -%% Finds out how Reg is used in the instruction sequence inside a block. -%% Returns one of: -%% killed - Reg is assigned a new value or killed by an -%% allocation instruction -%% not_used - The value is not used, but the register is referenced -%% e.g. by an allocation instruction -%% transparent - Reg is neither used nor killed -%% alloc_used - Used only in an allocate instruction -%% used - Reg is explicitly used by an instruction -%% -%% Annotations are not allowed. -%% -%% (Unknown instructions will cause an exception.) - -check_liveness_block({x,X}=R, [{set,Ds,Ss,{alloc,Live,Op}}|Is], St0) -> - if - X >= Live -> - {killed,St0}; - true -> - case check_liveness_block_1(R, Ss, Ds, Op, Is, St0) of - {transparent,St} -> {alloc_used,St}; - {_,_}=Res -> not_used(Res) - end - end; -check_liveness_block({y,_}=R, [{set,Ds,Ss,{alloc,_Live,Op}}|Is], St0) -> - case check_liveness_block_1(R, Ss, Ds, Op, Is, St0) of - {transparent,St} -> {alloc_used,St}; - {_,_}=Res -> not_used(Res) - end; -check_liveness_block({y,_}=R, [{set,Ds,Ss,{try_catch,_,Op}}|Is], St0) -> - case Ds of - [R] -> - {killed,St0}; - _ -> - case check_liveness_block_1(R, Ss, Ds, Op, Is, St0) of - {exit_not_used,St} -> - {used,St}; - {transparent,St} -> - %% Conservatively assumed that it is used. - {used,St}; - {_,_}=Res -> - Res - end - end; -check_liveness_block(R, [{set,Ds,Ss,Op}|Is], St) -> - check_liveness_block_1(R, Ss, Ds, Op, Is, St); -check_liveness_block(_, [], St) -> {transparent,St}. - -check_liveness_block_1(R, Ss, Ds, Op, Is, St0) -> - case member(R, Ss) of - true -> - {used,St0}; - false -> - case check_liveness_block_2(R, Op, Ss, St0) of - {killed,St} -> - case member(R, Ds) of - true -> {killed,St}; - false -> check_liveness_block(R, Is, St) - end; - {exit_not_used,St} -> - case member(R, Ds) of - true -> {exit_not_used,St}; - false -> check_liveness_block(R, Is, St) - end; - {not_used,St} -> - not_used(case member(R, Ds) of - true -> {killed,St}; - false -> check_liveness_block(R, Is, St) - end); - {used,St} -> - {used,St} - end - end. - -check_liveness_block_2(R, {gc_bif,Op,{f,Lbl}}, Ss, St) -> - check_liveness_block_3(R, Lbl, {Op,length(Ss)}, St); -check_liveness_block_2(R, {bif,Op,{f,Lbl}}, Ss, St) -> - Arity = length(Ss), - - %% Note that is_function/2 is a type test but is not safe. - case erl_internal:comp_op(Op, Arity) orelse - (erl_internal:new_type_test(Op, Arity) andalso - erl_bifs:is_safe(erlang, Op, Arity)) of - true -> - {killed,St}; - false -> - check_liveness_block_3(R, Lbl, {Op,length(Ss)}, St) - end; -check_liveness_block_2(R, {put_map,_Op,{f,Lbl}}, _Ss, St) -> - check_liveness_block_3(R, Lbl, {unsafe,0}, St); -check_liveness_block_2(_, _, _, St) -> - {killed,St}. - -check_liveness_block_3({x,_}, 0, _FA, St) -> - {killed,St}; -check_liveness_block_3({y,_}, 0, {F,A}, St) -> - %% If the exception is thrown, the stack may be scanned, - %% thus implicitly using the y register. - case erl_bifs:is_safe(erlang, F, A) of - true -> {killed,St}; - false -> {used,St} - end; -check_liveness_block_3(R, Lbl, _FA, St0) -> - check_liveness_at(R, Lbl, St0). - -index_labels_1([{label,Lbl}|Is0], Acc) -> - Is = drop_labels(Is0), - index_labels_1(Is0, [{Lbl,Is}|Acc]); -index_labels_1([_|Is], Acc) -> - index_labels_1(Is, Acc); -index_labels_1([], Acc) -> gb_trees:from_orddict(sort(Acc)). - -drop_labels([{label,_}|Is]) -> drop_labels(Is); -drop_labels(Is) -> Is. - - replace_labels_1([{test,Test,{f,Lbl},Ops}|Is], Acc, D, Fb) -> replace_labels_1(Is, [{test,Test,{f,label(Lbl, D, Fb)},Ops}|Acc], D, Fb); replace_labels_1([{test,Test,{f,Lbl},Live,Ops,Dst}|Is], Acc, D, Fb) -> @@ -838,485 +144,7 @@ label(Old, D, Fb) -> _ -> Fb(Old) end. -%% Help function for combine_heap_needs. - -combine_alloc_lists(Al0) -> - Al1 = flatmap(fun(Words) when is_integer(Words) -> - [{words,Words}]; - ({alloc,List}) -> - List - end, Al0), - Al2 = sofs:relation(Al1), - Al3 = sofs:relation_to_family(Al2), - Al4 = sofs:to_external(Al3), - [{Tag,lists:sum(L)} || {Tag,L} <- Al4]. - -%% live_opt/4. - -%% Bit syntax instructions. -live_opt([{bs_context_to_binary,Src}=I|Is], Regs0, D, Acc) -> - Regs = x_live([Src], Regs0), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{bs_init,Fail,_,none,Ss,Dst}=I|Is], Regs0, D, Acc) -> - Regs1 = x_live(Ss, x_dead([Dst], Regs0)), - Regs = live_join_label(Fail, D, Regs1), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{bs_init,Fail,Info,Live0,Ss,Dst}|Is], Regs0, D, Acc) -> - Regs1 = x_dead([Dst], Regs0), - Live = live_regs(Regs1), - true = Live =< Live0, %Assertion. - Regs2 = live_call(Live), - Regs3 = x_live(Ss, Regs2), - Regs = live_join_label(Fail, D, Regs3), - I = {bs_init,Fail,Info,Live,Ss,Dst}, - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{bs_put,Fail,_,Ss}=I|Is], Regs0, D, Acc) -> - Regs1 = x_live(Ss, Regs0), - Regs = live_join_label(Fail, D, Regs1), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{bs_restore2,Src,_}=I|Is], Regs0, D, Acc) -> - Regs = x_live([Src], Regs0), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{bs_save2,Src,_}=I|Is], Regs0, D, Acc) -> - Regs = x_live([Src], Regs0), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{test,bs_start_match2,Fail,Live,[Src,_],_}=I|Is], _, D, Acc) -> - Regs0 = live_call(Live), - Regs1 = x_live([Src], Regs0), - Regs = live_join_label(Fail, D, Regs1), - live_opt(Is, Regs, D, [I|Acc]); - -%% Other instructions. -live_opt([{block,Bl0}|Is], Regs0, D, Acc) -> - Live0 = make_anno({used,Regs0}), - {Bl,Regs} = live_opt_block(reverse(Bl0), Regs0, D, [Live0]), - Live = make_anno({used,Regs}), - live_opt(Is, Regs, D, [{block,[Live|Bl]}|Acc]); -live_opt([build_stacktrace=I|Is], _, D, Acc) -> - live_opt(Is, live_call(1), D, [I|Acc]); -live_opt([raw_raise=I|Is], _, D, Acc) -> - live_opt(Is, live_call(3), D, [I|Acc]); -live_opt([{label,L}=I|Is], Regs, D0, Acc) -> - D = gb_trees:insert(L, Regs, D0), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{jump,{f,L}}=I|Is], _, D, Acc) -> - Regs = gb_trees:get(L, D), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([return=I|Is], _, D, Acc) -> - live_opt(Is, 1, D, [I|Acc]); -live_opt([{catch_end,_}=I|Is], _, D, Acc) -> - live_opt(Is, live_call(1), D, [I|Acc]); -live_opt([{badmatch,Src}=I|Is], _, D, Acc) -> - Regs = x_live([Src], 0), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{case_end,Src}=I|Is], _, D, Acc) -> - Regs = x_live([Src], 0), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{try_case_end,Src}=I|Is], _, D, Acc) -> - Regs = x_live([Src], 0), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([if_end=I|Is], _, D, Acc) -> - Regs = 0, - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{call,Arity,_}=I|Is], _, D, Acc) -> - live_opt(Is, live_call(Arity), D, [I|Acc]); -live_opt([{call_ext,Arity,_}=I|Is], _, D, Acc) -> - live_opt(Is, live_call(Arity), D, [I|Acc]); -live_opt([{call_fun,Arity}=I|Is], _, D, Acc) -> - live_opt(Is, live_call(Arity+1), D, [I|Acc]); -live_opt([{apply,Arity}=I|Is], _, D, Acc) -> - live_opt(Is, live_call(Arity+2), D, [I|Acc]); -live_opt([{make_fun2,_,_,_,Arity}=I|Is], _, D, Acc) -> - live_opt(Is, live_call(Arity), D, [I|Acc]); -live_opt([{test,_,Fail,Ss}=I|Is], Regs0, D, Acc) -> - Regs1 = x_live(Ss, Regs0), - Regs = live_join_label(Fail, D, Regs1), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{test,_,Fail,Live,Ss,_}=I|Is], _, D, Acc) -> - Regs0 = live_call(Live), - Regs1 = x_live(Ss, Regs0), - Regs = live_join_label(Fail, D, Regs1), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{select,_,Src,Fail,List}=I|Is], _, D, Acc) -> - Regs0 = 0, - Regs1 = x_live([Src], Regs0), - Regs = live_join_labels([Fail|List], D, Regs1), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{try_case,Y}=I|Is], Regs0, D, Acc) -> - Regs = live_call(1), - case Regs0 of - 0 -> - live_opt(Is, Regs, D, [{try_end,Y}|Acc]); - _ -> - live_opt(Is, live_call(1), D, [I|Acc]) - end; -live_opt([{loop_rec,_Fail,_Dst}=I|Is], _, D, Acc) -> - live_opt(Is, 0, D, [I|Acc]); -live_opt([timeout=I|Is], _, D, Acc) -> - live_opt(Is, 0, D, [I|Acc]); -live_opt([{wait,_}=I|Is], _, D, Acc) -> - live_opt(Is, 0, D, [I|Acc]); -live_opt([{get_map_elements,Fail,Src,{list,List}}=I|Is], Regs0, D, Acc) -> - {Ss,Ds} = split_even(List), - Regs1 = x_live([Src|Ss], x_dead(Ds, Regs0)), - Regs = live_join_label(Fail, D, Regs1), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{gc_bif,N,F,R,As,Dst}=I|Is], Regs0, D, Acc) -> - Bl = [{set,[Dst],As,{alloc,R,{gc_bif,N,F}}}], - {_,Regs} = live_opt_block(Bl, Regs0, D, []), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{bif,N,F,As,Dst}=I|Is], Regs0, D, Acc) -> - Bl = [{set,[Dst],As,{bif,N,F}}], - {_,Regs} = live_opt_block(Bl, Regs0, D, []), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{get_tuple_element,Src,Idx,Dst}=I|Is], Regs0, D, Acc) -> - Bl = [{set,[Dst],[Src],{get_tuple_element,Idx}}], - {_,Regs} = live_opt_block(Bl, Regs0, D, []), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{move,Src,Dst}=I|Is], Regs0, D, Acc) -> - Regs = x_live([Src], x_dead([Dst], Regs0)), - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{put_map,F,Op,S,Dst,R,{list,Puts}}=I|Is], Regs0, D, Acc) -> - Bl = [{set,[Dst],[S|Puts],{alloc,R,{put_map,Op,F}}}], - {_,Regs} = live_opt_block(Bl, Regs0, D, []), - live_opt(Is, Regs, D, [I|Acc]); - -%% Transparent instructions - they neither use nor modify x registers. -live_opt([{deallocate,_}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{kill,_}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{try_end,_}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{loop_rec_end,_}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{wait_timeout,_,nil}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{wait_timeout,_,{Tag,_}}=I|Is], Regs, D, Acc) when Tag =/= x -> - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{line,_}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{'catch',_,_}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{'try',_,_}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); - -%% The following instructions can occur if the "compilation" has been -%% started from a .S file using the 'from_asm' option. -live_opt([{trim,_,_}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{'%',_}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{recv_set,_}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); -live_opt([{recv_mark,_}=I|Is], Regs, D, Acc) -> - live_opt(Is, Regs, D, [I|Acc]); - -live_opt([], _, _, Acc) -> Acc. - -live_opt_block([{set,[{x,X}]=Ds,Ss,move}=I|Is], Regs0, D, Acc) -> - Regs = x_live(Ss, x_dead(Ds, Regs0)), - case is_live(X, Regs0) of - true -> - live_opt_block(Is, Regs, D, [I|Acc]); - false -> - %% Useless move, will never be used. - live_opt_block(Is, Regs, D, Acc) - end; -live_opt_block([{set,Ds,Ss,{alloc,Live0,AllocOp}}|Is], Regs0, D, Acc) -> - %% Calculate liveness from the point of view of the GC. - %% There will never be a GC if the instruction fails, so we should - %% ignore the failure branch. - GcRegs1 = x_dead(Ds, Regs0), - GcRegs = x_live(Ss, GcRegs1), - Live = live_regs(GcRegs), - - %% The life-time analysis used by the code generator is sometimes too - %% conservative, so it may be possible to lower the number of live - %% registers based on the exact liveness information. The main benefit is - %% that more optimizations that depend on liveness information (such as the - %% beam_dead pass) may be applied. - true = Live =< Live0, %Assertion. - I = {set,Ds,Ss,{alloc,Live,AllocOp}}, - - %% Calculate liveness from the point of view of the preceding instruction. - %% The liveness is the union of live registers in the GC and the live - %% registers at the failure label. - Regs1 = live_call(Live), - Regs = live_join_alloc(AllocOp, D, Regs1), - live_opt_block(Is, Regs, D, [I|Acc]); -live_opt_block([{set,Ds,Ss,{bif,_,Fail}}=I|Is], Regs0, D, Acc) -> - Regs1 = x_dead(Ds, Regs0), - Regs2 = x_live(Ss, Regs1), - Regs = live_join_label(Fail, D, Regs2), - live_opt_block(Is, Regs, D, [I|Acc]); -live_opt_block([{set,Ds,Ss,_}=I|Is], Regs0, D, Acc) -> - Regs = x_live(Ss, x_dead(Ds, Regs0)), - live_opt_block(Is, Regs, D, [I|Acc]); -live_opt_block([{'%anno',_}|Is], Regs, D, Acc) -> - live_opt_block(Is, Regs, D, Acc); -live_opt_block([], Regs, _, Acc) -> {Acc,Regs}. - -live_join_alloc({Kind,_Name,Fail}, D, Regs) when Kind =:= gc_bif; Kind =:= put_map -> - live_join_label(Fail, D, Regs); -live_join_alloc(_, _, Regs) -> Regs. - -live_join_labels([{f,L}|T], D, Regs0) when L =/= 0 -> - Regs = gb_trees:get(L, D) bor Regs0, - live_join_labels(T, D, Regs); -live_join_labels([_|T], D, Regs) -> - live_join_labels(T, D, Regs); -live_join_labels([], _, Regs) -> Regs. - -live_join_label({f,0}, _, Regs) -> - Regs; -live_join_label({f,L}, D, Regs) -> - gb_trees:get(L, D) bor Regs. - -live_call(Live) -> (1 bsl Live) - 1. - -live_regs(Regs) -> - live_regs_1(0, Regs). - -live_regs_1(N, 0) -> N; -live_regs_1(N, Regs) -> live_regs_1(N+1, Regs bsr 1). - -x_dead([{x,N}|Rs], Regs) -> x_dead(Rs, Regs band (bnot (1 bsl N))); -x_dead([_|Rs], Regs) -> x_dead(Rs, Regs); -x_dead([], Regs) -> Regs. - -x_live([{x,N}|Rs], Regs) -> x_live(Rs, Regs bor (1 bsl N)); -x_live([_|Rs], Regs) -> x_live(Rs, Regs); -x_live([], Regs) -> Regs. - -is_live(X, Regs) -> ((Regs bsr X) band 1) =:= 1. - split_even([], Ss, Ds) -> {reverse(Ss),reverse(Ds)}; split_even([S,D|Rs], Ss, Ds) -> split_even(Rs, [S|Ss], [D|Ds]). - -%%% -%%% Add annotations for defined registers. -%%% -%%% This analysis is done by scanning the instructions in -%%% execution order. -%%% - -defs([{apply,_}=I|Is], _Regs, D) -> - [I|defs(Is, 1, D)]; -defs([{bif,_,{f,Fail},_Src,Dst}=I|Is], Regs0, D) -> - Regs = def_regs([Dst], Regs0), - [I|defs(Is, Regs, update_regs(Fail, Regs0, D))]; -defs([{block,Block0}|Is], Regs0, D0) -> - {Block,Regs,D} = defs_list(Block0, Regs0, D0), - [{block,[make_anno({def,Regs0})|Block]}|defs(Is, Regs, D)]; -defs([{bs_init,{f,L},_,Live,_,Dst}=I|Is], Regs0, D) -> - Regs1 = case Live of - none -> Regs0; - _ -> init_def_regs(Live) - end, - Regs = def_regs([Dst], Regs1), - [I|defs(Is, Regs, update_regs(L, Regs, D))]; -defs([{test,bs_start_match2,{f,L},Live,_,Dst}=I|Is], _Regs, D) -> - Regs0 = init_def_regs(Live), - Regs = def_regs([Dst], Regs0), - [I|defs(Is, Regs, update_regs(L, Regs0, D))]; -defs([{bs_put,{f,L},_,_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, update_regs(L, Regs, D))]; -defs([build_stacktrace=I|Is], _Regs, D) -> - [I|defs(Is, 1, D)]; -defs([{call,_,_}=I|Is], _Regs, D) -> - [I|defs(Is, 1, D)]; -defs([{call_ext,_,{extfunc,M,F,A}}=I|Is], _Regs, D) -> - case erl_bifs:is_exit_bif(M, F, A) of - false -> - [I|defs(Is, 1, D)]; - true -> - [I|defs_unreachable(Is, D)] - end; -defs([{call_ext,_,_}=I|Is], _Regs, D) -> - [I|defs(Is, 1, D)]; -defs([{call_fun,_}=I|Is], _Regs, D) -> - [I|defs(Is, 1, D)]; -defs([{'catch',_,{f,L}}=I|Is], Regs, D) -> - RegsAtLabel = init_def_regs(1), - [I|defs(Is, Regs, update_regs(L, RegsAtLabel, D))]; -defs([{catch_end,_}=I|Is], _Regs, D) -> - Regs = init_def_regs(1), - [I|defs(Is, Regs, D)]; -defs([{gc_bif,_,{f,Fail},Live,_Src,Dst}=I|Is], Regs0, D) -> - true = all_defined(Live, Regs0), %Assertion. - Regs = def_regs([Dst], init_def_regs(Live)), - [I|defs(Is, Regs, update_regs(Fail, Regs0, D))]; -defs([{get_map_elements,{f,L},_Src,{list,DstList}}=I|Is], Regs0, D) -> - {_,Ds} = beam_utils:split_even(DstList), - Regs = def_regs(Ds, Regs0), - [I|defs(Is, Regs, update_regs(L, Regs0, D))]; -defs([{get_tuple_element,_,_,Dst}=I|Is], Regs0, D) -> - Regs = def_regs([Dst], Regs0), - [I|defs(Is, Regs, D)]; -defs([{jump,{f,L}}=I|Is], Regs, D) -> - [I|defs_unreachable(Is, update_regs(L, Regs, D))]; -defs([{label,L}=I|Is], Regs0, D) -> - case D of - #{L:=Regs1} -> - Regs = Regs0 band Regs1, - [I|defs(Is, Regs, D)]; - #{} -> - [I|defs(Is, Regs0, D)] - end; -defs([{loop_rec,{f,L},{x,0}}=I|Is], _Regs, D0) -> - RegsAtLabel = init_def_regs(0), - D = update_regs(L, RegsAtLabel, D0), - [I|defs(Is, init_def_regs(1), D)]; -defs([{loop_rec_end,_}=I|Is], _Regs, D) -> - [I|defs_unreachable(Is, D)]; -defs([{make_fun2,_,_,_,_}=I|Is], _Regs, D) -> - [I|defs(Is, 1, D)]; -defs([{move,_,Dst}=I|Is], Regs0, D) -> - Regs = def_regs([Dst], Regs0), - [I|defs(Is, Regs, D)]; -defs([{put_map,{f,Fail},_,_,Dst,_,_}=I|Is], Regs0, D) -> - Regs = def_regs([Dst], Regs0), - [I|defs(Is, Regs, update_regs(Fail, Regs0, D))]; -defs([raw_raise=I|Is], _Regs, D) -> - [I|defs(Is, 1, D)]; -defs([return=I|Is], _Regs, D) -> - [I|defs_unreachable(Is, D)]; -defs([{select,_,_Src,Fail,List}=I|Is], Regs, D0) -> - D = update_list([Fail|List], Regs, D0), - [I|defs_unreachable(Is, D)]; -defs([{test,_,{f,L},_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, update_regs(L, Regs, D))]; -defs([{test,_,{f,L},Live,_,Dst}=I|Is], Regs0, D) -> - true = all_defined(Live, Regs0), %Assertion. - Regs = def_regs([Dst], init_def_regs(Live)), - [I|defs(Is, Regs, update_regs(L, Regs0, D))]; -defs([{'try',_,{f,L}}=I|Is], Regs, D) -> - RegsAtLabel = init_def_regs(3), - [I|defs(Is, Regs, update_regs(L, RegsAtLabel, D))]; -defs([{try_case,_}=I|Is], _Regs, D) -> - [I|defs(Is, init_def_regs(3), D)]; -defs([{wait,_}=I|Is], _Regs, D) -> - [I|defs_unreachable(Is, D)]; -defs([{wait_timeout,_,_}=I|Is], _Regs, D) -> - [I|defs(Is, 0, D)]; - -%% Exceptions. -defs([{badmatch,_}=I|Is], _Regs, D) -> - [I|defs_unreachable(Is, D)]; -defs([{case_end,_}=I|Is], _Regs, D) -> - [I|defs_unreachable(Is, D)]; -defs([if_end=I|Is], _Regs, D) -> - [I|defs_unreachable(Is, D)]; -defs([{try_case_end,_}=I|Is], _Regs, D) -> - [I|defs_unreachable(Is, D)]; - -%% Neutral instructions -defs([{bs_context_to_binary,_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([{bs_restore2,_,_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([{bs_save2,_,_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([{deallocate,_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([{kill,_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([{line,_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([{recv_mark,_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([{recv_set,_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([timeout=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([{trim,_,_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([{try_end,_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([{'%',_}=I|Is], Regs, D) -> - [I|defs(Is, Regs, D)]; -defs([], _, _) -> []. - -defs_unreachable([{label,L}=I|Is], D) -> - case D of - #{L:=Regs} -> - [I|defs(Is, Regs, D)]; - #{} -> - defs_unreachable(Is, D) - end; -defs_unreachable([_|Is], D) -> - defs_unreachable(Is, D); -defs_unreachable([], _D) -> []. - -defs_list(Is, Regs, D) -> - defs_list(Is, Regs, D, []). - -defs_list([{set,Ds,_,{alloc,Live,Info}}=I|Is], Regs0, D0, Acc) -> - true = all_defined(Live, Regs0), %Assertion. - D = case Info of - {gc_bif,_,{f,Fail}} -> - update_regs(Fail, Regs0, D0); - {put_map,_,{f,Fail}} -> - update_regs(Fail, Regs0, D0); - _ -> - D0 - end, - Regs = def_regs(Ds, init_def_regs(Live)), - defs_list(Is, Regs, D, [I|Acc]); -defs_list([{set,Ds,_,Info}=I|Is], Regs0, D0, Acc) -> - D = case Info of - {bif,_,{f,Fail}} -> - update_regs(Fail, Regs0, D0); - {try_catch,'catch',{f,Fail}} -> - update_regs(Fail, init_def_regs(1), D0); - {try_catch,'try',{f,Fail}} -> - update_regs(Fail, init_def_regs(3), D0); - _ -> - D0 - end, - Regs = def_regs(Ds, Regs0), - defs_list(Is, Regs, D, [I|Acc]); -defs_list([], Regs, D, Acc) -> - {reverse(Acc),Regs,D}. - -init_def_regs(Arity) -> - (1 bsl Arity) - 1. - -def_regs([{x,X}|T], Regs) -> - def_regs(T, Regs bor (1 bsl X)); -def_regs([_|T], Regs) -> - def_regs(T, Regs); -def_regs([], Regs) -> Regs. - -update_list([{f,L}|T], Regs, D0) -> - D = update_regs(L, Regs, D0), - update_list(T, Regs, D); -update_list([_|T], Regs, D) -> - update_list(T, Regs, D); -update_list([], _Regs, D) -> D. - -update_regs(L, Regs0, D) -> - case D of - #{L:=Regs1} -> - Regs = Regs0 band Regs1, - D#{L:=Regs}; - #{} -> - D#{L=>Regs0} - end. - -all_defined(Live, Regs) -> - All = (1 bsl Live) - 1, - Regs band All =:= All. - -%%% -%%% Utilities. -%%% - -%% make_anno(Anno) -> WrappedAnno. -%% Wrap an annotation term. - -make_anno(Anno) -> - {'%anno',Anno}. diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index fb2e7df65c..09a5a6c104 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -26,8 +26,9 @@ %% Interface for compiler. -export([module/2, format_error/1]). +-export([type_anno/1, type_anno/2, type_anno/4]). --import(lists, [any/2,dropwhile/2,foldl/3,foreach/2,reverse/1]). +-import(lists, [dropwhile/2,foldl/3,member/2,reverse/1,sort/1,zip/2]). %% To be called by the compiler. @@ -44,6 +45,34 @@ module({Mod,Exp,Attr,Fs,Lc}=Code, _Opts) {error,[{atom_to_list(Mod),Es}]} end. +%% Provides a stable interface for type annotations, used by certain passes to +%% indicate that we can safely assume that a register has a given type. +-spec type_anno(term()) -> term(). +type_anno(atom) -> {atom,[]}; +type_anno(bool) -> bool; +type_anno({binary,_}) -> binary; +type_anno(cons) -> cons; +type_anno(float) -> {float,[]}; +type_anno(integer) -> {integer,[]}; +type_anno(list) -> list; +type_anno(map) -> map; +type_anno(match_context) -> match_context; +type_anno(number) -> number; +type_anno(nil) -> nil. + +-spec type_anno(term(), term()) -> term(). +type_anno(atom, Value) when is_atom(Value) -> {atom, Value}; +type_anno(float, Value) when is_float(Value) -> {float, Value}; +type_anno(integer, Value) when is_integer(Value) -> {integer, Value}. + +-spec type_anno(term(), term(), term(), term()) -> term(). +type_anno(tuple, Size, Exact, Elements) when is_integer(Size), Size >= 0, + is_map(Elements) -> + case Exact of + true -> {tuple, Size, Elements}; + false -> {tuple, [Size], Elements} + end. + -spec format_error(term()) -> iolist(). format_error({{_M,F,A},{I,Off,limit}}) -> @@ -90,34 +119,9 @@ format_error(Error) -> %% format as used in the compiler and in .S files. validate(Module, Fs) -> - Ft = index_bs_start_match(Fs, []), + Ft = index_parameter_types(Fs, []), validate_0(Module, Fs, Ft). -index_bs_start_match([{function,_,_,Entry,Code0}|Fs], Acc0) -> - Code = dropwhile(fun({label,L}) when L =:= Entry -> false; - (_) -> true - end, Code0), - case Code of - [{label,Entry}|Is] -> - Acc = index_bs_start_match_1(Is, Entry, Acc0), - index_bs_start_match(Fs, Acc); - _ -> - %% Something serious is wrong. Ignore it for now. - %% It will be detected and diagnosed later. - index_bs_start_match(Fs, Acc0) - end; -index_bs_start_match([], Acc) -> - gb_trees:from_orddict(lists:sort(Acc)). - -index_bs_start_match_1([{test,bs_start_match2,_,_,_,_}=I|_], Entry, Acc) -> - [{Entry,[I]}|Acc]; -index_bs_start_match_1([{test,_,{f,F},_},{bs_context_to_binary,_}|Is0], Entry, Acc) -> - [{label,F}|Is] = dropwhile(fun({label,L}) when L =:= F -> false; - (_) -> true - end, Is0), - index_bs_start_match_1(Is, Entry, Acc); -index_bs_start_match_1(_, _, Acc) -> Acc. - validate_0(_Module, [], _) -> []; validate_0(Module, [{function,Name,Ar,Entry,Code}|Fs], Ft) -> try validate_1(Code, Name, Ar, Entry, Ft) of @@ -132,41 +136,126 @@ validate_0(Module, [{function,Name,Ar,Entry,Code}|Fs], Ft) -> erlang:raise(Class, Error, Stack) end. +-record(value_ref, {id :: index()}). +-record(value, {op :: term(), args :: [argument()], type :: type()}). + +-type argument() :: #value_ref{} | literal(). + -type index() :: non_neg_integer(). --type reg_tab() :: gb_trees:tree(index(), 'none' | {'value', _}). - --record(st, %Emulation state - {x=init_regs(0, term) :: reg_tab(),%x register info. - y=init_regs(0, initialized) :: reg_tab(),%y register info. - f=init_fregs(), % - numy=none, %Number of y registers. - h=0, %Available heap size. - hf=0, %Available heap size for floats. - fls=undefined, %Floating point state. - ct=[], %List of hot catch/try labels - setelem=false, %Previous instruction was setelement/3. - puts_left=none %put/1 instructions left. - }). + +-type literal() :: {atom, [] | atom()} | + {float, [] | float()} | + {integer, [] | integer()} | + {literal, term()} | + nil. + +-type tuple_sz() :: [non_neg_integer()] | %% Inexact + non_neg_integer(). %% Exact. + +%% Match context type. +-record(ms, + {id=make_ref() :: reference(), %Unique ID. + valid=0 :: non_neg_integer(), %Valid slots + slots=0 :: non_neg_integer() %Number of slots + }). + +-type type() :: binary | + cons | + list | + map | + nil | + #ms{} | + ms_position | + none | + number | + term | + tuple_in_progress | + {tuple, tuple_sz(), #{ literal() => type() }} | + literal(). + +-type tag() :: initialized | + uninitialized | + {catchtag, [label()]} | + {trytag, [label()]}. + +-type x_regs() :: #{ {x, index()} => #value_ref{} }. +-type y_regs() :: #{ {y, index()} => tag() | #value_ref{} }. + +%% Emulation state +-record(st, + {%% All known values. + vs=#{} :: #{ #value_ref{} => #value{} }, + %% Register states. + xs=#{} :: x_regs(), + ys=#{} :: y_regs(), + f=init_fregs(), + %% A set of all registers containing "fragile" terms. That is, terms + %% that don't exist on our process heap and would be destroyed by a + %% GC. + fragile=cerl_sets:new() :: cerl_sets:set(), + %% Number of Y registers. + %% + %% Note that this may be 0 if there's a frame without saved values, + %% such as on a body-recursive call. + numy=none :: none | undecided | index(), + %% Available heap size. + h=0, + %Available heap size for floats. + hf=0, + %% Floating point state. + fls=undefined, + %% List of hot catch/try labels + ct=[], + %% Previous instruction was setelement/3. + setelem=false, + %% put/1 instructions left. + puts_left=none + }). -type label() :: integer(). -type label_set() :: gb_sets:set(label()). -type branched_tab() :: gb_trees:tree(label(), #st{}). -type ft_tab() :: gb_trees:tree(). --record(vst, %Validator state - {current=none :: #st{} | 'none', %Current state - branched=gb_trees:empty() :: branched_tab(), %States at jumps - labels=gb_sets:empty() :: label_set(), %All defined labels - ft=gb_trees:empty() :: ft_tab() %Some other functions - % in the module (those that start with bs_start_match2). - }). - -%% Match context type. --record(ms, - {id=make_ref() :: reference(), %Unique ID. - valid=0 :: non_neg_integer(), %Valid slots - slots=0 :: non_neg_integer() %Number of slots - }). +%% Validator state +-record(vst, + {%% Current state + current=none :: #st{} | 'none', + %% States at labels + branched=gb_trees:empty() :: branched_tab(), + %% All defined labels + labels=gb_sets:empty() :: label_set(), + %% Argument information of other functions in the module + ft=gb_trees:empty() :: ft_tab(), + %% Counter for #value_ref{} creation + ref_ctr=0 :: index() + }). + +index_parameter_types([{function,_,_,Entry,Code0}|Fs], Acc0) -> + Code = dropwhile(fun({label,L}) when L =:= Entry -> false; + (_) -> true + end, Code0), + case Code of + [{label,Entry}|Is] -> + Acc = index_parameter_types_1(Is, Entry, Acc0), + index_parameter_types(Fs, Acc); + _ -> + %% Something serious is wrong. Ignore it for now. + %% It will be detected and diagnosed later. + index_parameter_types(Fs, Acc0) + end; +index_parameter_types([], Acc) -> + gb_trees:from_orddict(sort(Acc)). + +index_parameter_types_1([{'%', {type_info, Reg, Type0}} | Is], Entry, Acc) -> + Type = case Type0 of + match_context -> #ms{}; + _ -> Type0 + end, + Key = {Entry, Reg}, + index_parameter_types_1(Is, Entry, [{Key, Type} | Acc]); +index_parameter_types_1(_, _, Acc) -> + Acc. validate_1(Is, Name, Arity, Entry, Ft) -> validate_2(labels(Is), Name, Arity, Entry, Ft). @@ -179,14 +268,10 @@ validate_2({Ls1,Is}, Name, Arity, _Entry, _Ft) -> validate_3({Ls2,Is}, Name, Arity, Entry, Mod, Ls1, Ft) -> Offset = 1 + length(Ls1) + 1 + length(Ls2), - EntryOK = lists:member(Entry, Ls2), + EntryOK = member(Entry, Ls2), if EntryOK -> - St = init_state(Arity), - Vst0 = #vst{current=St, - branched=gb_trees_from_list([{L,St} || L <- Ls1]), - labels=gb_sets:from_list(Ls1++Ls2), - ft=Ft}, + Vst0 = init_vst(Arity, Ls1, Ls2, Ft), MFA = {Mod,Name,Arity}, Vst = valfun(Is, MFA, Offset, Vst0), validate_fun_info_branches(Ls1, MFA, Vst); @@ -203,7 +288,13 @@ validate_fun_info_branches([], _, _) -> ok. validate_fun_info_branches_1(Arity, {_,_,Arity}, _) -> ok; validate_fun_info_branches_1(X, {Mod,Name,Arity}=MFA, Vst) -> try - get_term_type({x,X}, Vst) + case Vst of + #vst{current=#st{numy=none}} -> + ok; + #vst{current=#st{numy=Size}} -> + error({unexpected_stack_frame,Size}) + end, + assert_term({x,X}, Vst) catch Error -> I = {func_info,{atom,Mod},{atom,Name},Arity}, Offset = 2, @@ -224,19 +315,22 @@ labels_1([{line,_}|Is], R) -> labels_1(Is, R) -> {reverse(R),Is}. -init_state(Arity) -> - Xs = init_regs(Arity, term), - Ys = init_regs(0, initialized), - kill_heap_allocation(#st{x=Xs,y=Ys,numy=none,ct=[]}). +init_vst(Arity, Ls1, Ls2, Ft) -> + Vst0 = init_function_args(Arity - 1, #vst{current=#st{}}), + Branches = gb_trees_from_list([{L,Vst0#vst.current} || L <- Ls1]), + Labels = gb_sets:from_list(Ls1++Ls2), + Vst0#vst{branched=Branches, + labels=Labels, + ft=Ft}. + +init_function_args(-1, Vst) -> + Vst; +init_function_args(X, Vst) -> + init_function_args(X - 1, create_term(term, argument, [], {x,X}, Vst)). kill_heap_allocation(St) -> St#st{h=0,hf=0}. -init_regs(0, _) -> - gb_trees:empty(); -init_regs(N, Type) -> - gb_trees_from_list([{R,Type} || R <- lists:seq(0, N-1)]). - valfun([], MFA, _Offset, #vst{branched=Targets0,labels=Labels0}=Vst) -> Targets = gb_trees:keys(Targets0), Labels = gb_sets:to_list(Labels0), @@ -257,20 +351,25 @@ valfun([I|Is], MFA, Offset, Vst0) -> %% Instructions that are allowed in dead code or when failing, %% that is while the state is undecided in some way. -valfun_1({label,Lbl}, #vst{current=St0,branched=B,labels=Lbls}=Vst) -> - St = merge_states(Lbl, St0, B), - Vst#vst{current=St,branched=gb_trees:enter(Lbl, St, B), - labels=gb_sets:add(Lbl, Lbls)}; +valfun_1({label,Lbl}, #vst{current=St0, + ref_ctr=Counter0, + branched=B, + labels=Lbls}=Vst) -> + {St, Counter} = merge_states(Lbl, St0, B, Counter0), + Vst#vst{current=St, + ref_ctr=Counter, + branched=gb_trees:enter(Lbl, St, B), + labels=gb_sets:add(Lbl, Lbls)}; valfun_1(_I, #vst{current=none}=Vst) -> %% Ignore instructions after erlang:error/1,2, which %% the original R10B compiler thought would return. Vst; valfun_1({badmatch,Src}, Vst) -> - assert_term(Src, Vst), + assert_durable_term(Src, Vst), verify_y_init(Vst), kill_state(Vst); valfun_1({case_end,Src}, Vst) -> - assert_term(Src, Vst), + assert_durable_term(Src, Vst), verify_y_init(Vst), kill_state(Vst); valfun_1(if_end, Vst) -> @@ -278,35 +377,21 @@ valfun_1(if_end, Vst) -> kill_state(Vst); valfun_1({try_case_end,Src}, Vst) -> verify_y_init(Vst), - assert_term(Src, Vst), + assert_durable_term(Src, Vst), kill_state(Vst); -%% Instructions that can not cause exceptions -valfun_1({bs_context_to_binary,Ctx}, #vst{current=#st{x=Xs}}=Vst) -> - case Ctx of - {Tag,X} when Tag =:= x; Tag =:= y -> - Type = case gb_trees:lookup(X, Xs) of - {value,#ms{}} -> term; - _ -> get_term_type(Ctx, Vst) - end, - set_type_reg(Type, Ctx, Vst); - _ -> - error({bad_source,Ctx}) - end; +%% Instructions that cannot cause exceptions +valfun_1({bs_get_tail,Ctx,Dst,Live}, Vst0) -> + bsm_validate_context(Ctx, Vst0), + verify_live(Live, Vst0), + verify_y_init(Vst0), + Vst = prune_x_regs(Live, Vst0), + extract_term(binary, bs_get_tail, [Ctx], Dst, Vst, Vst0); valfun_1(bs_init_writable=I, Vst) -> call(I, 1, Vst); valfun_1(build_stacktrace=I, Vst) -> call(I, 1, Vst); -valfun_1({move,{y,_}=Src,{y,_}=Dst}, Vst) -> - %% The stack trimming optimization may generate a move from an initialized - %% but unassigned Y register to another Y register. - case get_term_type_1(Src, Vst) of - {catchtag,_} -> error({catchtag,Src}); - {trytag,_} -> error({trytag,Src}); - Type -> set_type_reg(Type, Dst, Vst) - end; valfun_1({move,Src,Dst}, Vst) -> - Type = get_move_term_type(Src, Vst), - set_type_reg(Type, Dst, Vst); + assign(Src, Dst, Vst); valfun_1({fmove,Src,{fr,_}=Dst}, Vst) -> assert_type(float, Src, Vst), set_freg(Dst, Vst); @@ -314,15 +399,15 @@ valfun_1({fmove,{fr,_}=Src,Dst}, Vst0) -> assert_freg_set(Src, Vst0), assert_fls(checked, Vst0), Vst = eat_heap_float(Vst0), - set_type_reg({float,[]}, Dst, Vst); -valfun_1({kill,{y,_}=Reg}, Vst) -> - set_type_y(initialized, Reg, Vst); -valfun_1({init,{y,_}=Reg}, Vst) -> - set_type_y(initialized, Reg, Vst); + create_term({float,[]}, fmove, [], Dst, Vst); +valfun_1({kill,Reg}, Vst) -> + create_tag(initialized, kill, [], Reg, Vst); +valfun_1({init,Reg}, Vst) -> + create_tag(initialized, init, [], Reg, Vst); valfun_1({test_heap,Heap,Live}, Vst) -> test_heap(Heap, Live, Vst); -valfun_1({bif,Op,{f,_},Src,Dst}=I, Vst) -> - case is_bif_safe(Op, length(Src)) of +valfun_1({bif,Op,{f,_},Ss,Dst}=I, Vst) -> + case is_bif_safe(Op, length(Ss)) of false -> %% Since the BIF can fail, make sure that any catch state %% is updated. @@ -330,21 +415,32 @@ valfun_1({bif,Op,{f,_},Src,Dst}=I, Vst) -> true -> %% It can't fail, so we finish handling it here (not updating %% catch state). - validate_src(Src, Vst), - Type = bif_type(Op, Src, Vst), - set_type_reg(Type, Dst, Vst) + validate_src(Ss, Vst), + Type = bif_return_type(Op, Ss, Vst), + extract_term(Type, {bif,Op}, Ss, Dst, Vst) end; %% Put instructions. valfun_1({put_list,A,B,Dst}, Vst0) -> assert_term(A, Vst0), assert_term(B, Vst0), Vst = eat_heap(2, Vst0), - set_type_reg(cons, Dst, Vst); + create_term(cons, put_list, [A, B], Dst, Vst); +valfun_1({put_tuple2,Dst,{list,Elements}}, Vst0) -> + _ = [assert_term(El, Vst0) || El <- Elements], + Size = length(Elements), + Vst = eat_heap(Size+1, Vst0), + {Es,_} = foldl(fun(Val, {Es0, Index}) -> + Type = get_term_type(Val, Vst0), + Es = set_element_type({integer,Index}, Type, Es0), + {Es, Index + 1} + end, {#{}, 1}, Elements), + Type = {tuple,Size,Es}, + create_term(Type, put_tuple2, [], Dst, Vst); valfun_1({put_tuple,Sz,Dst}, Vst0) when is_integer(Sz) -> Vst1 = eat_heap(1, Vst0), - Vst = set_type_reg(tuple_in_progress, Dst, Vst1), + Vst = create_term(tuple_in_progress, put_tuple, [], Dst, Vst1), #vst{current=St0} = Vst, - St = St0#st{puts_left={Sz,{Dst,{tuple,Sz}}}}, + St = St0#st{puts_left={Sz,{Dst,Sz,#{}}}}, Vst#vst{current=St}; valfun_1({put,Src}, Vst0) -> assert_term(Src, Vst0), @@ -353,11 +449,14 @@ valfun_1({put,Src}, Vst0) -> case St0 of #st{puts_left=none} -> error(not_building_a_tuple); - #st{puts_left={1,{Dst,Type}}} -> + #st{puts_left={1,{Dst,Sz,Es0}}} -> + Es = Es0#{ {integer,Sz} => get_term_type(Src, Vst0) }, St = St0#st{puts_left=none}, - set_type_reg(Type, Dst, Vst#vst{current=St}); - #st{puts_left={PutsLeft,Info}} when is_integer(PutsLeft) -> - St = St0#st{puts_left={PutsLeft-1,Info}}, + create_term({tuple,Sz,Es}, put_tuple, [], Dst, Vst#vst{current=St}); + #st{puts_left={PutsLeft,{Dst,Sz,Es0}}} when is_integer(PutsLeft) -> + Index = Sz - PutsLeft + 1, + Es = Es0#{ {integer,Index} => get_term_type(Src, Vst0) }, + St = St0#st{puts_left={PutsLeft-1,{Dst,Sz,Es}}}, Vst#vst{current=St} end; %% Instructions for optimization of selective receives. @@ -370,13 +469,28 @@ valfun_1(remove_message, Vst) -> %% The message term is no longer fragile. It can be used %% without restrictions. remove_fragility(Vst); +valfun_1({'%', {type_info, Reg, match_context}}, Vst) -> + update_type(fun meet/2, #ms{}, Reg, Vst); +valfun_1({'%', {type_info, Reg, Type}}, Vst) -> + %% Explicit type information inserted by optimization passes to indicate + %% that Reg has a certain type, so that we can accept cross-function type + %% optimizations. + update_type(fun meet/2, Type, Reg, Vst); +valfun_1({'%', {remove_fragility, Reg}}, Vst) -> + %% This is a hack to make prim_eval:'receive'/2 work. + %% + %% Normally it's illegal to pass fragile terms as a function argument as we + %% have no way of knowing what the callee will do with it, but we know that + %% prim_eval:'receive'/2 won't leak the term, nor cause a GC since it's + %% disabled while matching messages. + remove_fragility(Reg, Vst); valfun_1({'%',_}, Vst) -> Vst; valfun_1({line,_}, Vst) -> Vst; %% Exception generating calls valfun_1({call_ext,Live,Func}=I, Vst) -> - case return_type(Func, Vst) of + case call_return_type(Func, Vst) of exception -> verify_live(Live, Vst), %% The stack will be scanned, so Y registers @@ -391,88 +505,122 @@ valfun_1(_I, #vst{current=#st{ct=undecided}}) -> %% %% Allocate and deallocate, et.al valfun_1({allocate,Stk,Live}, Vst) -> - allocate(false, Stk, 0, Live, Vst); + allocate(uninitialized, Stk, 0, Live, Vst); valfun_1({allocate_heap,Stk,Heap,Live}, Vst) -> - allocate(false, Stk, Heap, Live, Vst); + allocate(uninitialized, Stk, Heap, Live, Vst); valfun_1({allocate_zero,Stk,Live}, Vst) -> - allocate(true, Stk, 0, Live, Vst); + allocate(initialized, Stk, 0, Live, Vst); valfun_1({allocate_heap_zero,Stk,Heap,Live}, Vst) -> - allocate(true, Stk, Heap, Live, Vst); + allocate(initialized, Stk, Heap, Live, Vst); valfun_1({deallocate,StkSize}, #vst{current=#st{numy=StkSize}}=Vst) -> verify_no_ct(Vst), deallocate(Vst); valfun_1({deallocate,_}, #vst{current=#st{numy=NumY}}) -> error({allocated,NumY}); -valfun_1({trim,N,Remaining}, #vst{current=#st{y=Yregs0,numy=NumY}=St}=Vst) -> +valfun_1({trim,N,Remaining}, #vst{current=St0}=Vst) -> + #st{numy=NumY} = St0, if - N =< NumY, N+Remaining =:= NumY -> - Yregs1 = [{Y-N,Type} || {Y,Type} <- gb_trees:to_list(Yregs0), Y >= N], - Yregs = gb_trees_from_list(Yregs1), - Vst#vst{current=St#st{y=Yregs,numy=NumY-N}}; - true -> - error({trim,N,Remaining,allocated,NumY}) + N =< NumY, N+Remaining =:= NumY -> + Vst#vst{current=trim_stack(N, 0, NumY, St0)}; + N > NumY; N+Remaining =/= NumY -> + error({trim,N,Remaining,allocated,NumY}) end; %% Catch & try. -valfun_1({'catch',Dst,{f,Fail}}, Vst0) when Fail /= none -> - Vst = #vst{current=#st{ct=Fails}=St} = - set_type_y({catchtag,[Fail]}, Dst, Vst0), - Vst#vst{current=St#st{ct=[[Fail]|Fails]}}; -valfun_1({'try',Dst,{f,Fail}}, Vst0) -> - Vst = #vst{current=#st{ct=Fails}=St} = - set_type_y({trytag,[Fail]}, Dst, Vst0), - Vst#vst{current=St#st{ct=[[Fail]|Fails]}}; -valfun_1({catch_end,Reg}, #vst{current=#st{ct=[Fail|Fails]}}=Vst0) -> - case get_special_y_type(Reg, Vst0) of - {catchtag,Fail} -> - Vst = #vst{current=St} = set_catch_end(Reg, Vst0), - Xs = gb_trees_from_list([{0,term}]), - Vst#vst{current=St#st{x=Xs,ct=Fails,fls=undefined}}; - Type -> - error({bad_type,Type}) +valfun_1({'catch',Dst,{f,Fail}}, Vst) when Fail =/= none -> + init_try_catch_branch(catchtag, Dst, Fail, Vst); +valfun_1({'try',Dst,{f,Fail}}, Vst) when Fail =/= none -> + init_try_catch_branch(trytag, Dst, Fail, Vst); +valfun_1({catch_end,Reg}, #vst{current=#st{ct=[Fail|_]}}=Vst0) -> + case get_tag_type(Reg, Vst0) of + {catchtag,Fail} -> + %% {x,0} contains the caught term, if any. + create_term(term, catch_end, [], {x,0}, kill_catch_tag(Reg, Vst0)); + Type -> + error({wrong_tag_type,Type}) end; -valfun_1({try_end,Reg}, #vst{current=#st{ct=[Fail|Fails]}=St0}=Vst0) -> - case get_special_y_type(Reg, Vst0) of - {trytag,Fail} -> - Vst = case Fail of - [FailLabel] -> branch_state(FailLabel, Vst0); - _ -> Vst0 - end, - St = St0#st{ct=Fails,fls=undefined}, - set_catch_end(Reg, Vst#vst{current=St}); - Type -> - error({bad_type,Type}) +valfun_1({try_end,Reg}, #vst{current=#st{ct=[Fail|_]}}=Vst) -> + case get_tag_type(Reg, Vst) of + {trytag,Fail} -> + %% Kill the catch tag, note that x registers are unaffected. + kill_catch_tag(Reg, Vst); + Type -> + error({wrong_tag_type,Type}) end; -valfun_1({try_case,Reg}, #vst{current=#st{ct=[Fail|Fails]}}=Vst0) -> - case get_special_y_type(Reg, Vst0) of - {trytag,Fail} -> - Vst = #vst{current=St} = set_catch_end(Reg, Vst0), - Xs = gb_trees_from_list([{0,{atom,[]}},{1,term},{2,term}]), - Vst#vst{current=St#st{x=Xs,ct=Fails,fls=undefined}}; - Type -> - error({bad_type,Type}) +valfun_1({try_case,Reg}, #vst{current=#st{ct=[Fail|_]}}=Vst0) -> + case get_tag_type(Reg, Vst0) of + {trytag,Fail} -> + %% Kill the catch tag and all x registers. + Vst1 = prune_x_regs(0, kill_catch_tag(Reg, Vst0)), + + %% Class:Error:Stacktrace + Vst2 = create_term({atom,[]}, try_case, [], {x,0}, Vst1), + Vst = create_term(term, try_case, [], {x,1}, Vst2), + create_term(term, try_case, [], {x,2}, Vst); + Type -> + error({wrong_tag_type,Type}) end; valfun_1({get_list,Src,D1,D2}, Vst0) -> + assert_not_literal(Src), assert_type(cons, Src, Vst0), - Vst = set_type_reg(term, Src, D1, Vst0), - set_type_reg(term, Src, D2, Vst); + Vst = extract_term(term, get_hd, [Src], D1, Vst0), + extract_term(term, get_tl, [Src], D2, Vst); valfun_1({get_hd,Src,Dst}, Vst) -> + assert_not_literal(Src), assert_type(cons, Src, Vst), - set_type_reg(term, Src, Dst, Vst); + extract_term(term, get_hd, [Src], Dst, Vst); valfun_1({get_tl,Src,Dst}, Vst) -> + assert_not_literal(Src), assert_type(cons, Src, Vst), - set_type_reg(term, Src, Dst, Vst); -valfun_1({get_tuple_element,Src,I,Dst}, Vst) -> - assert_type({tuple_element,I+1}, Src, Vst), - set_type_reg(term, Src, Dst, Vst); + extract_term(term, get_tl, [Src], Dst, Vst); +valfun_1({get_tuple_element,Src,N,Dst}, Vst) -> + assert_not_literal(Src), + assert_type({tuple_element,N+1}, Src, Vst), + Index = {integer,N+1}, + Type = get_element_type(Index, Src, Vst), + extract_term(Type, {bif,element}, [Index, Src], Dst, Vst); +valfun_1({jump,{f,Lbl}}, Vst) -> + branch(Lbl, Vst, + fun(SuccVst) -> + %% The next instruction is never executed. + kill_state(SuccVst) + end); valfun_1(I, Vst) -> valfun_2(I, Vst). -%% Update branched state if necessary and try next set of instructions. -valfun_2(I, #vst{current=#st{ct=[]}}=Vst) -> - valfun_3(I, Vst); +init_try_catch_branch(Tag, Dst, Fail, Vst0) -> + Vst1 = create_tag({Tag,[Fail]}, 'try_catch', [], Dst, Vst0), + #vst{current=#st{ct=Fails}=St0} = Vst1, + St = St0#st{ct=[[Fail]|Fails]}, + Vst = Vst0#vst{current=St}, + + branch(Fail, Vst, + fun(CatchVst) -> + #vst{current=#st{ys=Ys}} = CatchVst, + maps:fold(fun init_catch_handler_1/3, CatchVst, Ys) + end, + fun(SuccVst) -> + %% All potentially-throwing instructions after this + %% one will implicitly branch to the fail label; + %% see valfun_2/2 + SuccVst + end). + +%% Set the initial state at the try/catch label. Assume that Y registers +%% contain terms or try/catch tags. +init_catch_handler_1(Reg, initialized, Vst) -> + create_term(term, 'catch_handler', [], Reg, Vst); +init_catch_handler_1(Reg, uninitialized, Vst) -> + create_term(term, 'catch_handler', [], Reg, Vst); +init_catch_handler_1(_, _, Vst) -> + Vst. + valfun_2(I, #vst{current=#st{ct=[[Fail]|_]}}=Vst) when is_integer(Fail) -> - %% Update branched state + %% We have an active try/catch tag and we can jump there from this + %% instruction, so we need to update the branched state of the try/catch + %% handler. valfun_3(I, branch_state(Fail, Vst)); +valfun_2(I, #vst{current=#st{ct=[]}}=Vst) -> + valfun_3(I, Vst); valfun_2(_, _) -> error(ambiguous_catch_try_state). @@ -480,17 +628,23 @@ valfun_2(_, _) -> %% Floating point. valfun_3({fconv,Src,{fr,_}=Dst}, Vst) -> assert_term(Src, Vst), - set_freg(Dst, Vst); -valfun_3({bif,fadd,_,[_,_]=Src,Dst}, Vst) -> - float_op(Src, Dst, Vst); -valfun_3({bif,fdiv,_,[_,_]=Src,Dst}, Vst) -> - float_op(Src, Dst, Vst); -valfun_3({bif,fmul,_,[_,_]=Src,Dst}, Vst) -> - float_op(Src, Dst, Vst); -valfun_3({bif,fnegate,_,[_]=Src,Dst}, Vst) -> - float_op(Src, Dst, Vst); -valfun_3({bif,fsub,_,[_,_]=Src,Dst}, Vst) -> - float_op(Src, Dst, Vst); + + %% An exception is raised on error, hence branching to 0. + branch(0, Vst, + fun(SuccVst0) -> + SuccVst = update_type(fun meet/2, number, Src, SuccVst0), + set_freg(Dst, SuccVst) + end); +valfun_3({bif,fadd,_,[_,_]=Ss,Dst}, Vst) -> + float_op(Ss, Dst, Vst); +valfun_3({bif,fdiv,_,[_,_]=Ss,Dst}, Vst) -> + float_op(Ss, Dst, Vst); +valfun_3({bif,fmul,_,[_,_]=Ss,Dst}, Vst) -> + float_op(Ss, Dst, Vst); +valfun_3({bif,fnegate,_,[_]=Ss,Dst}, Vst) -> + float_op(Ss, Dst, Vst); +valfun_3({bif,fsub,_,[_,_]=Ss,Dst}, Vst) -> + float_op(Ss, Dst, Vst); valfun_3(fclearerror, Vst) -> case get_fls(Vst) of undefined -> ok; @@ -541,67 +695,87 @@ valfun_4({call_ext_last,_,_,_}, #vst{current=#st{numy=NumY}}) -> valfun_4({make_fun2,_,_,_,Live}, Vst) -> call(make_fun, Live, Vst); %% Other BIFs -valfun_4({bif,tuple_size,{f,Fail},[Tuple],Dst}, Vst0) -> - TupleType0 = get_term_type(Tuple, Vst0), - Vst1 = branch_state(Fail, Vst0), - TupleType = upgrade_tuple_type({tuple,[0]}, TupleType0), - Vst = set_type(TupleType, Tuple, Vst1), - set_type_reg({integer,[]}, Dst, Vst); -valfun_4({bif,element,{f,Fail},[Pos,Tuple],Dst}, Vst0) -> - TupleType0 = get_term_type(Tuple, Vst0), - PosType = get_term_type(Pos, Vst0), - Vst1 = branch_state(Fail, Vst0), - TupleType = upgrade_tuple_type({tuple,[get_tuple_size(PosType)]}, TupleType0), - Vst = set_type(TupleType, Tuple, Vst1), - set_type_reg(term, Tuple, Dst, Vst); +valfun_4({bif,element,{f,Fail},[Pos,Src],Dst}, Vst) -> + branch(Fail, Vst, + fun(SuccVst0) -> + PosType = get_term_type(Pos, SuccVst0), + TupleType = {tuple,[get_tuple_size(PosType)],#{}}, + + SuccVst1 = update_type(fun meet/2, TupleType, + Src, SuccVst0), + SuccVst = update_type(fun meet/2, {integer,[]}, + Pos, SuccVst1), + + ElementType = get_element_type(PosType, Src, SuccVst), + extract_term(ElementType, {bif,element}, [Pos,Src], + Dst, SuccVst) + end); valfun_4({bif,raise,{f,0},Src,_Dst}, Vst) -> validate_src(Src, Vst), kill_state(Vst); valfun_4(raw_raise=I, Vst) -> call(I, 3, Vst); -valfun_4({bif,map_get,{f,Fail},[_Key,Map]=Src,Dst}, Vst0) -> - validate_src(Src, Vst0), - Vst1 = branch_state(Fail, Vst0), - Vst = set_type(map, Map, Vst1), - Type = propagate_fragility(term, Src, Vst), - set_type_reg(Type, Dst, Vst); -valfun_4({bif,is_map_key,{f,Fail},[_Key,Map]=Src,Dst}, Vst0) -> - validate_src(Src, Vst0), - Vst1 = branch_state(Fail, Vst0), - Vst = set_type(map, Map, Vst1), - Type = propagate_fragility(bool, Src, Vst), - set_type_reg(Type, Dst, Vst); -valfun_4({bif,Op,{f,Fail},Src,Dst}, Vst0) -> - validate_src(Src, Vst0), - Vst = branch_state(Fail, Vst0), - Type0 = bif_type(Op, Src, Vst), - Type = propagate_fragility(Type0, Src, Vst), - set_type_reg(Type, Dst, Vst); -valfun_4({gc_bif,Op,{f,Fail},Live,Src,Dst}, #vst{current=St0}=Vst0) -> +valfun_4({bif,Op,{f,Fail},[Src]=Ss,Dst}, Vst) when Op =:= hd; Op =:= tl -> + assert_term(Src, Vst), + branch(Fail, Vst, + fun(FailVst) -> + update_type(fun subtract/2, cons, Src, FailVst) + end, + fun(SuccVst0) -> + SuccVst = update_type(fun meet/2, cons, Src, SuccVst0), + extract_term(term, {bif,Op}, Ss, Dst, SuccVst) + end); +valfun_4({bif,Op,{f,Fail},Ss,Dst}, Vst) -> + validate_src(Ss, Vst), + branch(Fail, Vst, + fun(SuccVst0) -> + %% Infer argument types. Note that we can't subtract + %% types as the BIF could fail for reasons other than + %% bad argument types. + ArgTypes = bif_arg_types(Op, Ss), + SuccVst = foldl(fun({Arg, T}, V) -> + update_type(fun meet/2, T, Arg, V) + end, SuccVst0, zip(Ss, ArgTypes)), + Type = bif_return_type(Op, Ss, SuccVst), + extract_term(Type, {bif,Op}, Ss, Dst, SuccVst) + end); +valfun_4({gc_bif,Op,{f,Fail},Live,Ss,Dst}, #vst{current=St0}=Vst0) -> + validate_src(Ss, Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), + + %% Heap allocations and X registers are killed regardless of whether we + %% fail or not, as we may fail after GC. St = kill_heap_allocation(St0), - Vst1 = Vst0#vst{current=St}, - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - validate_src(Src, Vst), - Type0 = bif_type(Op, Src, Vst), - Type = propagate_fragility(Type0, Src, Vst), - set_type_reg(Type, Dst, Vst); + Vst = prune_x_regs(Live, Vst0#vst{current=St}), + + branch(Fail, Vst, + fun(SuccVst0) -> + ArgTypes = bif_arg_types(Op, Ss), + SuccVst = foldl(fun({Arg, T}, V) -> + update_type(fun meet/2, T, Arg, V) + end, SuccVst0, zip(Ss, ArgTypes)), + + Type = bif_return_type(Op, Ss, SuccVst), + + %% We're passing Vst0 as the original because the + %% registers were pruned before the branch. + extract_term(Type, {gc_bif,Op}, Ss, Dst, SuccVst, Vst0) + end); valfun_4(return, #vst{current=#st{numy=none}}=Vst) -> - assert_term({x,0}, Vst), + assert_durable_term({x,0}, Vst), kill_state(Vst); valfun_4(return, #vst{current=#st{numy=NumY}}) -> error({stack_frame,NumY}); -valfun_4({jump,{f,Lbl}}, Vst) -> - kill_state(branch_state(Lbl, Vst)); -valfun_4({loop_rec,{f,Fail},Dst}, Vst0) -> - Vst = branch_state(Fail, Vst0), - %% This term may not be part of the root set until - %% remove_message/0 is executed. If control transfers - %% to the loop_rec_end/1 instruction, no part of - %% this term must be stored in a Y register. - set_type_reg({fragile,term}, Dst, Vst); +valfun_4({loop_rec,{f,Fail},Dst}, Vst) -> + %% This term may not be part of the root set until remove_message/0 is + %% executed. If control transfers to the loop_rec_end/1 instruction, no + %% part of this term must be stored in a Y register. + branch(Fail, Vst, + fun(SuccVst0) -> + {Ref, SuccVst} = new_value(term, loop_rec, [], SuccVst0), + mark_fragile(Dst, set_reg_vref(Ref, Dst, SuccVst)) + end); valfun_4({wait,_}, Vst) -> verify_y_init(Vst), kill_state(Vst); @@ -612,131 +786,169 @@ valfun_4({wait_timeout,_,Src}, Vst) -> valfun_4({loop_rec_end,_}, Vst) -> verify_y_init(Vst), kill_state(Vst); -valfun_4(timeout, #vst{current=St}=Vst) -> - Vst#vst{current=St#st{x=init_regs(0, term)}}; +valfun_4(timeout, Vst) -> + prune_x_regs(0, Vst); valfun_4(send, Vst) -> call(send, 2, Vst); -valfun_4({set_tuple_element,Src,Tuple,I}, Vst) -> +valfun_4({set_tuple_element,Src,Tuple,N}, Vst) -> + I = N + 1, assert_term(Src, Vst), - assert_type({tuple_element,I+1}, Tuple, Vst), - Vst; + assert_type({tuple_element,I}, Tuple, Vst), + %% Manually update the tuple type; we can't rely on the ordinary update + %% helpers as we must support overwriting (rather than just widening or + %% narrowing) known elements, and we can't use extract_term either since + %% the source tuple may be aliased. + {tuple, Sz, Es0} = get_term_type(Tuple, Vst), + Es = set_element_type({integer,I}, get_term_type(Src, Vst), Es0), + override_type({tuple, Sz, Es}, Tuple, Vst); %% Match instructions. valfun_4({select_val,Src,{f,Fail},{list,Choices}}, Vst) -> assert_term(Src, Vst), - Lbls = [L || {f,L} <- Choices]++[Fail], - kill_state(foldl(fun(L, S) -> branch_state(L, S) end, Vst, Lbls)); + assert_choices(Choices), + validate_select_val(Fail, Choices, Src, Vst); valfun_4({select_tuple_arity,Tuple,{f,Fail},{list,Choices}}, Vst) -> assert_type(tuple, Tuple, Vst), - kill_state(branch_arities(Choices, Tuple, branch_state(Fail, Vst))); + assert_arities(Choices), + validate_select_tuple_arity(Fail, Choices, Tuple, Vst); %% New bit syntax matching instructions. -valfun_4({test,bs_start_match2,{f,Fail},Live,[Ctx,NeedSlots],Ctx}, Vst0) -> - %% If source and destination registers are the same, match state - %% is OK as input. - CtxType = get_move_term_type(Ctx, Vst0), - verify_live(Live, Vst0), - verify_y_init(Vst0), - Vst1 = prune_x_regs(Live, Vst0), - BranchVst = case CtxType of - #ms{} -> - %% The failure branch will never be taken when Ctx - %% is a match context. Therefore, the type for Ctx - %% at the failure label must not be match_context - %% (or we could reject legal code). - set_type_reg(term, Ctx, Vst1); - _ -> - Vst1 - end, - Vst = branch_state(Fail, BranchVst), - set_type_reg(bsm_match_state(NeedSlots), Ctx, Vst); -valfun_4({test,bs_start_match2,{f,Fail},Live,[Src,Slots],Dst}, Vst0) -> - assert_term(Src, Vst0), - verify_live(Live, Vst0), - verify_y_init(Vst0), - Vst1 = prune_x_regs(Live, Vst0), - Vst = branch_state(Fail, Vst1), - set_type_reg(bsm_match_state(Slots), Src, Dst, Vst); +valfun_4({test,bs_start_match3,{f,Fail},Live,[Src],Dst}, Vst) -> + validate_bs_start_match(Fail, Live, bsm_match_state(), Src, Dst, Vst); +valfun_4({test,bs_start_match2,{f,Fail},Live,[Src,Slots],Dst}, Vst) -> + validate_bs_start_match(Fail, Live, bsm_match_state(Slots), Src, Dst, Vst); valfun_4({test,bs_match_string,{f,Fail},[Ctx,_,_]}, Vst) -> bsm_validate_context(Ctx, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_skip_bits2,{f,Fail},[Ctx,Src,_,_]}, Vst) -> bsm_validate_context(Ctx, Vst), assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_test_tail2,{f,Fail},[Ctx,_]}, Vst) -> bsm_validate_context(Ctx, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_test_unit,{f,Fail},[Ctx,_]}, Vst) -> bsm_validate_context(Ctx, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, fun(V) -> V end); valfun_4({test,bs_skip_utf8,{f,Fail},[Ctx,Live,_]}, Vst) -> validate_bs_skip_utf(Fail, Ctx, Live, Vst); valfun_4({test,bs_skip_utf16,{f,Fail},[Ctx,Live,_]}, Vst) -> validate_bs_skip_utf(Fail, Ctx, Live, Vst); valfun_4({test,bs_skip_utf32,{f,Fail},[Ctx,Live,_]}, Vst) -> validate_bs_skip_utf(Fail, Ctx, Live, Vst); -valfun_4({test,bs_get_integer2,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> - validate_bs_get(Fail, Ctx, Live, {integer, []}, Dst, Vst); -valfun_4({test,bs_get_float2,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> - validate_bs_get(Fail, Ctx, Live, {float, []}, Dst, Vst); -valfun_4({test,bs_get_binary2,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> - Type = propagate_fragility(term, [Ctx], Vst), - validate_bs_get(Fail, Ctx, Live, Type, Dst, Vst); -valfun_4({test,bs_get_utf8,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> - validate_bs_get(Fail, Ctx, Live, {integer, []}, Dst, Vst); -valfun_4({test,bs_get_utf16,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> - validate_bs_get(Fail, Ctx, Live, {integer, []}, Dst, Vst); -valfun_4({test,bs_get_utf32,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> - validate_bs_get(Fail, Ctx, Live, {integer, []}, Dst, Vst); +valfun_4({test,bs_get_integer2=Op,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> + validate_bs_get(Op, Fail, Ctx, Live, {integer, []}, Dst, Vst); +valfun_4({test,bs_get_float2=Op,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> + validate_bs_get(Op, Fail, Ctx, Live, {float, []}, Dst, Vst); +valfun_4({test,bs_get_binary2=Op,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> + validate_bs_get(Op, Fail, Ctx, Live, binary, Dst, Vst); +valfun_4({test,bs_get_utf8=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> + validate_bs_get(Op, Fail, Ctx, Live, {integer, []}, Dst, Vst); +valfun_4({test,bs_get_utf16=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> + validate_bs_get(Op, Fail, Ctx, Live, {integer, []}, Dst, Vst); +valfun_4({test,bs_get_utf32=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> + validate_bs_get(Op, Fail, Ctx, Live, {integer, []}, Dst, Vst); valfun_4({bs_save2,Ctx,SavePoint}, Vst) -> bsm_save(Ctx, SavePoint, Vst); valfun_4({bs_restore2,Ctx,SavePoint}, Vst) -> bsm_restore(Ctx, SavePoint, Vst); +valfun_4({bs_get_position, Ctx, Dst, Live}, Vst0) -> + bsm_validate_context(Ctx, Vst0), + verify_live(Live, Vst0), + verify_y_init(Vst0), + Vst = prune_x_regs(Live, Vst0), + create_term(ms_position, bs_get_position, [Ctx], Dst, Vst, Vst0); +valfun_4({bs_set_position, Ctx, Pos}, Vst) -> + bsm_validate_context(Ctx, Vst), + assert_type(ms_position, Pos, Vst), + Vst; %% Other test instructions. -valfun_4({test,is_float,{f,Lbl},[Float]}, Vst) -> - assert_term(Float, Vst), - set_type({float,[]}, Float, branch_state(Lbl, Vst)); -valfun_4({test,is_tuple,{f,Lbl},[Tuple]}, Vst) -> - Type0 = get_term_type(Tuple, Vst), - Type = upgrade_tuple_type({tuple,[0]}, Type0), - set_type(Type, Tuple, branch_state(Lbl, Vst)); -valfun_4({test,is_nonempty_list,{f,Lbl},[Cons]}, Vst) -> - assert_term(Cons, Vst), - set_type(cons, Cons, branch_state(Lbl, Vst)); -valfun_4({test,test_arity,{f,Lbl},[Tuple,Sz]}, Vst) when is_integer(Sz) -> - assert_type(tuple, Tuple, Vst), - set_type_reg({tuple,Sz}, Tuple, branch_state(Lbl, Vst)); -valfun_4({test,is_tagged_tuple,{f,Lbl},[Src,Sz,_Atom]}, Vst) -> - validate_src([Src], Vst), - set_type_reg({tuple, Sz}, Src, branch_state(Lbl, Vst)); valfun_4({test,has_map_fields,{f,Lbl},Src,{list,List}}, Vst) -> assert_type(map, Src, Vst), assert_unique_map_keys(List), - branch_state(Lbl, Vst); -valfun_4({test,is_map,{f,Lbl},[Src]}, Vst0) -> - Vst = branch_state(Lbl, Vst0), - case Src of - {Tag,_} when Tag =:= x; Tag =:= y -> - set_type_reg(map, Src, Vst); - {literal,Map} when is_map(Map) -> - Vst; - _ -> - kill_state(Vst) - end; + branch(Lbl, Vst, fun(V) -> V end); +valfun_4({test,is_atom,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, {atom,[]}, Src, Vst); +valfun_4({test,is_binary,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, binary, Src, Vst); +valfun_4({test,is_bitstr,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, binary, Src, Vst); +valfun_4({test,is_boolean,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, bool, Src, Vst); +valfun_4({test,is_float,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, {float,[]}, Src, Vst); +valfun_4({test,is_tuple,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, {tuple,[0],#{}}, Src, Vst); +valfun_4({test,is_integer,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, {integer,[]}, Src, Vst); +valfun_4({test,is_nonempty_list,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, cons, Src, Vst); +valfun_4({test,is_number,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, number, Src, Vst); +valfun_4({test,is_list,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, list, Src, Vst); +valfun_4({test,is_map,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, map, Src, Vst); +valfun_4({test,is_nil,{f,Lbl},[Src]}, Vst) -> + %% is_nil is an exact check against the 'nil' value, and should not be + %% treated as a simple type test. + assert_term(Src, Vst), + branch(Lbl, Vst, + fun(FailVst) -> + update_ne_types(Src, nil, FailVst) + end, + fun(SuccVst) -> + update_eq_types(Src, nil, SuccVst) + end); +valfun_4({test,test_arity,{f,Lbl},[Tuple,Sz]}, Vst) when is_integer(Sz) -> + assert_type(tuple, Tuple, Vst), + Type = {tuple, Sz, #{}}, + type_test(Lbl, Type, Tuple, Vst); +valfun_4({test,is_tagged_tuple,{f,Lbl},[Src,Sz,Atom]}, Vst) -> + assert_term(Src, Vst), + Type = {tuple, Sz, #{ {integer,1} => Atom }}, + type_test(Lbl, Type, Src, Vst); +valfun_4({test,is_eq_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> + validate_src(Ss, Vst), + branch(Lbl, Vst, + fun(FailVst) -> + update_ne_types(Src, Val, FailVst) + end, + fun(SuccVst) -> + update_eq_types(Src, Val, SuccVst) + end); +valfun_4({test,is_ne_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> + validate_src(Ss, Vst), + branch(Lbl, Vst, + fun(FailVst) -> + update_eq_types(Src, Val, FailVst) + end, + fun(SuccVst) -> + update_ne_types(Src, Val, SuccVst) + end); valfun_4({test,_Op,{f,Lbl},Src}, Vst) -> + %% is_pid, is_reference, et cetera. validate_src(Src, Vst), - branch_state(Lbl, Vst); + branch(Lbl, Vst, fun(V) -> V end); valfun_4({bs_add,{f,Fail},[A,B,_],Dst}, Vst) -> assert_term(A, Vst), assert_term(B, Vst), - set_type_reg({integer,[]}, Dst, branch_state(Fail, Vst)); + branch(Fail, Vst, + fun(SuccVst) -> + create_term({integer,[]}, bs_add, [A, B], Dst, SuccVst) + end); valfun_4({bs_utf8_size,{f,Fail},A,Dst}, Vst) -> assert_term(A, Vst), - set_type_reg({integer,[]}, Dst, branch_state(Fail, Vst)); + branch(Fail, Vst, + fun(SuccVst) -> + create_term({integer,[]}, bs_utf8_size, [A], Dst, SuccVst) + end); valfun_4({bs_utf16_size,{f,Fail},A,Dst}, Vst) -> assert_term(A, Vst), - set_type_reg({integer,[]}, Dst, branch_state(Fail, Vst)); + branch(Fail, Vst, + fun(SuccVst) -> + create_term({integer,[]}, bs_utf16_size, [A], Dst, SuccVst) + end); valfun_4({bs_init2,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), @@ -746,10 +958,12 @@ valfun_4({bs_init2,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> true -> assert_term(Sz, Vst0) end, - Vst1 = heap_alloc(Heap, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - set_type_reg(binary, Dst, Vst); + Vst = heap_alloc(Heap, Vst0), + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + create_term(binary, bs_init2, [], Dst, SuccVst, SuccVst0) + end); valfun_4({bs_init_bits,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), @@ -759,116 +973,203 @@ valfun_4({bs_init_bits,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> true -> assert_term(Sz, Vst0) end, - Vst1 = heap_alloc(Heap, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - set_type_reg(binary, Dst, Vst); + Vst = heap_alloc(Heap, Vst0), + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + create_term(binary, bs_init_bits, [], Dst, SuccVst) + end); valfun_4({bs_append,{f,Fail},Bits,Heap,Live,_Unit,Bin,_Flags,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), assert_term(Bits, Vst0), assert_term(Bin, Vst0), - Vst1 = heap_alloc(Heap, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - set_type_reg(binary, Dst, Vst); -valfun_4({bs_private_append,{f,Fail},Bits,_Unit,Bin,_Flags,Dst}, Vst0) -> - assert_term(Bits, Vst0), - assert_term(Bin, Vst0), - Vst = branch_state(Fail, Vst0), - set_type_reg(binary, Dst, Vst); + Vst = heap_alloc(Heap, Vst0), + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + create_term(binary, bs_append, [Bin], Dst, SuccVst, SuccVst0) + end); +valfun_4({bs_private_append,{f,Fail},Bits,_Unit,Bin,_Flags,Dst}, Vst) -> + assert_term(Bits, Vst), + assert_term(Bin, Vst), + branch(Fail, Vst, + fun(SuccVst) -> + create_term(binary, bs_private_append, [Bin], Dst, SuccVst) + end); valfun_4({bs_put_string,Sz,_}, Vst) when is_integer(Sz) -> Vst; valfun_4({bs_put_binary,{f,Fail},Sz,_,_,Src}, Vst) -> assert_term(Sz, Vst), assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, binary, Src, SuccVst) + end); valfun_4({bs_put_float,{f,Fail},Sz,_,_,Src}, Vst) -> assert_term(Sz, Vst), assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {float,[]}, Src, SuccVst) + end); valfun_4({bs_put_integer,{f,Fail},Sz,_,_,Src}, Vst) -> assert_term(Sz, Vst), assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); valfun_4({bs_put_utf8,{f,Fail},_,Src}, Vst) -> assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); valfun_4({bs_put_utf16,{f,Fail},_,Src}, Vst) -> assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); valfun_4({bs_put_utf32,{f,Fail},_,Src}, Vst) -> assert_term(Src, Vst), - branch_state(Fail, Vst); + branch(Fail, Vst, + fun(SuccVst) -> + update_type(fun meet/2, {integer,[]}, Src, SuccVst) + end); %% Map instructions. -valfun_4({put_map_assoc,{f,Fail},Src,Dst,Live,{list,List}}, Vst) -> - verify_put_map(Fail, Src, Dst, Live, List, Vst); -valfun_4({put_map_exact,{f,Fail},Src,Dst,Live,{list,List}}, Vst) -> - verify_put_map(Fail, Src, Dst, Live, List, Vst); +valfun_4({put_map_assoc=Op,{f,Fail},Src,Dst,Live,{list,List}}, Vst) -> + verify_put_map(Op, Fail, Src, Dst, Live, List, Vst); +valfun_4({put_map_exact=Op,{f,Fail},Src,Dst,Live,{list,List}}, Vst) -> + verify_put_map(Op, Fail, Src, Dst, Live, List, Vst); valfun_4({get_map_elements,{f,Fail},Src,{list,List}}, Vst) -> verify_get_map(Fail, Src, List, Vst); valfun_4(_, _) -> error(unknown_instruction). verify_get_map(Fail, Src, List, Vst0) -> + assert_not_literal(Src), %OTP 22. assert_type(map, Src, Vst0), - Vst1 = foldl(fun(D, Vsti) -> - case is_reg_defined(D,Vsti) of - true -> set_type_reg(term,D,Vsti); - false -> Vsti - end - end, Vst0, extract_map_vals(List)), - Vst2 = branch_state(Fail, Vst1), - Keys = extract_map_keys(List), - assert_unique_map_keys(Keys), - verify_get_map_pair(List, Src, Vst0, Vst2). - -extract_map_vals([_Key,Val|T]) -> - [Val|extract_map_vals(T)]; -extract_map_vals([]) -> []. + + branch(Fail, Vst0, + fun(FailVst) -> + clobber_map_vals(List, Src, FailVst) + end, + fun(SuccVst) -> + Keys = extract_map_keys(List), + assert_unique_map_keys(Keys), + extract_map_vals(List, Src, SuccVst, SuccVst) + end). + +%% get_map_elements may leave its destinations in an inconsistent state when +%% the fail label is taken. Consider the following: +%% +%% {get_map_elements,{f,7},{x,1},{list,[{atom,a},{x,1},{atom,b},{x,2}]}}. +%% +%% If 'a' exists but not 'b', {x,1} is overwritten when we jump to {f,7}. +clobber_map_vals([Key,Dst|T], Map, Vst0) -> + case is_reg_defined(Dst, Vst0) of + true -> + Vst = extract_term(term, {bif,map_get}, [Key, Map], Dst, Vst0), + clobber_map_vals(T, Map, Vst); + false -> + clobber_map_vals(T, Map, Vst0) + end; +clobber_map_vals([], _Map, Vst) -> + Vst. extract_map_keys([Key,_Val|T]) -> [Key|extract_map_keys(T)]; extract_map_keys([]) -> []. -verify_get_map_pair([Src,Dst|Vs], Map, Vst0, Vsti0) -> - assert_term(Src, Vst0), - Vsti = set_type_reg(term, Map, Dst, Vsti0), - verify_get_map_pair(Vs, Map, Vst0, Vsti); -verify_get_map_pair([], _Map, _Vst0, Vst) -> Vst. +extract_map_vals([Key,Dst|Vs], Map, Vst0, Vsti0) -> + assert_term(Key, Vst0), + Vsti = extract_term(term, {bif,map_get}, [Key, Map], Dst, Vsti0), + extract_map_vals(Vs, Map, Vst0, Vsti); +extract_map_vals([], _Map, _Vst0, Vst) -> + Vst. -verify_put_map(Fail, Src, Dst, Live, List, Vst0) -> +verify_put_map(Op, Fail, Src, Dst, Live, List, Vst0) -> assert_type(map, Src, Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), - foreach(fun (Term) -> assert_term(Term, Vst0) end, List), - Vst1 = heap_alloc(0, Vst0), - Vst2 = branch_state(Fail, Vst1), - Vst = prune_x_regs(Live, Vst2), - Keys = extract_map_keys(List), - assert_unique_map_keys(Keys), - set_type_reg(map, Dst, Vst). + _ = [assert_term(Term, Vst0) || Term <- List], + Vst = heap_alloc(0, Vst0), + + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + Keys = extract_map_keys(List), + assert_unique_map_keys(Keys), + create_term(map, Op, [Src], Dst, SuccVst, SuccVst0) + end). + +%% +%% Common code for validating bs_start_match* instructions. +%% + +validate_bs_start_match(Fail, Live, Type, Src, Dst, Vst) -> + verify_live(Live, Vst), + verify_y_init(Vst), + + %% #ms{} can represent either a match context or a term, so we have to mark + %% the source as a term if it fails with a match context as an input. This + %% hack is only needed until we get proper union types. + branch(Fail, Vst, + fun(FailVst) -> + case get_movable_term_type(Src, FailVst) of + #ms{} -> override_type(term, Src, FailVst); + _ -> FailVst + end + end, + fun(SuccVst0) -> + SuccVst1 = update_type(fun meet/2, binary, + Src, SuccVst0), + SuccVst = prune_x_regs(Live, SuccVst1), + extract_term(Type, bs_start_match, [Src], Dst, + SuccVst, SuccVst0) + end). %% %% Common code for validating bs_get* instructions. %% -validate_bs_get(Fail, Ctx, Live, Type, Dst, Vst0) -> - bsm_validate_context(Ctx, Vst0), - verify_live(Live, Vst0), - verify_y_init(Vst0), - Vst1 = prune_x_regs(Live, Vst0), - Vst = branch_state(Fail, Vst1), - set_type_reg(Type, Dst, Vst). +validate_bs_get(Op, Fail, Ctx, Live, Type, Dst, Vst) -> + bsm_validate_context(Ctx, Vst), + verify_live(Live, Vst), + verify_y_init(Vst), + + branch(Fail, Vst, + fun(SuccVst0) -> + SuccVst = prune_x_regs(Live, SuccVst0), + extract_term(Type, Op, [Ctx], Dst, SuccVst, SuccVst0) + end). %% %% Common code for validating bs_skip_utf* instructions. %% -validate_bs_skip_utf(Fail, Ctx, Live, Vst0) -> - bsm_validate_context(Ctx, Vst0), - verify_y_init(Vst0), - verify_live(Live, Vst0), - Vst = prune_x_regs(Live, Vst0), - branch_state(Fail, Vst). +validate_bs_skip_utf(Fail, Ctx, Live, Vst) -> + bsm_validate_context(Ctx, Vst), + verify_y_init(Vst), + verify_live(Live, Vst), + + branch(Fail, Vst, + fun(SuccVst) -> + prune_x_regs(Live, SuccVst) + end). + +%% +%% Common code for is_$type instructions. +%% +type_test(Fail, Type, Reg, Vst) -> + assert_term(Reg, Vst), + branch(Fail, Vst, + fun(FailVst) -> + update_type(fun subtract/2, Type, Reg, FailVst) + end, + fun(SuccVst) -> + update_type(fun meet/2, Type, Reg, SuccVst) + end). %% %% Special state handling for setelement/3 and set_tuple_element/3 instructions. @@ -885,34 +1186,29 @@ val_dsetel({set_tuple_element,_,_,_}, #vst{current=#st{setelem=false}}) -> error(illegal_context_for_set_tuple_element); val_dsetel({set_tuple_element,_,_,_}, #vst{current=#st{setelem=true}}=Vst) -> Vst; +val_dsetel({get_tuple_element,_,_,_}, Vst) -> + Vst; val_dsetel({line,_}, Vst) -> Vst; val_dsetel(_, #vst{current=#st{setelem=true}=St}=Vst) -> Vst#vst{current=St#st{setelem=false}}; val_dsetel(_, Vst) -> Vst. -kill_state(#vst{current=#st{ct=[[Fail]|_]}}=Vst) when is_integer(Fail) -> - %% There is an active catch. Make sure that we merge the state into - %% the catch label before clearing it, so that that we can be sure - %% that the label gets a state. - kill_state_1(branch_state(Fail, Vst)); kill_state(Vst) -> - kill_state_1(Vst). - -kill_state_1(Vst) -> Vst#vst{current=none}. %% A "plain" call. %% The stackframe must be initialized. %% The instruction will return to the instruction following the call. -call(Name, Live, #vst{current=St}=Vst) -> - verify_call_args(Name, Live, Vst), - verify_y_init(Vst), - case return_type(Name, Vst) of - Type when Type =/= exception -> - %% Type is never 'exception' because it has been handled earlier. - Xs = gb_trees_from_list([{0,Type}]), - Vst#vst{current=St#st{x=Xs,f=init_fregs()}} +call(Name, Live, #vst{current=St0}=Vst0) -> + verify_call_args(Name, Live, Vst0), + verify_y_init(Vst0), + case call_return_type(Name, Vst0) of + Type when Type =/= exception -> + %% Type is never 'exception' because it has been handled earlier. + St = St0#st{f=init_fregs()}, + Vst = prune_x_regs(0, Vst0#vst{current=St}), + create_term(Type, call, [], {x,0}, Vst) end. %% Tail call. @@ -928,79 +1224,131 @@ tail_call(Name, Live, Vst0) -> verify_call_args(_, 0, #vst{}) -> ok; verify_call_args({f,Lbl}, Live, Vst) when is_integer(Live)-> - verify_local_call(Lbl, Live, Vst); + verify_local_args(Live - 1, Lbl, #{}, Vst); verify_call_args(_, Live, Vst) when is_integer(Live)-> - verify_call_args_1(Live, Vst); + verify_remote_args_1(Live - 1, Vst); verify_call_args(_, Live, _) -> error({bad_number_of_live_regs,Live}). -verify_call_args_1(0, _) -> ok; -verify_call_args_1(N, Vst) -> - X = N - 1, - get_term_type({x,X}, Vst), - verify_call_args_1(X, Vst). - -verify_local_call(Lbl, Live, Vst) -> - case all_ms_in_x_regs(Live, Vst) of - [{R,Ctx}] -> - %% Verify that there is a suitable bs_start_match2 instruction. - verify_call_match_context(Lbl, R, Vst), - - %% Since the callee has consumed the match context, - %% there must be no additional copies in Y registers. - #ms{id=Id} = Ctx, - case ms_in_y_regs(Id, Vst) of - [] -> - ok; - [_|_]=Ys -> - error({multiple_match_contexts,[R|Ys]}) - end; - [_,_|_]=Xs0 -> - Xs = [R || {R,_} <- Xs0], - error({multiple_match_contexts,Xs}); - [] -> - ok +verify_remote_args_1(-1, _) -> + ok; +verify_remote_args_1(X, Vst) -> + assert_durable_term({x, X}, Vst), + verify_remote_args_1(X - 1, Vst). + +verify_local_args(-1, _Lbl, _CtxIds, _Vst) -> + ok; +verify_local_args(X, Lbl, CtxIds, Vst) -> + Reg = {x, X}, + assert_not_fragile(Reg, Vst), + case get_movable_term_type(Reg, Vst) of + #ms{id=Id}=Type -> + case CtxIds of + #{ Id := Other } -> + error({multiple_match_contexts, [Reg, Other]}); + #{} -> + verify_arg_type(Lbl, Reg, Type, Vst), + verify_local_args(X - 1, Lbl, CtxIds#{ Id => Reg }, Vst) + end; + Type -> + verify_arg_type(Lbl, Reg, Type, Vst), + verify_local_args(X - 1, Lbl, CtxIds, Vst) end. -all_ms_in_x_regs(0, _Vst) -> - []; -all_ms_in_x_regs(Live0, Vst) -> - Live = Live0 - 1, - R = {x,Live}, - case get_move_term_type(R, Vst) of - #ms{}=M -> - [{R,M}|all_ms_in_x_regs(Live, Vst)]; - _ -> - all_ms_in_x_regs(Live, Vst) +%% Verifies that the given argument narrows to what the function expects. +verify_arg_type(Lbl, Reg, #ms{}, #vst{ft=Ft}) -> + %% Match contexts require explicit support, and may not be passed to a + %% function that accepts arbitrary terms. + case gb_trees:lookup({Lbl, Reg}, Ft) of + {value, #ms{}} -> ok; + _ -> error(no_bs_start_match2) + end; +verify_arg_type(Lbl, Reg, GivenType, #vst{ft=Ft}) -> + case gb_trees:lookup({Lbl, Reg}, Ft) of + {value, #ms{}} -> + %% Functions that accept match contexts also accept all other + %% terms. This will change once we support union types. + ok; + {value, RequiredType} -> + case vat_1(GivenType, RequiredType) of + true -> ok; + false -> error({bad_arg_type, Reg, GivenType, RequiredType}) + end; + none -> + ok end. -ms_in_y_regs(Id, #vst{current=#st{y=Ys0}}) -> - Ys = gb_trees:to_list(Ys0), - [{y,Y} || {Y,#ms{id=OtherId}} <- Ys, OtherId =:= Id]. +%% Checks whether the Given argument is compatible with the Required one. This +%% is essentially a relaxed version of 'meet(Given, Req) =:= Given', where we +%% accept that the Given value has the right type but not necessarily the exact +%% same value; if {atom,gurka} is required, we'll consider {atom,[]} valid. +%% +%% This will catch all problems that could crash the emulator, like passing a +%% 1-tuple when the callee expects a 3-tuple, but some value errors might slip +%% through. +vat_1(Same, Same) -> true; +vat_1({atom,A}, {atom,B}) -> A =:= B orelse is_list(A) orelse is_list(B); +vat_1({atom,A}, bool) -> is_boolean(A) orelse is_list(A); +vat_1(bool, {atom,B}) -> is_boolean(B) orelse is_list(B); +vat_1(cons, list) -> true; +vat_1({float,A}, {float,B}) -> A =:= B orelse is_list(A) orelse is_list(B); +vat_1({float,_}, number) -> true; +vat_1({integer,A}, {integer,B}) -> A =:= B orelse is_list(A) orelse is_list(B); +vat_1({integer,_}, number) -> true; +vat_1(_, {literal,_}) -> false; +vat_1({literal,_}=Lit, Required) -> vat_1(get_literal_type(Lit), Required); +vat_1(nil, list) -> true; +vat_1({tuple,SzA,EsA}, {tuple,SzB,EsB}) -> + if + is_list(SzB) -> + tuple_sz(SzA) >= tuple_sz(SzB) andalso vat_elements(EsA, EsB); + SzA =:= SzB -> + vat_elements(EsA, EsB); + SzA =/= SzB -> + false + end; +vat_1(_, _) -> false. -verify_call_match_context(Lbl, Ctx, #vst{ft=Ft}) -> - case gb_trees:lookup(Lbl, Ft) of - none -> - error(no_bs_start_match2); - {value,[{test,bs_start_match2,_,_,[Ctx,_],Ctx}|_]} -> - ok; - {value,[{test,bs_start_match2,_,_,_,_}=I|_]} -> - error({unsuitable_bs_start_match2,I}) - end. +vat_elements(EsA, EsB) -> + maps:fold(fun(Key, Req, Acc) -> + case EsA of + #{ Key := Given } -> Acc andalso vat_1(Given, Req); + #{} -> false + end + end, true, EsB). -allocate(Zero, Stk, Heap, Live, #vst{current=#st{numy=none}}=Vst0) -> +allocate(Tag, Stk, Heap, Live, #vst{current=#st{numy=none}=St}=Vst0) -> verify_live(Live, Vst0), - Vst = #vst{current=St} = prune_x_regs(Live, Vst0), - Ys = init_regs(Stk, case Zero of - true -> initialized; - false -> uninitialized - end), - heap_alloc(Heap, Vst#vst{current=St#st{y=Ys,numy=Stk}}); + Vst1 = Vst0#vst{current=St#st{numy=Stk}}, + Vst2 = prune_x_regs(Live, Vst1), + Vst = init_stack(Tag, Stk - 1, Vst2), + heap_alloc(Heap, Vst); allocate(_, _, _, _, #vst{current=#st{numy=Numy}}) -> error({existing_stack_frame,{size,Numy}}). deallocate(#vst{current=St}=Vst) -> - Vst#vst{current=St#st{y=init_regs(0, initialized),numy=none}}. + Vst#vst{current=St#st{ys=#{},numy=none}}. + +init_stack(_Tag, -1, Vst) -> + Vst; +init_stack(Tag, Y, Vst) -> + init_stack(Tag, Y - 1, create_tag(Tag, allocate, [], {y,Y}, Vst)). + +trim_stack(From, To, Top, #st{ys=Ys0}=St) when From =:= Top -> + Ys = maps:filter(fun({y,Y}, _) -> Y < To end, Ys0), + St#st{numy=To,ys=Ys}; +trim_stack(From, To, Top, St0) -> + Src = {y, From}, + Dst = {y, To}, + + #st{ys=Ys0} = St0, + Ys = case Ys0 of + #{ Src := Ref } -> Ys0#{ Dst => Ref }; + #{} -> error({invalid_shift,Src,Dst}) + end, + St = St0#st{ys=Ys}, + + trim_stack(From + 1, To + 1, Top, St). test_heap(Heap, Live, Vst0) -> verify_live(Live, Vst0), @@ -1025,13 +1373,43 @@ heap_alloc_2([{floats,Floats}|T], St0) -> St = St0#st{hf=Floats}, heap_alloc_2(T, St); heap_alloc_2([], St) -> St. - -prune_x_regs(Live, #vst{current=#st{x=Xs0}=St0}=Vst) when is_integer(Live) -> - Xs1 = gb_trees:to_list(Xs0), - Xs = [P || {R,_}=P <- Xs1, R < Live], - St = St0#st{x=gb_trees:from_orddict(Xs)}, + +prune_x_regs(Live, #vst{current=St0}=Vst) when is_integer(Live) -> + #st{fragile=Fragile0,xs=Xs0} = St0, + Fragile = cerl_sets:filter(fun({x,X}) -> + X < Live; + ({y,_}) -> + true + end, Fragile0), + Xs = maps:filter(fun({x,X}, _) -> + X < Live + end, Xs0), + St = St0#st{fragile=Fragile,xs=Xs}, Vst#vst{current=St}. +%% All choices in a select_val list must be integers, floats, or atoms. +%% All must be of the same type. +assert_choices([{Tag,_},{f,_}|T]) -> + if + Tag =:= atom; Tag =:= float; Tag =:= integer -> + assert_choices_1(T, Tag); + true -> + error(bad_select_list) + end; +assert_choices([]) -> ok. + +assert_choices_1([{Tag,_},{f,_}|T], Tag) -> + assert_choices_1(T, Tag); +assert_choices_1([_,{f,_}|_], _Tag) -> + error(bad_select_list); +assert_choices_1([], _Tag) -> ok. + +assert_arities([Arity,{f,_}|T]) when is_integer(Arity) -> + assert_arities(T); +assert_arities([]) -> ok; +assert_arities(_) -> error(bad_tuple_arity_list). + + %%% %%% Floating point checking. %%% @@ -1051,8 +1429,8 @@ prune_x_regs(Live, #vst{current=#st{x=Xs0}=St0}=Vst) when is_integer(Live) -> %%% fmove Src {fr,_} %% Move INTO floating point register. %%% -float_op(Src, Dst, Vst0) -> - foreach (fun(S) -> assert_freg_set(S, Vst0) end, Src), +float_op(Ss, Dst, Vst0) -> + _ = [assert_freg_set(S, Vst0) || S <- Ss], assert_fls(cleared, Vst0), Vst = set_fls(cleared, Vst0), set_freg(Dst, Vst). @@ -1070,8 +1448,7 @@ get_fls(#vst{current=#st{fls=Fls}}) when is_atom(Fls) -> Fls. init_fregs() -> 0. -set_freg({fr,Fr}=Freg, #vst{current=#st{f=Fregs0}=St}=Vst) - when is_integer(Fr), 0 =< Fr -> +set_freg({fr,Fr}=Freg, #vst{current=#st{f=Fregs0}=St}=Vst) -> check_limit(Freg), Bit = 1 bsl Fr, if @@ -1107,7 +1484,10 @@ assert_unique_map_keys([]) -> assert_unique_map_keys([_]) -> ok; assert_unique_map_keys([_,_|_]=Ls) -> - Vs = [get_literal(L) || L <- Ls], + Vs = [begin + assert_literal(L), + L + end || L <- Ls], case length(Vs) =:= sets:size(sets:from_list(Vs)) of true -> ok; false -> error(keys_not_unique) @@ -1117,6 +1497,8 @@ assert_unique_map_keys([_,_|_]=Ls) -> %%% New binary matching instructions. %%% +bsm_match_state() -> + #ms{}. bsm_match_state(Slots) -> #ms{slots=Slots}. @@ -1124,13 +1506,13 @@ bsm_validate_context(Reg, Vst) -> _ = bsm_get_context(Reg, Vst), ok. -bsm_get_context({x,X}=Reg, #vst{current=#st{x=Xs}}=_Vst) when is_integer(X) -> - case gb_trees:lookup(X, Xs) of - {value,#ms{}=Ctx} -> Ctx; - {value,{fragile,#ms{}=Ctx}} -> Ctx; - _ -> error({no_bsm_context,Reg}) +bsm_get_context({Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y-> + case get_movable_term_type(Reg, Vst) of + #ms{}=Ctx -> Ctx; + _ -> error({no_bsm_context,Reg}) end; -bsm_get_context(Reg, _) -> error({bad_source,Reg}). +bsm_get_context(Reg, _) -> + error({bad_source,Reg}). bsm_save(Reg, {atom,start}, Vst) -> %% Save point refering to where the match started. @@ -1141,7 +1523,7 @@ bsm_save(Reg, SavePoint, Vst) -> case bsm_get_context(Reg, Vst) of #ms{valid=Bits,slots=Slots}=Ctxt0 when SavePoint < Slots -> Ctx = Ctxt0#ms{valid=Bits bor (1 bsl SavePoint),slots=Slots}, - set_type_reg(Ctx, Reg, Vst); + override_type(Ctx, Reg, Vst); _ -> error({illegal_save,SavePoint}) end. @@ -1160,104 +1542,380 @@ bsm_restore(Reg, SavePoint, Vst) -> _ -> error({illegal_restore,SavePoint,range}) end. +validate_select_val(_Fail, _Choices, _Src, #vst{current=none}=Vst) -> + %% We've already branched on all of Src's possible values, so we know we + %% can't reach the fail label or any of the remaining choices. + Vst; +validate_select_val(Fail, [Val,{f,L}|T], Src, Vst0) -> + Vst = branch(L, Vst0, + fun(BranchVst) -> + update_eq_types(Src, Val, BranchVst) + end, + fun(FailVst) -> + update_ne_types(Src, Val, FailVst) + end), + validate_select_val(Fail, T, Src, Vst); +validate_select_val(Fail, [], _, Vst) -> + branch(Fail, Vst, + fun(SuccVst) -> + %% The next instruction is never executed. + kill_state(SuccVst) + end). + +validate_select_tuple_arity(_Fail, _Choices, _Src, #vst{current=none}=Vst) -> + %% We've already branched on all of Src's possible values, so we know we + %% can't reach the fail label or any of the remaining choices. + Vst; +validate_select_tuple_arity(Fail, [Arity,{f,L}|T], Tuple, Vst0) -> + Type = {tuple, Arity, #{}}, + Vst = branch(L, Vst0, + fun(BranchVst) -> + update_type(fun meet/2, Type, Tuple, BranchVst) + end, + fun(FailVst) -> + update_type(fun subtract/2, Type, Tuple, FailVst) + end), + validate_select_tuple_arity(Fail, T, Tuple, Vst); +validate_select_tuple_arity(Fail, [], _, #vst{}=Vst) -> + branch(Fail, Vst, + fun(SuccVst) -> + %% The next instruction is never executed. + kill_state(SuccVst) + end). + +infer_types({Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y -> + infer_types(get_reg_vref(Reg, Vst), Vst); +infer_types(#value_ref{}=Ref, #vst{current=#st{vs=Vs}}) -> + case Vs of + #{ Ref := Entry } -> infer_types_1(Entry); + #{} -> fun(_, S) -> S end + end; +infer_types(_, #vst{}) -> + fun(_, S) -> S end. + +infer_types_1(#value{op={bif,'=:='},args=[LHS,RHS]}) -> + fun({atom,true}, S) -> + %% Either side might contain something worth inferring, so we need + %% to check them both. + Infer_L = infer_types(RHS, S), + Infer_R = infer_types(LHS, S), + Infer_R(RHS, Infer_L(LHS, S)); + (_, S) -> S + end; +infer_types_1(#value{op={bif,element},args=[{integer,Index}=Key,Tuple]}) -> + fun(Val, S) -> + case is_value_alive(Tuple, S) of + true -> + Type = {tuple,[Index], #{ Key => get_term_type(Val, S) }}, + update_type(fun meet/2, Type, Tuple, S); + false -> + S + end + end; +infer_types_1(#value{op={bif,is_atom},args=[Src]}) -> + infer_type_test_bif({atom,[]}, Src); +infer_types_1(#value{op={bif,is_boolean},args=[Src]}) -> + infer_type_test_bif(bool, Src); +infer_types_1(#value{op={bif,is_binary},args=[Src]}) -> + infer_type_test_bif(binary, Src); +infer_types_1(#value{op={bif,is_bitstring},args=[Src]}) -> + infer_type_test_bif(binary, Src); +infer_types_1(#value{op={bif,is_float},args=[Src]}) -> + infer_type_test_bif(float, Src); +infer_types_1(#value{op={bif,is_integer},args=[Src]}) -> + infer_type_test_bif({integer,{}}, Src); +infer_types_1(#value{op={bif,is_list},args=[Src]}) -> + infer_type_test_bif(list, Src); +infer_types_1(#value{op={bif,is_map},args=[Src]}) -> + infer_type_test_bif(map, Src); +infer_types_1(#value{op={bif,is_number},args=[Src]}) -> + infer_type_test_bif(number, Src); +infer_types_1(#value{op={bif,is_tuple},args=[Src]}) -> + infer_type_test_bif({tuple,[0],#{}}, Src); +infer_types_1(#value{op={bif,tuple_size}, args=[Tuple]}) -> + fun({integer,Arity}, S) -> + case is_value_alive(Tuple, S) of + true -> update_type(fun meet/2, {tuple,Arity,#{}}, Tuple, S); + false -> S + end; + (_, S) -> S + end; +infer_types_1(_) -> + fun(_, S) -> S end. + +infer_type_test_bif(Type, Src) -> + fun({atom,true}, S) -> + case is_value_alive(Src, S) of + true -> update_type(fun meet/2, Type, Src, S); + false -> S + end; + (_, S) -> + S + end. + %%% %%% Keeping track of types. %%% -set_type(Type, {x,_}=Reg, Vst) -> set_type_reg(Type, Reg, Vst); -set_type(Type, {y,_}=Reg, Vst) -> set_type_y(Type, Reg, Vst); -set_type(_, _, #vst{}=Vst) -> Vst. - -set_type_reg(Type, Src, Dst, Vst) -> - case get_term_type_1(Src, Vst) of - {fragile,_} -> - set_type_reg(make_fragile(Type), Dst, Vst); +%% Assigns Src to Dst and marks them as aliasing each other. +assign({y,_}=Src, {y,_}=Dst, Vst) -> + %% The stack trimming optimization may generate a move from an initialized + %% but unassigned Y register to another Y register. + case get_raw_type(Src, Vst) of + initialized -> create_tag(initialized, init, [], Dst, Vst); + _ -> assign_1(Src, Dst, Vst) + end; +assign({Kind,_}=Src, Dst, Vst) when Kind =:= x; Kind =:= y -> + assign_1(Src, Dst, Vst); +assign(Literal, Dst, Vst) -> + Type = get_literal_type(Literal), + create_term(Type, move, [Literal], Dst, Vst). + +%% Creates a special tag value that isn't a regular term, such as 'initialized' +%% or 'catchtag' +create_tag(Tag, _Op, _Ss, {y,_}=Dst, #vst{current=#st{ys=Ys0}=St0}=Vst) -> + case maps:get(Dst, Ys0, uninitialized) of + {catchtag,_}=Prev -> + error(Prev); + {trytag,_}=Prev -> + error(Prev); _ -> - set_type_reg(Type, Dst, Vst) + check_try_catch_tags(Tag, Dst, Vst), + Ys = Ys0#{ Dst => Tag }, + St = St0#st{ys=Ys}, + remove_fragility(Dst, Vst#vst{current=St}) + end; +create_tag(_Tag, _Op, _Ss, Dst, _Vst) -> + error({invalid_tag_register, Dst}). + +%% Wipes a special tag, leaving the register initialized but empty. +kill_tag({y,_}=Reg, #vst{current=#st{ys=Ys0}=St0}=Vst) -> + _ = get_tag_type(Reg, Vst), %Assertion. + Ys = Ys0#{ Reg => initialized }, + Vst#vst{current=St0#st{ys=Ys}}. + +%% Creates a completely new term with the given type. +create_term(Type, Op, Ss0, Dst, Vst0) -> + create_term(Type, Op, Ss0, Dst, Vst0, Vst0). + +%% As create_term/4, but uses the incoming Vst for argument resolution in +%% case x-regs have been pruned and the sources can no longer be found. +create_term(Type, Op, Ss0, Dst, Vst0, OrigVst) -> + {Ref, Vst1} = new_value(Type, Op, resolve_args(Ss0, OrigVst), Vst0), + Vst = remove_fragility(Dst, Vst1), + set_reg_vref(Ref, Dst, Vst). + +%% Extracts a term from Ss, propagating fragility. +extract_term(Type, Op, Ss0, Dst, Vst0) -> + extract_term(Type, Op, Ss0, Dst, Vst0, Vst0). + +%% As extract_term/4, but uses the incoming Vst for argument resolution in +%% case x-regs have been pruned and the sources can no longer be found. +extract_term(Type, Op, Ss0, Dst, Vst0, OrigVst) -> + {Ref, Vst1} = new_value(Type, Op, resolve_args(Ss0, OrigVst), Vst0), + Vst = propagate_fragility(Dst, Ss0, Vst1), + set_reg_vref(Ref, Dst, Vst). + +%% Translates instruction arguments into the argument() type, decoupling them +%% from their registers, allowing us to infer their types after they've been +%% clobbered or moved. +resolve_args([{Kind,_}=Src | Args], Vst) when Kind =:= x; Kind =:= y -> + [get_reg_vref(Src, Vst) | resolve_args(Args, Vst)]; +resolve_args([Lit | Args], Vst) -> + assert_literal(Lit), + [Lit | resolve_args(Args, Vst)]; +resolve_args([], _Vst) -> + []. + +%% Overrides the type of Reg. This is ugly but a necessity for certain +%% destructive operations. +override_type(Type, Reg, Vst) -> + update_type(fun(_, T) -> T end, Type, Reg, Vst). + +%% This is used when linear code finds out more and more information about a +%% type, so that the type gets more specialized. +update_type(Merge, With, #value_ref{}=Ref, Vst) -> + %% If the old type can't be merged with the new one, the type information + %% is inconsistent and we know that some instructions will never be + %% executed at run-time. For example: + %% + %% {test,is_list,Fail,[Reg]}. + %% {test,is_tuple,Fail,[Reg]}. + %% {test,test_arity,Fail,[Reg,5]}. + %% + %% Note that the test_arity instruction can never be reached, so we need to + %% kill the state to avoid raising an error when we encounter it. + %% + %% Simply returning `kill_state(Vst)` is unsafe however as we might be in + %% the middle of an instruction, and altering the rest of the validator + %% (eg. prune_x_regs/2) to no-op on dead states is prone to error. + %% + %% We therefore throw a 'type_conflict' error instead, which causes + %% validation to fail unless we're in a context where such errors can be + %% handled, such as in a branch handler. + Current = get_raw_type(Ref, Vst), + case Merge(Current, With) of + none -> throw({type_conflict, Current, With}); + Type -> set_type(Type, Ref, Vst) + end; +update_type(Merge, With, {Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y -> + update_type(Merge, With, get_reg_vref(Reg, Vst), Vst); +update_type(Merge, With, Literal, Vst) -> + assert_literal(Literal), + %% Literals always retain their type, but we still need to bail on type + %% conflicts. + case Merge(Literal, With) of + none -> throw({type_conflict, Literal, With}); + _Type -> Vst end. -set_type_reg(Type, {x,_}=Reg, Vst) -> - set_type_x(Type, Reg, Vst); -set_type_reg(Type, Reg, Vst) -> - set_type_y(Type, Reg, Vst). - -set_type_x(Type, {x,X}=Reg, #vst{current=#st{x=Xs0}=St}=Vst) - when is_integer(X), 0 =< X -> - check_limit(Reg), - Xs = case gb_trees:lookup(X, Xs0) of - none -> - gb_trees:insert(X, Type, Xs0); - {value,{fragile,_}} -> - gb_trees:update(X, make_fragile(Type), Xs0); - {value,_} -> - gb_trees:update(X, Type, Xs0) - end, - Vst#vst{current=St#st{x=Xs}}; -set_type_x(Type, Reg, #vst{}) -> - error({invalid_store,Reg,Type}). - -set_type_y(Type, {y,Y}=Reg, #vst{current=#st{y=Ys0}=St}=Vst) - when is_integer(Y), 0 =< Y -> - check_limit(Reg), - Ys = case gb_trees:lookup(Y, Ys0) of - none -> - error({invalid_store,Reg,Type}); - {value,{catchtag,_}=Tag} -> - error(Tag); - {value,{trytag,_}=Tag} -> - error(Tag); - {value,_} -> - gb_trees:update(Y, Type, Ys0) - end, - check_try_catch_tags(Type, Y, Ys0), - Vst#vst{current=St#st{y=Ys}}; -set_type_y(Type, Reg, #vst{}) -> error({invalid_store,Reg,Type}). - -make_fragile({fragile,_}=Type) -> Type; -make_fragile(Type) -> {fragile,Type}. - -set_catch_end({y,Y}, #vst{current=#st{y=Ys0}=St}=Vst) -> - Ys = gb_trees:update(Y, initialized, Ys0), - Vst#vst{current=St#st{y=Ys}}. - -check_try_catch_tags(Type, LastY, Ys) -> - case is_try_catch_tag(Type) of - false -> - ok; - true -> - %% Every catch or try/catch must use a lower Y register - %% number than any enclosing catch or try/catch. That will - %% ensure that when the stack is scanned when an - %% exception occurs, the innermost try/catch tag is found - %% first. - Bad = [{{y,Y},Tag} || {Y,Tag} <- gb_trees:to_list(Ys), - Y < LastY, is_try_catch_tag(Tag)], - case Bad of - [] -> - ok; - [_|_] -> - error({bad_try_catch_nesting,{y,LastY},Bad}) - end +update_ne_types(LHS, RHS, Vst) -> + %% While updating types on equality is fairly straightforward, inequality + %% is a bit trickier since all we know is that the *value* of LHS differs + %% from RHS, so we can't blindly subtract their types. + %% + %% Consider `number =/= {integer,[]}`; all we know is that LHS isn't equal + %% to some *specific integer* of unknown value, and if we were to subtract + %% {integer,[]} we would erroneously infer that the new type is {float,[]}. + %% + %% Therefore, we only subtract when we know that RHS has a specific value. + RType = get_term_type(RHS, Vst), + case is_literal(RType) of + true -> update_type(fun subtract/2, RType, LHS, Vst); + false -> Vst end. -is_try_catch_tag({catchtag,_}) -> true; -is_try_catch_tag({trytag,_}) -> true; -is_try_catch_tag(_) -> false. +update_eq_types(LHS, RHS, Vst0) -> + %% Either side might contain something worth inferring, so we need + %% to check them both. + Infer_L = infer_types(RHS, Vst0), + Infer_R = infer_types(LHS, Vst0), + Vst1 = Infer_R(RHS, Infer_L(LHS, Vst0)), -is_reg_defined({x,_}=Reg, Vst) -> is_type_defined_x(Reg, Vst); -is_reg_defined({y,_}=Reg, Vst) -> is_type_defined_y(Reg, Vst); -is_reg_defined(V, #vst{}) -> error({not_a_register, V}). + T1 = get_term_type(LHS, Vst1), + T2 = get_term_type(RHS, Vst1), + + Vst = update_type(fun meet/2, T2, LHS, Vst1), + update_type(fun meet/2, T1, RHS, Vst). + +%% Helper functions for the above. + +assign_1(Src, Dst, Vst0) -> + assert_movable(Src, Vst0), + Vst = propagate_fragility(Dst, [Src], Vst0), + set_reg_vref(get_reg_vref(Src, Vst), Dst, Vst). + +set_reg_vref(Ref, {x,_}=Dst, Vst) -> + check_limit(Dst), + #vst{current=#st{xs=Xs0}=St0} = Vst, + St = St0#st{xs=Xs0#{ Dst => Ref }}, + Vst#vst{current=St}; +set_reg_vref(Ref, {y,_}=Dst, #vst{current=#st{ys=Ys0}=St0} = Vst) -> + check_limit(Dst), + case Ys0 of + #{ Dst := {catchtag,_}=Tag } -> + error(Tag); + #{ Dst := {trytag,_}=Tag } -> + error(Tag); + #{ Dst := _ } -> + St = St0#st{ys=Ys0#{ Dst => Ref }}, + Vst#vst{current=St}; + #{} -> + %% Storing into a non-existent Y register means that we haven't set + %% up a (sufficiently large) stack. + error({invalid_store, Dst}) + end. -is_type_defined_x({x,X}, #vst{current=#st{x=Xs}}) -> - gb_trees:is_defined(X,Xs). +get_reg_vref({x,_}=Src, #vst{current=#st{xs=Xs}}) -> + check_limit(Src), + case Xs of + #{ Src := #value_ref{}=Ref } -> + Ref; + #{} -> + error({uninitialized_reg, Src}) + end; +get_reg_vref({y,_}=Src, #vst{current=#st{ys=Ys}}) -> + check_limit(Src), + case Ys of + #{ Src := #value_ref{}=Ref } -> + Ref; + #{ Src := initialized } -> + error({unassigned, Src}); + #{ Src := Tag } when Tag =/= uninitialized -> + error(Tag); + #{} -> + error({uninitialized_reg, Src}) + end. + +set_type(Type, #value_ref{}=Ref, #vst{current=#st{vs=Vs0}=St}=Vst) -> + case Vs0 of + #{ Ref := #value{}=Entry } -> + Vs = Vs0#{ Ref => Entry#value{type=Type} }, + Vst#vst{current=St#st{vs=Vs}}; + #{} -> + %% Dead references may happen during type inference and are not an + %% error in and of themselves. If a problem were to arise from this + %% it'll explode elsewhere. + Vst + end. + +new_value(Type, Op, Ss, #vst{current=#st{vs=Vs0}=St,ref_ctr=Counter}=Vst) -> + Ref = #value_ref{id=Counter}, + Vs = Vs0#{ Ref => #value{op=Op,args=Ss,type=Type} }, + + {Ref, Vst#vst{current=St#st{vs=Vs},ref_ctr=Counter+1}}. -is_type_defined_y({y,Y}, #vst{current=#st{y=Ys}}) -> - gb_trees:is_defined(Y,Ys). +kill_catch_tag(Reg, #vst{current=#st{ct=[Fail|Fails]}=St}=Vst0) -> + Vst = Vst0#vst{current=St#st{ct=Fails,fls=undefined}}, + {_, Fail} = get_tag_type(Reg, Vst), %Assertion. + kill_tag(Reg, Vst). + +check_try_catch_tags(Type, {y,N}=Reg, Vst) -> + %% Every catch or try/catch must use a lower Y register number than any + %% enclosing catch or try/catch. That will ensure that when the stack is + %% scanned when an exception occurs, the innermost try/catch tag is found + %% first. + case is_try_catch_tag(Type) of + true -> + case collect_try_catch_tags(N - 1, Vst, []) of + [_|_]=Bad -> error({bad_try_catch_nesting, Reg, Bad}); + [] -> ok + end; + false -> + ok + end. + +is_reg_defined({x,_}=Reg, #vst{current=#st{xs=Xs}}) -> is_map_key(Reg, Xs); +is_reg_defined({y,_}=Reg, #vst{current=#st{ys=Ys}}) -> is_map_key(Reg, Ys); +is_reg_defined(V, #vst{}) -> error({not_a_register, V}). assert_term(Src, Vst) -> - get_term_type(Src, Vst), + _ = get_term_type(Src, Vst), + ok. + +assert_movable(Src, Vst) -> + _ = get_movable_term_type(Src, Vst), ok. +assert_literal(Src) -> + case is_literal(Src) of + true -> ok; + false -> error({literal_required,Src}) + end. + +assert_not_literal(Src) -> + case is_literal(Src) of + true -> error({literal_not_allowed,Src}); + false -> ok + end. + +is_literal(nil) -> true; +is_literal({atom,A}) when is_atom(A) -> true; +is_literal({float,F}) when is_float(F) -> true; +is_literal({integer,I}) when is_integer(I) -> true; +is_literal({literal,_L}) -> true; +is_literal(_) -> false. + %% The possible types. %% %% First non-term types: @@ -1276,10 +1934,10 @@ assert_term(Src, Vst) -> %% used by the catch instructions; NOT safe to use in other %% instructions. %% -%% exception Can only be used as a type returned by return_type/2 -%% (which gives the type of the value returned by a BIF). -%% Thus 'exception' is never stored as type descriptor -%% for a register. +%% exception Can only be used as a type returned by +%% call_return_type/2 (which gives the type of the value +%% returned by a call). Thus 'exception' is never stored +%% as type descriptor for a register. %% %% #ms{} A match context for bit syntax matching. We do allow %% it to moved/to from stack, but otherwise it must only @@ -1290,17 +1948,22 @@ assert_term(Src, Vst) -> %% %% term Any valid Erlang (but not of the special types above). %% +%% binary Binary or bitstring. +%% %% bool The atom 'true' or the atom 'false'. %% %% cons Cons cell: [_|_] %% %% nil Empty list: [] %% -%% {tuple,[Sz]} Tuple. An element has been accessed using -%% element/2 or setelement/3 so that it is known that -%% the type is a tuple of size at least Sz. +%% list List: [] or [_|_] +%% +%% {tuple,[Sz],Es} Tuple. An element has been accessed using +%% element/2 or setelement/3 so that it is known that +%% the type is a tuple of size at least Sz. Es is a map +%% containing known types by tuple index. %% -%% {tuple,Sz} Tuple. A test_arity instruction has been seen +%% {tuple,Sz,Es} Tuple. A test_arity instruction has been seen %% so that it is known that the size is exactly Sz. %% %% {atom,[]} Atom. @@ -1316,35 +1979,214 @@ assert_term(Src, Vst) -> %% %% map Map. %% +%% none A conflict in types. There will be an exception at runtime. %% -%% -%% FRAGILITY -%% --------- -%% -%% The loop_rec/2 instruction may return a reference to a term that is -%% not part of the root set. That term or any part of it must not be -%% included in a garbage collection. Therefore, the term (or any part -%% of it) must not be stored in an Y register. -%% -%% Such terms are wrapped in a {fragile,Type} tuple, where Type is one -%% of the types described above. -assert_type(WantedType, Term, Vst) -> - case get_term_type(Term, Vst) of - {fragile,Type} -> - assert_type(WantedType, Type); - Type -> - assert_type(WantedType, Type) +%% join(Type1, Type2) -> Type +%% Return the most specific type possible. +join(Same, Same) -> + Same; +join(none, Other) -> + Other; +join(Other, none) -> + Other; +join({literal,_}=T1, T2) -> + join_literal(T1, T2); +join(T1, {literal,_}=T2) -> + join_literal(T2, T1); +join({tuple,Size,EsA}, {tuple,Size,EsB}) -> + Es = join_tuple_elements(tuple_sz(Size), EsA, EsB), + {tuple, Size, Es}; +join({tuple,A,EsA}, {tuple,B,EsB}) -> + Size = min(tuple_sz(A), tuple_sz(B)), + Es = join_tuple_elements(Size, EsA, EsB), + {tuple, [Size], Es}; +join({Type,A}, {Type,B}) + when Type =:= atom; Type =:= integer; Type =:= float -> + if A =:= B -> {Type,A}; + true -> {Type,[]} + end; +join({Type,_}, number) + when Type =:= integer; Type =:= float -> + number; +join(number, {Type,_}) + when Type =:= integer; Type =:= float -> + number; +join({integer,_}, {float,_}) -> + number; +join({float,_}, {integer,_}) -> + number; +join(bool, {atom,A}) -> + join_bool(A); +join({atom,A}, bool) -> + join_bool(A); +join({atom,A}, {atom,B}) when is_boolean(A), is_boolean(B) -> + bool; +join({atom,_}, {atom,_}) -> + {atom,[]}; +join(#ms{id=Id1,valid=B1,slots=Slots1}, + #ms{id=Id2,valid=B2,slots=Slots2}) -> + Id = if + Id1 =:= Id2 -> Id1; + true -> make_ref() + end, + #ms{id=Id,valid=B1 band B2,slots=min(Slots1, Slots2)}; +join(T1, T2) when T1 =/= T2 -> + %% We've exhaused all other options, so the type must either be a list or + %% a 'term'. + join_list(T1, T2). + +join_tuple_elements(Limit, EsA, EsB) -> + Es0 = join_elements(EsA, EsB), + maps:filter(fun({integer,Index}, _Type) -> Index =< Limit end, Es0). + +join_elements(Es1, Es2) -> + Keys = if + map_size(Es1) =< map_size(Es2) -> maps:keys(Es1); + map_size(Es1) > map_size(Es2) -> maps:keys(Es2) + end, + join_elements_1(Keys, Es1, Es2, #{}). + +join_elements_1([Key | Keys], Es1, Es2, Acc0) -> + Type = case {Es1, Es2} of + {#{ Key := Same }, #{ Key := Same }} -> Same; + {#{ Key := Type1 }, #{ Key := Type2 }} -> join(Type1, Type2); + {#{}, #{}} -> term + end, + Acc = set_element_type(Key, Type, Acc0), + join_elements_1(Keys, Es1, Es2, Acc); +join_elements_1([], _Es1, _Es2, Acc) -> + Acc. + +%% Joins types of literals; note that the left argument must either be a +%% literal or exactly equal to the second argument. +join_literal(Same, Same) -> + Same; +join_literal({literal,_}=Lit, T) -> + join_literal(T, get_literal_type(Lit)); +join_literal(T1, T2) -> + %% We're done extracting the types, try merging them again. + join(T1, T2). + +join_list(nil, cons) -> list; +join_list(nil, list) -> list; +join_list(cons, list) -> list; +join_list(T, nil) -> join_list(nil, T); +join_list(T, cons) -> join_list(cons, T); +join_list(_, _) -> + %% Not a list, so it must be a term. + term. + +join_bool([]) -> {atom,[]}; +join_bool(true) -> bool; +join_bool(false) -> bool; +join_bool(_) -> {atom,[]}. + +%% meet(Type1, Type2) -> Type +%% Return the meet of two types. The meet is a more specific type. +%% It will be 'none' if the types are in conflict. + +meet(Same, Same) -> + Same; +meet(term, Other) -> + Other; +meet(Other, term) -> + Other; +meet(#ms{}, binary) -> + #ms{}; +meet(binary, #ms{}) -> + #ms{}; +meet({literal,_}, {literal,_}) -> + none; +meet(T1, {literal,_}=T2) -> + meet(T2, T1); +meet({literal,_}=T1, T2) -> + case meet(get_literal_type(T1), T2) of + none -> none; + _ -> T1 + end; +meet(T1, T2) -> + case {erlang:min(T1, T2),erlang:max(T1, T2)} of + {{atom,_}=A,{atom,[]}} -> A; + {bool,{atom,B}=Atom} when is_boolean(B) -> Atom; + {bool,{atom,[]}} -> bool; + {cons,list} -> cons; + {{float,_}=T,{float,[]}} -> T; + {{integer,_}=T,{integer,[]}} -> T; + {list,nil} -> nil; + {number,{integer,_}=T} -> T; + {number,{float,_}=T} -> T; + {{tuple,Size1,Es1},{tuple,Size2,Es2}} -> + Es = meet_elements(Es1, Es2), + case {Size1,Size2,Es} of + {_, _, none} -> + none; + {[Sz1],[Sz2],_} -> + Sz = erlang:max(Sz1, Sz2), + assert_tuple_elements(Sz, Es), + {tuple,[Sz],Es}; + {Sz1,[Sz2],_} when Sz2 =< Sz1 -> + assert_tuple_elements(Sz1, Es), + {tuple,Sz1,Es}; + {Sz,Sz,_} -> + assert_tuple_elements(Sz, Es), + {tuple,Sz,Es}; + {_,_,_} -> + none + end; + {_,_} -> none end. +meet_elements(Es1, Es2) -> + Keys = maps:keys(Es1) ++ maps:keys(Es2), + meet_elements_1(Keys, Es1, Es2, #{}). + +meet_elements_1([Key | Keys], Es1, Es2, Acc) -> + case {Es1, Es2} of + {#{ Key := Type1 }, #{ Key := Type2 }} -> + case meet(Type1, Type2) of + none -> none; + Type -> meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type }) + end; + {#{ Key := Type1 }, _} -> + meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type1 }); + {_, #{ Key := Type2 }} -> + meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type2 }) + end; +meet_elements_1([], _Es1, _Es2, Acc) -> + Acc. + +%% No tuple elements may have an index above the known size. +assert_tuple_elements(Limit, Es) -> + true = maps:fold(fun({integer,Index}, _T, true) -> + Index =< Limit + end, true, Es). %Assertion. + +%% subtract(Type1, Type2) -> Type +%% Subtract Type2 from Type2. Example: +%% subtract(list, nil) -> cons + +subtract(Same, Same) -> none; +subtract(list, nil) -> cons; +subtract(list, cons) -> nil; +subtract(number, {integer,[]}) -> {float,[]}; +subtract(number, {float,[]}) -> {integer,[]}; +subtract(bool, {atom,false}) -> {atom, true}; +subtract(bool, {atom,true}) -> {atom, false}; +subtract(Type, _) -> Type. + +assert_type(WantedType, Term, Vst) -> + Type = get_term_type(Term, Vst), + assert_type(WantedType, Type). + assert_type(Correct, Correct) -> ok; assert_type(float, {float,_}) -> ok; -assert_type(tuple, {tuple,_}) -> ok; +assert_type(tuple, {tuple,_,_}) -> ok; assert_type(tuple, {literal,Tuple}) when is_tuple(Tuple) -> ok; -assert_type({tuple_element,I}, {tuple,[Sz]}) +assert_type({tuple_element,I}, {tuple,[Sz],_}) when 1 =< I, I =< Sz -> ok; -assert_type({tuple_element,I}, {tuple,Sz}) +assert_type({tuple_element,I}, {tuple,Sz,_}) when is_integer(Sz), 1 =< I, I =< Sz -> ok; assert_type({tuple_element,I}, {literal,Lit}) when I =< tuple_size(Lit) -> @@ -1354,141 +2196,300 @@ assert_type(cons, {literal,[_|_]}) -> assert_type(Needed, Actual) -> error({bad_type,{needed,Needed},{actual,Actual}}). -%% upgrade_tuple_type(NewTupleType, OldType) -> TupleType. -%% upgrade_tuple_type/2 is used when linear code finds out more and -%% more information about a tuple type, so that the type gets more -%% specialized. If OldType is not a tuple type, the type information -%% is inconsistent, and we know that some instructions will never -%% be executed at run-time. - -upgrade_tuple_type(NewType, {fragile,OldType}) -> - make_fragile(upgrade_tuple_type_1(NewType, OldType)); -upgrade_tuple_type(NewType, OldType) -> - upgrade_tuple_type_1(NewType, OldType). - -upgrade_tuple_type_1({tuple,[Sz]}, {tuple,[OldSz]}=T) when Sz < OldSz -> - %% The old type has a higher value for the least tuple size. - T; -upgrade_tuple_type_1({tuple,[Sz]}, {tuple,OldSz}=T) - when is_integer(Sz), is_integer(OldSz), Sz =< OldSz -> - %% The old size is exact, and the new size is smaller than the old size. - T; -upgrade_tuple_type_1({tuple,_}=T, _) -> - %% The new type information is exact or has a higher value for - %% the least tuple size. - %% Note that inconsistencies are also handled in this - %% clause, e.g. if the old type was an integer or a tuple accessed - %% outside its size; inconsistences will generally cause an exception - %% at run-time but are safe from our point of view. - T. +get_element_type(Key, Src, Vst) -> + get_element_type_1(Key, get_term_type(Src, Vst)). + +get_element_type_1({integer,_}=Key, {tuple,_Sz,Es}) -> + case Es of + #{ Key := Type } -> Type; + #{} -> term + end; +get_element_type_1(_Index, _Type) -> + term. + +set_element_type(_Key, none, Es) -> + Es; +set_element_type(Key, term, Es) -> + maps:remove(Key, Es); +set_element_type(Key, Type, Es) -> + Es#{ Key => Type }. get_tuple_size({integer,[]}) -> 0; get_tuple_size({integer,Sz}) -> Sz; get_tuple_size(_) -> 0. validate_src(Ss, Vst) when is_list(Ss) -> - foreach(fun(S) -> get_term_type(S, Vst) end, Ss). + _ = [assert_term(S, Vst) || S <- Ss], + ok. -%% get_move_term_type(Src, ValidatorState) -> Type +%% get_term_type(Src, ValidatorState) -> Type %% Get the type of the source Src. The returned type Type will be -%% a standard Erlang type (no catch/try tags). Match contexts are OK. +%% a standard Erlang type (no catch/try tags or match contexts). -get_move_term_type(Src, Vst) -> - case get_term_type_1(Src, Vst) of - initialized -> error({unassigned,Src}); - {catchtag,_} -> error({catchtag,Src}); - {trytag,_} -> error({trytag,Src}); - tuple_in_progress -> error({tuple_in_progress,Src}); - Type -> Type +get_term_type(Src, Vst) -> + case get_movable_term_type(Src, Vst) of + #ms{} -> error({match_context,Src}); + Type -> Type end. -%% get_term_type(Src, ValidatorState) -> Type +%% get_movable_term_type(Src, ValidatorState) -> Type %% Get the type of the source Src. The returned type Type will be -%% a standard Erlang type (no catch/try tags or match contexts). +%% a standard Erlang type (no catch/try tags). Match contexts are OK. -get_term_type(Src, Vst) -> - case get_move_term_type(Src, Vst) of - #ms{} -> error({match_context,Src}); - Type -> Type +get_movable_term_type(Src, Vst) -> + case get_raw_type(Src, Vst) of + initialized -> error({unassigned,Src}); + uninitialized -> error({uninitialized_reg,Src}); + {catchtag,_} -> error({catchtag,Src}); + {trytag,_} -> error({trytag,Src}); + tuple_in_progress -> error({tuple_in_progress,Src}); + {literal,_}=Lit -> get_literal_type(Lit); + Type -> Type end. -%% get_special_y_type(Src, ValidatorState) -> Type -%% Return the type for the Y register without doing any validity checks. - -get_special_y_type({y,_}=Reg, Vst) -> get_term_type_1(Reg, Vst); -get_special_y_type(Src, _) -> error({source_not_y_reg,Src}). - -get_term_type_1(nil=T, _) -> T; -get_term_type_1({atom,A}=T, _) when is_atom(A) -> T; -get_term_type_1({float,F}=T, _) when is_float(F) -> T; -get_term_type_1({integer,I}=T, _) when is_integer(I) -> T; -get_term_type_1({literal,Map}, _) when is_map(Map) -> map; -get_term_type_1({literal,_}=T, _) -> T; -get_term_type_1({x,X}=Reg, #vst{current=#st{x=Xs}}) when is_integer(X) -> - case gb_trees:lookup(X, Xs) of - {value,Type} -> Type; - none -> error({uninitialized_reg,Reg}) +%% get_tag_type(Src, ValidatorState) -> Type +%% Return the tag type of a Y register, erroring out if it contains a term. + +get_tag_type({y,_}=Src, Vst) -> + case get_raw_type(Src, Vst) of + {catchtag, _}=Tag -> Tag; + {trytag, _}=Tag -> Tag; + uninitialized=Tag -> Tag; + initialized=Tag -> Tag; + Other -> error({invalid_tag,Src,Other}) end; -get_term_type_1({y,Y}=Reg, #vst{current=#st{y=Ys}}) when is_integer(Y) -> - case gb_trees:lookup(Y, Ys) of - none -> error({uninitialized_reg,Reg}); - {value,uninitialized} -> error({uninitialized_reg,Reg}); - {value,Type} -> Type +get_tag_type(Src, _) -> + error({invalid_tag_register,Src}). + +%% get_raw_type(Src, ValidatorState) -> Type +%% Return the type of a register without doing any validity checks or +%% conversions. +get_raw_type({x,X}=Src, #vst{current=#st{xs=Xs}}=Vst) when is_integer(X) -> + check_limit(Src), + case Xs of + #{ Src := #value_ref{}=Ref } -> get_raw_type(Ref, Vst); + #{} -> uninitialized end; -get_term_type_1(Src, _) -> error({bad_source,Src}). - +get_raw_type({y,Y}=Src, #vst{current=#st{ys=Ys}}=Vst) when is_integer(Y) -> + check_limit(Src), + case Ys of + #{ Src := #value_ref{}=Ref } -> get_raw_type(Ref, Vst); + #{ Src := Tag } -> Tag; + #{} -> uninitialized + end; +get_raw_type(#value_ref{}=Ref, #vst{current=#st{vs=Vs}}) -> + case Vs of + #{ Ref := #value{type=Type} } -> Type; + #{} -> none + end; +get_raw_type(Src, #vst{}) -> + get_literal_type(Src). + +is_value_alive(#value_ref{}=Ref, #vst{current=#st{vs=Vs}}) -> + is_map_key(Ref, Vs). + +get_literal_type(nil=T) -> T; +get_literal_type({atom,A}=T) when is_atom(A) -> T; +get_literal_type({float,F}=T) when is_float(F) -> T; +get_literal_type({integer,I}=T) when is_integer(I) -> T; +get_literal_type({literal,[_|_]}) -> cons; +get_literal_type({literal,Bitstring}) when is_bitstring(Bitstring) -> binary; +get_literal_type({literal,Map}) when is_map(Map) -> map; +get_literal_type({literal,Tuple}) when is_tuple(Tuple) -> glt_1(Tuple); +get_literal_type({literal,_}) -> term; +get_literal_type(T) -> error({not_literal,T}). + +glt_1([]) -> nil; +glt_1(A) when is_atom(A) -> {atom, A}; +glt_1(F) when is_float(F) -> {float, F}; +glt_1(I) when is_integer(I) -> {integer, I}; +glt_1(T) when is_tuple(T) -> + {Es,_} = foldl(fun(Val, {Es0, Index}) -> + Type = glt_1(Val), + Es = set_element_type({integer,Index}, Type, Es0), + {Es, Index + 1} + end, {#{}, 1}, tuple_to_list(T)), + {tuple, tuple_size(T), Es}; +glt_1(L) -> + {literal, L}. -%% get_literal(Src) -> literal_value(). -get_literal(nil) -> []; -get_literal({atom,A}) when is_atom(A) -> A; -get_literal({float,F}) when is_float(F) -> F; -get_literal({integer,I}) when is_integer(I) -> I; -get_literal({literal,L}) -> L; -get_literal(T) -> error({not_literal,T}). +%%% +%%% Branch tracking +%%% +%% Forks the execution flow, with the provided funs returning the new state of +%% their respective branch; the "fail" fun returns the state where the branch +%% is taken, and the "success" fun returns the state where it's not. +%% +%% If either path is known not to be taken at runtime (eg. due to a type +%% conflict), it will simply be discarded. +-spec branch(Lbl :: label(), + Original :: #vst{}, + FailFun :: BranchFun, + SuccFun :: BranchFun) -> #vst{} when + BranchFun :: fun((#vst{}) -> #vst{}). +branch(Lbl, Vst0, FailFun, SuccFun) -> + #vst{current=St0} = Vst0, + try FailFun(Vst0) of + Vst1 -> + Vst2 = branch_state(Lbl, Vst1), + Vst = Vst2#vst{current=St0}, + try SuccFun(Vst) of + V -> V + catch + {type_conflict, _, _} -> + %% The instruction is guaranteed to fail; kill the state. + kill_state(Vst) + end + catch + {type_conflict, _, _} -> + %% This instruction is guaranteed not to fail, so we run the + %% success branch *without* catching type conflicts to avoid hiding + %% errors in the validator itself; one of the branches must + %% succeed. + SuccFun(Vst0) + end. -branch_arities([], _, #vst{}=Vst) -> Vst; -branch_arities([Sz,{f,L}|T], Tuple, #vst{current=St}=Vst0) - when is_integer(Sz) -> - Vst1 = set_type_reg({tuple,Sz}, Tuple, Vst0), - Vst = branch_state(L, Vst1), - branch_arities(T, Tuple, Vst#vst{current=St}). +%% A shorthand version of branch/4 for when the state is only altered on +%% success. +branch(Fail, Vst, SuccFun) -> + branch(Fail, Vst, fun(V) -> V end, SuccFun). +%% Directly branches off the state. This is an "internal" operation that should +%% be used sparingly. branch_state(0, #vst{}=Vst) -> - %% If the instruction fails, the stack may be scanned - %% looking for a catch tag. Therefore the Y registers - %% must be initialized at this point. + %% If the instruction fails, the stack may be scanned looking for a catch + %% tag. Therefore the Y registers must be initialized at this point. verify_y_init(Vst), Vst; -branch_state(L, #vst{current=St,branched=B}=Vst) -> - Vst#vst{ - branched=case gb_trees:is_defined(L, B) of - false -> - gb_trees:insert(L, St, B); - true -> - MergedSt = merge_states(L, St, B), - gb_trees:update(L, MergedSt, B) - end}. - -%% merge_states/3 is used when there are more than one way to arrive -%% at this point, and the type states for the different paths has -%% to be merged. The type states are downgraded to the least common -%% subset for the subsequent code. - -merge_states(L, St, Branched) when L =/= 0 -> +branch_state(L, #vst{current=St,branched=B,ref_ctr=Counter0}=Vst) -> + case gb_trees:is_defined(L, B) of + true -> + {MergedSt, Counter} = merge_states(L, St, B, Counter0), + Branched = gb_trees:update(L, MergedSt, B), + Vst#vst{branched=Branched,ref_ctr=Counter}; + false -> + Vst#vst{branched=gb_trees:insert(L, St, B)} + end. + +%% merge_states/3 is used when there's more than one way to arrive at a +%% certain point, requiring the states to be merged down to the least +%% common subset for the subsequent code. + +merge_states(L, St, Branched, Counter) when L =/= 0 -> case gb_trees:lookup(L, Branched) of - none -> St; - {value,OtherSt} when St =:= none -> OtherSt; - {value,OtherSt} -> merge_states_1(St, OtherSt) + none -> + {St, Counter}; + {value,OtherSt} when St =:= none -> + {OtherSt, Counter}; + {value,OtherSt} -> + merge_states_1(St, OtherSt, Counter) end. -merge_states_1(#st{x=Xs0,y=Ys0,numy=NumY0,h=H0,ct=Ct0}, - #st{x=Xs1,y=Ys1,numy=NumY1,h=H1,ct=Ct1}) -> - NumY = merge_stk(NumY0, NumY1), - Xs = merge_regs(Xs0, Xs1), - Ys = merge_y_regs(Ys0, Ys1), - Ct = merge_ct(Ct0, Ct1), - #st{x=Xs,y=Ys,numy=NumY,h=min(H0, H1),ct=Ct}. +merge_states_1(#st{xs=XsA,ys=YsA,vs=VsA,fragile=FragA,numy=NumYA,h=HA,ct=CtA}, + #st{xs=XsB,ys=YsB,vs=VsB,fragile=FragB,numy=NumYB,h=HB,ct=CtB}, + Counter0) -> + %% When merging registers we drop all registers that aren't defined in both + %% states, and resolve conflicts by creating new values (similar to phi + %% nodes in SSA). + %% + %% While doing this we build a "merge map" detailing which values need to + %% be kept and which new values need to be created to resolve conflicts. + %% This map is then used to create a new value database where the types of + %% all values have been joined. + {Xs, Merge0, Counter1} = merge_regs(XsA, XsB, #{}, Counter0), + {Ys, Merge, Counter} = merge_regs(YsA, YsB, Merge0, Counter1), + Vs = merge_values(Merge, VsA, VsB), + + Fragile = merge_fragility(FragA, FragB), + NumY = merge_stk(NumYA, NumYB), + Ct = merge_ct(CtA, CtB), + + St = #st{xs=Xs,ys=Ys,vs=Vs,fragile=Fragile,numy=NumY,h=min(HA, HB),ct=Ct}, + {St, Counter}. + +%% Merges the contents of two register maps, returning the updated "merge map" +%% and the new registers. +merge_regs(RsA, RsB, Merge, Counter) -> + Keys = if + map_size(RsA) =< map_size(RsB) -> maps:keys(RsA); + map_size(RsA) > map_size(RsB) -> maps:keys(RsB) + end, + merge_regs_1(Keys, RsA, RsB, #{}, Merge, Counter). + +merge_regs_1([Reg | Keys], RsA, RsB, Regs, Merge0, Counter0) -> + case {RsA, RsB} of + {#{ Reg := #value_ref{}=RefA }, #{ Reg := #value_ref{}=RefB }} -> + {Ref, Merge, Counter} = merge_vrefs(RefA, RefB, Merge0, Counter0), + merge_regs_1(Keys, RsA, RsB, Regs#{ Reg => Ref }, Merge, Counter); + {#{ Reg := TagA }, #{ Reg := TagB }} -> + %% Tags describe the state of the register rather than the value it + %% contains, so if a register contains a tag in one state we have + %% to merge it as a tag regardless of whether the other state says + %% it's a value. + {y, _} = Reg, %Assertion. + merge_regs_1(Keys, RsA, RsB, Regs#{ Reg => merge_tags(TagA,TagB) }, + Merge0, Counter0); + {#{}, #{}} -> + merge_regs_1(Keys, RsA, RsB, Regs, Merge0, Counter0) + end; +merge_regs_1([], _, _, Regs, Merge, Counter) -> + {Regs, Merge, Counter}. + +merge_tags(Same, Same) -> + Same; +merge_tags(uninitialized, _) -> + uninitialized; +merge_tags(_, uninitialized) -> + uninitialized; +merge_tags({catchtag,T0}, {catchtag,T1}) -> + {catchtag, ordsets:from_list(T0 ++ T1)}; +merge_tags({trytag,T0}, {trytag,T1}) -> + {trytag, ordsets:from_list(T0 ++ T1)}; +merge_tags(_A, _B) -> + %% All other combinations leave the register initialized. Errors arising + %% from this will be caught later on. + initialized. + +merge_vrefs(Ref, Ref, Merge, Counter) -> + %% We have two (potentially) different versions of the same value, so we + %% should join their types into the same value. + {Ref, Merge#{ Ref => Ref }, Counter}; +merge_vrefs(RefA, RefB, Merge, Counter) -> + %% We have two different values, so we need to create a new value from + %% their joined type if we haven't already done so. + Key = {RefA, RefB}, + case Merge of + #{ Key := Ref } -> + {Ref, Merge, Counter}; + #{} -> + Ref = #value_ref{id=Counter}, + {Ref, Merge#{ Key => Ref }, Counter + 1} + end. + +merge_values(Merge, VsA, VsB) -> + maps:fold(fun(Spec, New, Acc) -> + merge_values_1(Spec, New, VsA, VsB, Acc) + end, #{}, Merge). + +merge_values_1(Same, Same, VsA, VsB, Acc) -> + %% We're merging different versions of the same value, so it's safe to + %% reuse old entries if the type's unchanged. + #value{type=TypeA}=EntryA = map_get(Same, VsA), + #value{type=TypeB}=EntryB = map_get(Same, VsB), + Entry = case join(TypeA, TypeB) of + TypeA -> EntryA; + TypeB -> EntryB; + JoinedType -> EntryA#value{type=JoinedType} + end, + Acc#{ Same => Entry }; +merge_values_1({RefA, RefB}, New, VsA, VsB, Acc) -> + #value{type=TypeA} = map_get(RefA, VsA), + #value{type=TypeB} = map_get(RefB, VsB), + Acc#{ New => #value{op=join,args=[],type=join(TypeA, TypeB)} }. + +merge_fragility(FragileA, FragileB) -> + cerl_sets:union(FragileA, FragileB). merge_stk(S, S) -> S; merge_stk(_, _) -> undecided. @@ -1501,135 +2502,70 @@ merge_ct_1([C0|Ct0], [C1|Ct1]) -> merge_ct_1([], []) -> []; merge_ct_1(_, _) -> undecided. -merge_regs(Rs0, Rs1) -> - Rs = merge_regs_1(gb_trees:to_list(Rs0), gb_trees:to_list(Rs1)), - gb_trees_from_list(Rs). - -merge_regs_1([Same|Rs1], [Same|Rs2]) -> - [Same|merge_regs_1(Rs1, Rs2)]; -merge_regs_1([{R1,_}|Rs1], [{R2,_}|_]=Rs2) when R1 < R2 -> - merge_regs_1(Rs1, Rs2); -merge_regs_1([{R1,_}|_]=Rs1, [{R2,_}|Rs2]) when R1 > R2 -> - merge_regs_1(Rs1, Rs2); -merge_regs_1([{R,Type1}|Rs1], [{R,Type2}|Rs2]) -> - [{R,merge_types(Type1, Type2)}|merge_regs_1(Rs1, Rs2)]; -merge_regs_1([], []) -> []; -merge_regs_1([], [_|_]) -> []; -merge_regs_1([_|_], []) -> []. - -merge_y_regs(Rs0, Rs1) -> - case {gb_trees:size(Rs0),gb_trees:size(Rs1)} of - {Sz0,Sz1} when Sz0 < Sz1 -> - merge_y_regs_1(Sz0-1, Rs1, Rs0); - {_,Sz1} -> - merge_y_regs_1(Sz1-1, Rs0, Rs1) - end. +tuple_sz([Sz]) -> Sz; +tuple_sz(Sz) -> Sz. -merge_y_regs_1(Y, S, Regs0) when Y >= 0 -> - Type0 = gb_trees:get(Y, Regs0), - case gb_trees:get(Y, S) of - Type0 -> - merge_y_regs_1(Y-1, S, Regs0); - Type1 -> - Type = merge_types(Type0, Type1), - Regs = gb_trees:update(Y, Type, Regs0), - merge_y_regs_1(Y-1, S, Regs) - end; -merge_y_regs_1(_, _, Regs) -> Regs. +verify_y_init(#vst{current=#st{numy=NumY,ys=Ys}}=Vst) when is_integer(NumY) -> + HighestY = maps:fold(fun({y,Y}, _, Acc) -> max(Y, Acc) end, -1, Ys), + true = NumY > HighestY, %Assertion. + verify_y_init_1(NumY - 1, Vst), + ok; +verify_y_init(#vst{current=#st{numy=undecided,ys=Ys}}=Vst) -> + HighestY = maps:fold(fun({y,Y}, _, Acc) -> max(Y, Acc) end, -1, Ys), + verify_y_init_1(HighestY, Vst); +verify_y_init(#vst{}) -> + ok. -%% merge_types(Type1, Type2) -> Type -%% Return the most specific type possible. -%% Note: Type1 must NOT be the same as Type2. -merge_types({fragile,Same}=Type, Same) -> - Type; -merge_types({fragile,T1}, T2) -> - make_fragile(merge_types(T1, T2)); -merge_types(Same, {fragile,Same}=Type) -> - Type; -merge_types(T1, {fragile,T2}) -> - make_fragile(merge_types(T1, T2)); -merge_types(uninitialized=I, _) -> I; -merge_types(_, uninitialized=I) -> I; -merge_types(initialized=I, _) -> I; -merge_types(_, initialized=I) -> I; -merge_types({catchtag,T0},{catchtag,T1}) -> - {catchtag,ordsets:from_list(T0++T1)}; -merge_types({trytag,T0},{trytag,T1}) -> - {trytag,ordsets:from_list(T0++T1)}; -merge_types({tuple,A}, {tuple,B}) -> - {tuple,[min(tuple_sz(A), tuple_sz(B))]}; -merge_types({Type,A}, {Type,B}) - when Type =:= atom; Type =:= integer; Type =:= float -> - if A =:= B -> {Type,A}; - true -> {Type,[]} - end; -merge_types({Type,_}, number) - when Type =:= integer; Type =:= float -> - number; -merge_types(number, {Type,_}) - when Type =:= integer; Type =:= float -> - number; -merge_types(bool, {atom,A}) -> - merge_bool(A); -merge_types({atom,A}, bool) -> - merge_bool(A); -merge_types(#ms{id=Id1,valid=B1,slots=Slots1}, - #ms{id=Id2,valid=B2,slots=Slots2}) -> - Id = if - Id1 =:= Id2 -> Id1; - true -> make_ref() - end, - #ms{id=Id,valid=B1 band B2,slots=min(Slots1, Slots2)}; -merge_types(T1, T2) when T1 =/= T2 -> - %% Too different. All we know is that the type is a 'term'. - term. +verify_y_init_1(-1, _Vst) -> + ok; +verify_y_init_1(Y, Vst) -> + Reg = {y, Y}, + assert_not_fragile(Reg, Vst), + case get_raw_type(Reg, Vst) of + uninitialized -> error({uninitialized_reg,Reg}); + _ -> verify_y_init_1(Y - 1, Vst) + end. -tuple_sz([Sz]) -> Sz; -tuple_sz(Sz) -> Sz. +verify_live(0, _Vst) -> + ok; +verify_live(Live, Vst) when is_integer(Live), 0 < Live, Live =< 1023 -> + verify_live_1(Live - 1, Vst); +verify_live(Live, _Vst) -> + error({bad_number_of_live_regs,Live}). -merge_bool([]) -> {atom,[]}; -merge_bool(true) -> bool; -merge_bool(false) -> bool; -merge_bool(_) -> {atom,[]}. - -verify_y_init(#vst{current=#st{y=Ys}}) -> - verify_y_init_1(gb_trees:to_list(Ys)). - -verify_y_init_1([]) -> ok; -verify_y_init_1([{Y,uninitialized}|_]) -> - error({uninitialized_reg,{y,Y}}); -verify_y_init_1([{Y,{fragile,_}}|_]) -> - %% Unsafe. This term may be outside any heap belonging - %% to the process and would be corrupted by a GC. - error({fragile_message_reference,{y,Y}}); -verify_y_init_1([{_,_}|Ys]) -> - verify_y_init_1(Ys). - -verify_live(0, #vst{}) -> ok; -verify_live(N, #vst{current=#st{x=Xs}}) -> - verify_live_1(N, Xs). - -verify_live_1(0, _) -> ok; -verify_live_1(N, Xs) when is_integer(N) -> - X = N-1, - case gb_trees:is_defined(X, Xs) of - false -> error({{x,X},not_live}); - true -> verify_live_1(X, Xs) - end; -verify_live_1(N, _) -> error({bad_number_of_live_regs,N}). +verify_live_1(-1, _) -> + ok; +verify_live_1(X, Vst) when is_integer(X) -> + Reg = {x, X}, + case get_raw_type(Reg, Vst) of + uninitialized -> error({Reg, not_live}); + _ -> verify_live_1(X - 1, Vst) + end. -verify_no_ct(#vst{current=#st{numy=none}}) -> ok; +verify_no_ct(#vst{current=#st{numy=none}}) -> + ok; verify_no_ct(#vst{current=#st{numy=undecided}}) -> error(unknown_size_of_stackframe); -verify_no_ct(#vst{current=#st{y=Ys}}) -> - case [Y || Y <- gb_trees:to_list(Ys), verify_no_ct_1(Y)] of - [] -> ok; - CT -> error({unfinished_catch_try,CT}) +verify_no_ct(#vst{current=St}=Vst) -> + case collect_try_catch_tags(St#st.numy - 1, Vst, []) of + [_|_]=Bad -> error({unfinished_catch_try,Bad}); + [] -> ok end. -verify_no_ct_1({_, {catchtag, _}}) -> true; -verify_no_ct_1({_, {trytag, _}}) -> true; -verify_no_ct_1({_, _}) -> false. +%% Collects all try/catch tags, walking down from the Nth stack position. +collect_try_catch_tags(-1, _Vst, Acc) -> + Acc; +collect_try_catch_tags(Y, Vst, Acc0) -> + Tag = get_raw_type({y, Y}, Vst), + Acc = case is_try_catch_tag(Tag) of + true -> [{{y, Y}, Tag} | Acc0]; + false -> Acc0 + end, + collect_try_catch_tags(Y - 1, Vst, Acc). + +is_try_catch_tag({catchtag,_}) -> true; +is_try_catch_tag({trytag,_}) -> true; +is_try_catch_tag(_) -> false. eat_heap(N, #vst{current=#st{h=Heap0}=St}=Vst) -> case Heap0-N of @@ -1647,89 +2583,190 @@ eat_heap_float(#vst{current=#st{hf=HeapFloats0}=St}=Vst) -> Vst#vst{current=St#st{hf=HeapFloats}} end. -remove_fragility(#vst{current=#st{x=Xs0,y=Ys0}=St0}=Vst) -> - F = fun(_, {fragile,Type}) -> Type; - (_, Type) -> Type - end, - Xs = gb_trees:map(F, Xs0), - Ys = gb_trees:map(F, Ys0), - St = St0#st{x=Xs,y=Ys}, +%%% FRAGILITY +%%% +%%% The loop_rec/2 instruction may return a reference to a term that is not +%%% part of the root set. That term or any part of it must not be included in a +%%% garbage collection. Therefore, the term (or any part of it) must not be +%%% passed to another function, placed in another term, or live in a Y register +%%% over an instruction that may GC. +%%% +%%% Fragility is marked on a per-register (rather than per-value) basis. + +%% Marks Reg as fragile. +mark_fragile(Reg, Vst) -> + #vst{current=#st{fragile=Fragile0}=St0} = Vst, + Fragile = cerl_sets:add_element(Reg, Fragile0), + St = St0#st{fragile=Fragile}, Vst#vst{current=St}. -propagate_fragility(Type, Ss, Vst) -> - F = fun(S) -> - case get_term_type_1(S, Vst) of - {fragile,_} -> true; - _ -> false - end - end, - case any(F, Ss) of - true -> make_fragile(Type); - false -> Type +propagate_fragility(Reg, Args, #vst{current=St0}=Vst) -> + #st{fragile=Fragile0} = St0, + + Sources = cerl_sets:from_list(Args), + Fragile = case cerl_sets:is_disjoint(Sources, Fragile0) of + true -> cerl_sets:del_element(Reg, Fragile0); + false -> cerl_sets:add_element(Reg, Fragile0) + end, + + St = St0#st{fragile=Fragile}, + Vst#vst{current=St}. + +%% Marks Reg as durable, must be used when assigning a newly created value to +%% a register. +remove_fragility(Reg, Vst) -> + #vst{current=#st{fragile=Fragile0}=St0} = Vst, + case cerl_sets:is_element(Reg, Fragile0) of + true -> + Fragile = cerl_sets:del_element(Reg, Fragile0), + St = St0#st{fragile=Fragile}, + Vst#vst{current=St}; + false -> + Vst end. -bif_type('-', Src, Vst) -> - arith_type(Src, Vst); -bif_type('+', Src, Vst) -> - arith_type(Src, Vst); -bif_type('*', Src, Vst) -> - arith_type(Src, Vst); -bif_type(abs, [Num], Vst) -> +%% Marks all registers as durable. +remove_fragility(#vst{current=St0}=Vst) -> + St = St0#st{fragile=cerl_sets:new()}, + Vst#vst{current=St}. + +assert_durable_term(Src, Vst) -> + assert_term(Src, Vst), + assert_not_fragile(Src, Vst). + +assert_not_fragile({Kind,_}=Src, Vst) when Kind =:= x; Kind =:= y -> + check_limit(Src), + #vst{current=#st{fragile=Fragile}} = Vst, + case cerl_sets:is_element(Src, Fragile) of + true -> error({fragile_message_reference, Src}); + false -> ok + end; +assert_not_fragile(Lit, #vst{}) -> + assert_literal(Lit), + ok. + +%%% +%%% Return/argument types of BIFs +%%% + +bif_return_type('-', Src, Vst) -> + arith_return_type(Src, Vst); +bif_return_type('+', Src, Vst) -> + arith_return_type(Src, Vst); +bif_return_type('*', Src, Vst) -> + arith_return_type(Src, Vst); +bif_return_type(abs, [Num], Vst) -> case get_term_type(Num, Vst) of - {float,_}=T -> T; - {integer,_}=T -> T; - _ -> number + {float,_}=T -> T; + {integer,_}=T -> T; + _ -> number end; -bif_type(float, _, _) -> {float,[]}; -bif_type('/', _, _) -> {float,[]}; +bif_return_type(float, _, _) -> {float,[]}; +bif_return_type('/', _, _) -> {float,[]}; +%% Binary operations +bif_return_type('binary_part', [_,_], _) -> binary; +bif_return_type('binary_part', [_,_,_], _) -> binary; +bif_return_type('bit_size', [_], _) -> {integer,[]}; +bif_return_type('byte_size', [_], _) -> {integer,[]}; %% Integer operations. -bif_type(ceil, [_], _) -> {integer,[]}; -bif_type('div', [_,_], _) -> {integer,[]}; -bif_type(floor, [_], _) -> {integer,[]}; -bif_type('rem', [_,_], _) -> {integer,[]}; -bif_type(length, [_], _) -> {integer,[]}; -bif_type(size, [_], _) -> {integer,[]}; -bif_type(trunc, [_], _) -> {integer,[]}; -bif_type(round, [_], _) -> {integer,[]}; -bif_type('band', [_,_], _) -> {integer,[]}; -bif_type('bor', [_,_], _) -> {integer,[]}; -bif_type('bxor', [_,_], _) -> {integer,[]}; -bif_type('bnot', [_], _) -> {integer,[]}; -bif_type('bsl', [_,_], _) -> {integer,[]}; -bif_type('bsr', [_,_], _) -> {integer,[]}; +bif_return_type(ceil, [_], _) -> {integer,[]}; +bif_return_type('div', [_,_], _) -> {integer,[]}; +bif_return_type(floor, [_], _) -> {integer,[]}; +bif_return_type('rem', [_,_], _) -> {integer,[]}; +bif_return_type(length, [_], _) -> {integer,[]}; +bif_return_type(size, [_], _) -> {integer,[]}; +bif_return_type(trunc, [_], _) -> {integer,[]}; +bif_return_type(round, [_], _) -> {integer,[]}; +bif_return_type('band', [_,_], _) -> {integer,[]}; +bif_return_type('bor', [_,_], _) -> {integer,[]}; +bif_return_type('bxor', [_,_], _) -> {integer,[]}; +bif_return_type('bnot', [_], _) -> {integer,[]}; +bif_return_type('bsl', [_,_], _) -> {integer,[]}; +bif_return_type('bsr', [_,_], _) -> {integer,[]}; %% Booleans. -bif_type('==', [_,_], _) -> bool; -bif_type('/=', [_,_], _) -> bool; -bif_type('=<', [_,_], _) -> bool; -bif_type('<', [_,_], _) -> bool; -bif_type('>=', [_,_], _) -> bool; -bif_type('>', [_,_], _) -> bool; -bif_type('=:=', [_,_], _) -> bool; -bif_type('=/=', [_,_], _) -> bool; -bif_type('not', [_], _) -> bool; -bif_type('and', [_,_], _) -> bool; -bif_type('or', [_,_], _) -> bool; -bif_type('xor', [_,_], _) -> bool; -bif_type(is_atom, [_], _) -> bool; -bif_type(is_boolean, [_], _) -> bool; -bif_type(is_binary, [_], _) -> bool; -bif_type(is_float, [_], _) -> bool; -bif_type(is_function, [_], _) -> bool; -bif_type(is_integer, [_], _) -> bool; -bif_type(is_list, [_], _) -> bool; -bif_type(is_map, [_], _) -> bool; -bif_type(is_number, [_], _) -> bool; -bif_type(is_pid, [_], _) -> bool; -bif_type(is_port, [_], _) -> bool; -bif_type(is_reference, [_], _) -> bool; -bif_type(is_tuple, [_], _) -> bool; +bif_return_type('==', [_,_], _) -> bool; +bif_return_type('/=', [_,_], _) -> bool; +bif_return_type('=<', [_,_], _) -> bool; +bif_return_type('<', [_,_], _) -> bool; +bif_return_type('>=', [_,_], _) -> bool; +bif_return_type('>', [_,_], _) -> bool; +bif_return_type('=:=', [_,_], _) -> bool; +bif_return_type('=/=', [_,_], _) -> bool; +bif_return_type('not', [_], _) -> bool; +bif_return_type('and', [_,_], _) -> bool; +bif_return_type('or', [_,_], _) -> bool; +bif_return_type('xor', [_,_], _) -> bool; +bif_return_type(is_atom, [_], _) -> bool; +bif_return_type(is_boolean, [_], _) -> bool; +bif_return_type(is_binary, [_], _) -> bool; +bif_return_type(is_float, [_], _) -> bool; +bif_return_type(is_function, [_], _) -> bool; +bif_return_type(is_function, [_,_], _) -> bool; +bif_return_type(is_integer, [_], _) -> bool; +bif_return_type(is_list, [_], _) -> bool; +bif_return_type(is_map, [_], _) -> bool; +bif_return_type(is_number, [_], _) -> bool; +bif_return_type(is_pid, [_], _) -> bool; +bif_return_type(is_port, [_], _) -> bool; +bif_return_type(is_reference, [_], _) -> bool; +bif_return_type(is_tuple, [_], _) -> bool; %% Misc. -bif_type(node, [], _) -> {atom,[]}; -bif_type(node, [_], _) -> {atom,[]}; -bif_type(hd, [_], _) -> term; -bif_type(tl, [_], _) -> term; -bif_type(get, [_], _) -> term; -bif_type(Bif, _, _) when is_atom(Bif) -> term. +bif_return_type(tuple_size, [_], _) -> {integer,[]}; +bif_return_type(map_size, [_], _) -> {integer,[]}; +bif_return_type(node, [], _) -> {atom,[]}; +bif_return_type(node, [_], _) -> {atom,[]}; +bif_return_type(hd, [_], _) -> term; +bif_return_type(tl, [_], _) -> term; +bif_return_type(get, [_], _) -> term; +bif_return_type(Bif, _, _) when is_atom(Bif) -> term. + +%% Generic +bif_arg_types(tuple_size, [_]) -> [{tuple,[0],#{}}]; +bif_arg_types(map_size, [_]) -> [map]; +bif_arg_types(is_map_key, [_,_]) -> [term, map]; +bif_arg_types(map_get, [_,_]) -> [term, map]; +bif_arg_types(length, [_]) -> [list]; +bif_arg_types(hd, [_]) -> [cons]; +bif_arg_types(tl, [_]) -> [cons]; +%% Boolean +bif_arg_types('not', [_]) -> [bool]; +bif_arg_types('and', [_,_]) -> [bool, bool]; +bif_arg_types('or', [_,_]) -> [bool, bool]; +bif_arg_types('xor', [_,_]) -> [bool, bool]; +%% Binary +bif_arg_types('binary_part', [_,_]) -> + PosLen = {tuple, 2, #{ {integer,1} => {integer,[]}, + {integer,2} => {integer,[]} }}, + [binary, PosLen]; +bif_arg_types('binary_part', [_,_,_]) -> + [binary, {integer,[]}, {integer,[]}]; +bif_arg_types('bit_size', [_]) -> [binary]; +bif_arg_types('byte_size', [_]) -> [binary]; +%% Numerical +bif_arg_types('-', [_]) -> [number]; +bif_arg_types('-', [_,_]) -> [number,number]; +bif_arg_types('+', [_]) -> [number]; +bif_arg_types('+', [_,_]) -> [number,number]; +bif_arg_types('*', [_,_]) -> [number, number]; +bif_arg_types('/', [_,_]) -> [number, number]; +bif_arg_types(abs, [_]) -> [number]; +bif_arg_types(ceil, [_]) -> [number]; +bif_arg_types(float, [_]) -> [number]; +bif_arg_types(floor, [_]) -> [number]; +bif_arg_types(trunc, [_]) -> [number]; +bif_arg_types(round, [_]) -> [number]; +%% Integer-specific +bif_arg_types('div', [_,_]) -> [{integer,[]}, {integer,[]}]; +bif_arg_types('rem', [_,_]) -> [{integer,[]}, {integer,[]}]; +bif_arg_types('band', [_,_]) -> [{integer,[]}, {integer,[]}]; +bif_arg_types('bor', [_,_]) -> [{integer,[]}, {integer,[]}]; +bif_arg_types('bxor', [_,_]) -> [{integer,[]}, {integer,[]}]; +bif_arg_types('bnot', [_]) -> [{integer,[]}]; +bif_arg_types('bsl', [_,_]) -> [{integer,[]}, {integer,[]}]; +bif_arg_types('bsr', [_,_]) -> [{integer,[]}, {integer,[]}]; +%% Unsafe type tests that may fail if an argument doesn't have the right type. +bif_arg_types(is_function, [_,_]) -> [term, {integer,[]}]; +bif_arg_types(_, Args) -> [term || _Arg <- Args]. is_bif_safe('/=', 2) -> true; is_bif_safe('<', 2) -> true; @@ -1758,86 +2795,190 @@ is_bif_safe(self, 0) -> true; is_bif_safe(node, 0) -> true; is_bif_safe(_, _) -> false. -arith_type([A,B], Vst) -> - case {get_term_type(A, Vst),get_term_type(B, Vst)} of +arith_return_type([A], Vst) -> + %% Unary '+' or '-'. + case get_term_type(A, Vst) of + {integer,_} -> {integer,[]}; + {float,_} -> {float,[]}; + _ -> number + end; +arith_return_type([A,B], Vst) -> + TypeA = get_term_type(A, Vst), + TypeB = get_term_type(B, Vst), + case {TypeA, TypeB} of + {{integer,_},{integer,_}} -> {integer,[]}; {{float,_},_} -> {float,[]}; {_,{float,_}} -> {float,[]}; {_,_} -> number end; -arith_type(_, _) -> number. +arith_return_type(_, _) -> number. + +%%% +%%% Return/argument types of calls +%%% -return_type({extfunc,M,F,A}, Vst) -> return_type_1(M, F, A, Vst); -return_type(_, _) -> term. +call_return_type({extfunc,M,F,A}, Vst) -> call_return_type_1(M, F, A, Vst); +call_return_type(_, _) -> term. -return_type_1(erlang, setelement, 3, Vst) -> - Tuple = {x,1}, +call_return_type_1(erlang, setelement, 3, Vst) -> + IndexType = get_term_type({x,0}, Vst), TupleType = - case get_term_type(Tuple, Vst) of - {tuple,_}=TT -> - TT; - {literal,Lit} when is_tuple(Lit) -> - {tuple,tuple_size(Lit)}; - _ -> - {tuple,[0]} - end, - case get_term_type({x,0}, Vst) of - {integer,[]} -> TupleType; - {integer,I} -> upgrade_tuple_type({tuple,[I]}, TupleType); - _ -> TupleType + case get_term_type({x,1}, Vst) of + {literal,Tuple}=Lit when is_tuple(Tuple) -> get_literal_type(Lit); + {tuple,_,_}=TT -> TT; + _ -> {tuple,[0],#{}} + end, + case IndexType of + {integer,I} when is_integer(I) -> + case meet({tuple,[I],#{}}, TupleType) of + {tuple, Sz, Es0} -> + ValueType = get_term_type({x,2}, Vst), + Es = set_element_type({integer,I}, ValueType, Es0), + {tuple, Sz, Es}; + none -> + TupleType + end; + _ -> + %% The index could point anywhere, so we must discard all element + %% information. + setelement(3, TupleType, #{}) end; -return_type_1(erlang, F, A, _) -> - return_type_erl(F, A); -return_type_1(math, F, A, _) -> - return_type_math(F, A); -return_type_1(M, F, A, _) when is_atom(M), is_atom(F), is_integer(A), A >= 0 -> +call_return_type_1(erlang, '++', 2, Vst) -> + case get_term_type({x,0}, Vst) =:= cons orelse + get_term_type({x,1}, Vst) =:= cons of + true -> cons; + false -> list + end; +call_return_type_1(erlang, '--', 2, _Vst) -> + list; +call_return_type_1(erlang, F, A, _) -> + erlang_mod_return_type(F, A); +call_return_type_1(lists, F, A, Vst) -> + lists_mod_return_type(F, A, Vst); +call_return_type_1(math, F, A, _) -> + math_mod_return_type(F, A); +call_return_type_1(M, F, A, _) when is_atom(M), is_atom(F), is_integer(A), A >= 0 -> term. -return_type_erl(exit, 1) -> exception; -return_type_erl(throw, 1) -> exception; -return_type_erl(error, 1) -> exception; -return_type_erl(error, 2) -> exception; -return_type_erl(F, A) when is_atom(F), is_integer(A), A >= 0 -> term. - -return_type_math(cos, 1) -> {float,[]}; -return_type_math(cosh, 1) -> {float,[]}; -return_type_math(sin, 1) -> {float,[]}; -return_type_math(sinh, 1) -> {float,[]}; -return_type_math(tan, 1) -> {float,[]}; -return_type_math(tanh, 1) -> {float,[]}; -return_type_math(acos, 1) -> {float,[]}; -return_type_math(acosh, 1) -> {float,[]}; -return_type_math(asin, 1) -> {float,[]}; -return_type_math(asinh, 1) -> {float,[]}; -return_type_math(atan, 1) -> {float,[]}; -return_type_math(atanh, 1) -> {float,[]}; -return_type_math(erf, 1) -> {float,[]}; -return_type_math(erfc, 1) -> {float,[]}; -return_type_math(exp, 1) -> {float,[]}; -return_type_math(log, 1) -> {float,[]}; -return_type_math(log2, 1) -> {float,[]}; -return_type_math(log10, 1) -> {float,[]}; -return_type_math(sqrt, 1) -> {float,[]}; -return_type_math(atan2, 2) -> {float,[]}; -return_type_math(pow, 2) -> {float,[]}; -return_type_math(ceil, 1) -> {float,[]}; -return_type_math(floor, 1) -> {float,[]}; -return_type_math(fmod, 2) -> {float,[]}; -return_type_math(pi, 0) -> {float,[]}; -return_type_math(F, A) when is_atom(F), is_integer(A), A >= 0 -> term. - -check_limit({x,X}) when is_integer(X), X < 1023 -> - %% Note: x(1023) is reserved for use by the BEAM loader. - ok; -check_limit({y,Y}) when is_integer(Y), Y < 1024 -> - ok; -check_limit({fr,Fr}) when is_integer(Fr), Fr < 1024 -> - ok; -check_limit(_) -> - error(limit). +erlang_mod_return_type(exit, 1) -> exception; +erlang_mod_return_type(throw, 1) -> exception; +erlang_mod_return_type(error, 1) -> exception; +erlang_mod_return_type(error, 2) -> exception; +erlang_mod_return_type(F, A) when is_atom(F), is_integer(A), A >= 0 -> term. + +math_mod_return_type(cos, 1) -> {float,[]}; +math_mod_return_type(cosh, 1) -> {float,[]}; +math_mod_return_type(sin, 1) -> {float,[]}; +math_mod_return_type(sinh, 1) -> {float,[]}; +math_mod_return_type(tan, 1) -> {float,[]}; +math_mod_return_type(tanh, 1) -> {float,[]}; +math_mod_return_type(acos, 1) -> {float,[]}; +math_mod_return_type(acosh, 1) -> {float,[]}; +math_mod_return_type(asin, 1) -> {float,[]}; +math_mod_return_type(asinh, 1) -> {float,[]}; +math_mod_return_type(atan, 1) -> {float,[]}; +math_mod_return_type(atanh, 1) -> {float,[]}; +math_mod_return_type(erf, 1) -> {float,[]}; +math_mod_return_type(erfc, 1) -> {float,[]}; +math_mod_return_type(exp, 1) -> {float,[]}; +math_mod_return_type(log, 1) -> {float,[]}; +math_mod_return_type(log2, 1) -> {float,[]}; +math_mod_return_type(log10, 1) -> {float,[]}; +math_mod_return_type(sqrt, 1) -> {float,[]}; +math_mod_return_type(atan2, 2) -> {float,[]}; +math_mod_return_type(pow, 2) -> {float,[]}; +math_mod_return_type(ceil, 1) -> {float,[]}; +math_mod_return_type(floor, 1) -> {float,[]}; +math_mod_return_type(fmod, 2) -> {float,[]}; +math_mod_return_type(pi, 0) -> {float,[]}; +math_mod_return_type(F, A) when is_atom(F), is_integer(A), A >= 0 -> term. + +lists_mod_return_type(all, 2, _Vst) -> + bool; +lists_mod_return_type(any, 2, _Vst) -> + bool; +lists_mod_return_type(keymember, 3, _Vst) -> + bool; +lists_mod_return_type(member, 2, _Vst) -> + bool; +lists_mod_return_type(prefix, 2, _Vst) -> + bool; +lists_mod_return_type(suffix, 2, _Vst) -> + bool; +lists_mod_return_type(dropwhile, 2, _Vst) -> + list; +lists_mod_return_type(duplicate, 2, _Vst) -> + list; +lists_mod_return_type(filter, 2, _Vst) -> + list; +lists_mod_return_type(flatten, 1, _Vst) -> + list; +lists_mod_return_type(map, 2, Vst) -> + same_length_type({x,1}, Vst); +lists_mod_return_type(MF, 3, Vst) when MF =:= mapfoldl; MF =:= mapfoldr -> + ListType = same_length_type({x,2}, Vst), + {tuple,2,#{ {integer,1} => ListType} }; +lists_mod_return_type(partition, 2, _Vst) -> + two_tuple(list, list); +lists_mod_return_type(reverse, 1, Vst) -> + same_length_type({x,0}, Vst); +lists_mod_return_type(seq, 2, _Vst) -> + list; +lists_mod_return_type(sort, 1, Vst) -> + same_length_type({x,0}, Vst); +lists_mod_return_type(sort, 2, Vst) -> + same_length_type({x,1}, Vst); +lists_mod_return_type(splitwith, 2, _Vst) -> + two_tuple(list, list); +lists_mod_return_type(takewhile, 2, _Vst) -> + list; +lists_mod_return_type(unzip, 1, Vst) -> + ListType = same_length_type({x,0}, Vst), + two_tuple(ListType, ListType); +lists_mod_return_type(usort, 1, Vst) -> + same_length_type({x,0}, Vst); +lists_mod_return_type(zip, 2, _Vst) -> + list; +lists_mod_return_type(zipwith, 3, _Vst) -> + list; +lists_mod_return_type(_, _, _) -> + term. + +two_tuple(Type1, Type2) -> + {tuple,2,#{ {integer,1} => Type1, + {integer,2} => Type2 }}. + +same_length_type(Reg, Vst) -> + case get_term_type(Reg, Vst) of + {literal,[_|_]} -> cons; + cons -> cons; + nil -> nil; + _ -> list + end. + +check_limit({x,X}=Src) when is_integer(X) -> + if + %% Note: x(1023) is reserved for use by the BEAM loader. + 0 =< X, X < 1023 -> ok; + 1023 =< X -> error(limit); + X < 0 -> error({bad_register, Src}) + end; +check_limit({y,Y}=Src) when is_integer(Y) -> + if + 0 =< Y, Y < 1024 -> ok; + 1024 =< Y -> error(limit); + Y < 0 -> error({bad_register, Src}) + end; +check_limit({fr,Fr}=Src) when is_integer(Fr) -> + if + 0 =< Fr, Fr < 1023 -> ok; + 1023 =< Fr -> error(limit); + Fr < 0 -> error({bad_register, Src}) + end. min(A, B) when is_integer(A), is_integer(B), A < B -> A; min(A, B) when is_integer(A), is_integer(B) -> B. -gb_trees_from_list(L) -> gb_trees:from_orddict(lists:sort(L)). +gb_trees_from_list(L) -> gb_trees:from_orddict(sort(L)). error(Error) -> throw(Error). diff --git a/lib/compiler/src/beam_z.erl b/lib/compiler/src/beam_z.erl index 1c9d762eb1..415b579240 100644 --- a/lib/compiler/src/beam_z.erl +++ b/lib/compiler/src/beam_z.erl @@ -71,6 +71,31 @@ undo_renames([{get_hd,Src,Dst1},{get_tl,Src,Dst2}|Is]) -> [{get_list,Src,Dst1,Dst2}|undo_renames(Is)]; undo_renames([{get_tl,Src,Dst2},{get_hd,Src,Dst1}|Is]) -> [{get_list,Src,Dst1,Dst2}|undo_renames(Is)]; +undo_renames([{bs_put,_,{bs_put_binary,1,_}, + [{atom,all},{literal,<<>>}]}|Is]) -> + undo_renames(Is); +undo_renames([{bs_put,Fail,{bs_put_binary,1,_Flags}, + [{atom,all},{literal,BinString}]}|Is0]) -> + Bits = bit_size(BinString), + Bytes = Bits div 8, + case Bits rem 8 of + 0 -> + I = {bs_put_string,byte_size(BinString), + {string,BinString}}, + [undo_rename(I)|undo_renames(Is0)]; + Rem -> + <<Binary:Bytes/bytes,Int:Rem>> = BinString, + PutInt = {bs_put_integer,Fail,{integer,Rem},1, + {field_flags,[unsigned,big]},{integer,Int}}, + Is = [PutInt|undo_renames(Is0)], + case Binary of + <<>> -> + Is; + _ -> + [{bs_put_string,byte_size(Binary), + {string,Binary}}|Is] + end + end; undo_renames([I|Is]) -> [undo_rename(I)|undo_renames(Is)]; undo_renames([]) -> []. @@ -79,8 +104,6 @@ undo_rename({bs_put,F,{I,U,Fl},[Sz,Src]}) -> {I,F,Sz,U,Fl,Src}; undo_rename({bs_put,F,{I,Fl},[Src]}) -> {I,F,Fl,Src}; -undo_rename({bs_put,{f,0},{bs_put_string,_,_}=I,[]}) -> - I; undo_rename({bif,bs_add=I,F,[Src1,Src2,{integer,U}],Dst}) -> {I,F,[Src1,Src2,U],Dst}; undo_rename({bif,bs_utf8_size=I,F,[Src],Dst}) -> @@ -101,7 +124,7 @@ undo_rename({test,bs_match_string=Op,F,[Ctx,Bin0]}) -> 0 -> Bin0; Rem -> <<Bin0/bitstring,0:(8-Rem)>> end, - {test,Op,F,[Ctx,Bits,{string,binary_to_list(Bin)}]}; + {test,Op,F,[Ctx,Bits,{string,Bin}]}; undo_rename({put_map,Fail,assoc,S,D,R,L}) -> {put_map_assoc,Fail,S,D,R,L}; undo_rename({put_map,Fail,exact,S,D,R,L}) -> @@ -110,6 +133,8 @@ undo_rename({test,has_map_fields,Fail,[Src|List]}) -> {test,has_map_fields,Fail,Src,{list,List}}; undo_rename({get_map_elements,Fail,Src,{list,List}}) -> {get_map_elements,Fail,Src,{list,List}}; +undo_rename({test,is_eq_exact,Fail,[Src,nil]}) -> + {test,is_nil,Fail,[Src]}; undo_rename({select,I,Reg,Fail,List}) -> {I,Reg,Fail,{list,List}}; undo_rename(I) -> I. diff --git a/lib/compiler/src/cerl.erl b/lib/compiler/src/cerl.erl index fce23bfd68..62cd5b5120 100644 --- a/lib/compiler/src/cerl.erl +++ b/lib/compiler/src/cerl.erl @@ -2157,12 +2157,16 @@ values_arity(Node) -> %% @spec c_binary(Segments::[cerl()]) -> cerl() %% -%% @doc Creates an abstract binary-template. A binary object is a -%% sequence of 8-bit bytes. It is specified by zero or more bit-string -%% template <em>segments</em> of arbitrary lengths (in number of bits), -%% such that the sum of the lengths is evenly divisible by 8. If -%% <code>Segments</code> is <code>[S1, ..., Sn]</code>, the result -%% represents "<code>#{<em>S1</em>, ..., <em>Sn</em>}#</code>". All the + +%% @doc Creates an abstract binary-template. A binary object is in +%% this context a sequence of an arbitrary number of bits. (The number +%% of bits used to be evenly divisible by 8, but after the +%% introduction of bit strings in the Erlang language, the choice was +%% made to use the binary template for all bit strings.) It is +%% specified by zero or more bit-string template <em>segments</em> of +%% arbitrary lengths (in number of bits). If <code>Segments</code> is +%% <code>[S1, ..., Sn]</code>, the result represents +%% "<code>#{<em>S1</em>, ..., <em>Sn</em>}#</code>". All the %% <code>Si</code> must have type <code>bitstr</code>. %% %% @see ann_c_binary/2 diff --git a/lib/compiler/src/cerl_clauses.erl b/lib/compiler/src/cerl_clauses.erl index fa5104c01b..3fd7ddd181 100644 --- a/lib/compiler/src/cerl_clauses.erl +++ b/lib/compiler/src/cerl_clauses.erl @@ -14,8 +14,8 @@ %% @author Richard Carlsson <[email protected]> %% @doc Utility functions for Core Erlang case/receive clauses. %% -%% <p>Syntax trees are defined in the module <a -%% href=""><code>cerl</code></a>.</p> +%% <p>Syntax trees are defined in the module +%% <a href="cerl"><code>cerl</code></a>.</p> %% %% @type cerl() = cerl:cerl() diff --git a/lib/compiler/src/cerl_sets.erl b/lib/compiler/src/cerl_sets.erl index 0361186713..f489baf238 100644 --- a/lib/compiler/src/cerl_sets.erl +++ b/lib/compiler/src/cerl_sets.erl @@ -204,4 +204,4 @@ fold(F, Init, D) -> Set2 :: set(Element). filter(F, D) -> - maps:from_list(lists:filter(fun({K,_}) -> F(K) end, maps:to_list(D))). + maps:filter(fun(K,_) -> F(K) end, D). diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 6510571441..28db8986ff 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -31,6 +31,9 @@ %% Erlc interface. -export([compile/3,compile_beam/3,compile_asm/3,compile_core/3]). +%% Utility functions for compiler passes. +-export([run_sub_passes/2]). + -export_type([option/0]). -include("erl_compile.hrl"). @@ -39,6 +42,8 @@ -import(lists, [member/2,reverse/1,reverse/2,keyfind/3,last/1, map/2,flatmap/2,foreach/2,foldr/3,any/2]). +-define(SUB_PASS_TIMES, compile__sub_pass_times). + %%---------------------------------------------------------------------- -type abstract_code() :: [erl_parse:abstract_form()]. @@ -64,6 +69,7 @@ -type err_ret() :: 'error' | {'error', errors(), warnings()}. -type comp_ret() :: mod_ret() | bin_ret() | err_ret(). + %%---------------------------------------------------------------------- %% @@ -143,6 +149,30 @@ noenv_output_generated(Opts) -> env_compiler_options() -> env_default_opts(). + +%%% +%%% Run sub passes from a compiler pass. +%%% + +-spec run_sub_passes([term()], term()) -> term(). + +run_sub_passes(Ps, St) -> + case get(?SUB_PASS_TIMES) of + undefined -> + Runner = fun(_Name, Run, S) -> Run(S) end, + run_sub_passes_1(Ps, Runner, St); + Times when is_list(Times) -> + Runner = fun(Name, Run, S0) -> + T1 = erlang:monotonic_time(), + S = Run(S0), + T2 = erlang:monotonic_time(), + put(?SUB_PASS_TIMES, + [{Name,T2-T1}|get(?SUB_PASS_TIMES)]), + S + end, + run_sub_passes_1(Ps, Runner, St) + end. + %% %% Local functions %% @@ -180,8 +210,11 @@ do_compile(Input, Opts0) -> {error,Reason} end end, - %% Dialyzer has already spawned workers. - case lists:member(dialyzer, Opts) of + %% Some tools, like Dialyzer, has already spawned workers + %% and spawning extra workers actually slow the compilation + %% down instead of speeding it up, so we provide a mechanism + %% to bypass the compiler process. + case lists:member(no_spawn_compiler_process, Opts) of true -> IntFun(); false -> @@ -218,23 +251,33 @@ expand_opt(report, Os) -> [report_errors,report_warnings|Os]; expand_opt(return, Os) -> [return_errors,return_warnings|Os]; +expand_opt(no_bsm3, Os) -> + %% The new bsm pass requires bsm3 instructions. + [no_bsm3,no_bsm_opt|Os]; expand_opt(r16, Os) -> - [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; + expand_opt_before_21(Os); expand_opt(r17, Os) -> - [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; + expand_opt_before_21(Os); expand_opt(r18, Os) -> - [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; + expand_opt_before_21(Os); expand_opt(r19, Os) -> - [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; + expand_opt_before_21(Os); expand_opt(r20, Os) -> - [no_get_hd_tl,no_record_opt,no_utf8_atoms|Os]; + expand_opt_before_21(Os); +expand_opt(r21, Os) -> + [no_put_tuple2 | expand_opt(no_bsm3, Os)]; expand_opt({debug_info_key,_}=O, Os) -> [encrypt_debug_info,O|Os]; -expand_opt(no_float_opt, Os) -> - %%Turn off the entire type optimization pass. - [no_topt|Os]; +expand_opt(no_type_opt, Os) -> + [no_ssa_opt_type_start, + no_ssa_opt_type_continue, + no_ssa_opt_type_finish | Os]; expand_opt(O, Os) -> [O|Os]. +expand_opt_before_21(Os) -> + [no_put_tuple2, no_get_hd_tl, no_ssa_opt_record, + no_utf8_atoms | expand_opt(no_bsm3, Os)]. + %% format_error(ErrorDescriptor) -> string() -spec format_error(term()) -> iolist(). @@ -247,6 +290,10 @@ format_error(bad_crypto_key) -> "invalid crypto key."; format_error(no_crypto_key) -> "no crypto key supplied."; +format_error({unimplemented_instruction,Instruction}) -> + io_lib:fwrite("native-code compilation failed because of an " + "unimplemented instruction (~s).", + [Instruction]); format_error({native, E}) -> io_lib:fwrite("native-code compilation failed with reason: ~tP.", [E, 25]); @@ -387,17 +434,57 @@ fold_comp([{Name,Pass}|Ps], Run, Code0, St0) -> end; fold_comp([], _Run, Code, St) -> {ok,Code,St}. +run_sub_passes_1([{Name,Run}|Ps], Runner, St0) + when is_atom(Name), is_function(Run, 1) -> + try Runner(Name, Run, St0) of + St -> + run_sub_passes_1(Ps, Runner, St) + catch + C:E:Stk -> + io:format("Sub pass ~s\n", [Name]), + erlang:raise(C, E, Stk) + end; +run_sub_passes_1([], _, St) -> St. + run_tc({Name,Fun}, Code, St) -> + put(?SUB_PASS_TIMES, []), T1 = erlang:monotonic_time(), Val = (catch Fun(Code, St)), T2 = erlang:monotonic_time(), - Elapsed = erlang:convert_time_unit(T2 - T1, native, millisecond), + Times = erase(?SUB_PASS_TIMES), + Elapsed = erlang:convert_time_unit(T2 - T1, native, microsecond), Mem0 = erts_debug:flat_size(Val)*erlang:system_info(wordsize), Mem = lists:flatten(io_lib:format("~.1f kB", [Mem0/1024])), io:format(" ~-30s: ~10.3f s ~12s\n", - [Name,Elapsed/1000,Mem]), + [Name,Elapsed/1000000,Mem]), + print_times(Times, Name), Val. +print_times(Times0, Name) -> + Fam0 = sofs:relation(Times0), + Fam1 = sofs:rel2fam(Fam0), + Fam2 = sofs:to_external(Fam1), + Fam3 = [{W,lists:sum(Times)} || {W,Times} <- Fam2], + Fam = reverse(lists:keysort(2, Fam3)), + Total = case lists:sum([T || {_,T} <- Fam]) of + 0 -> 1; + Total0 -> Total0 + end, + case Fam of + [] -> + ok; + [_|_] -> + io:format(" %% Sub passes of ~s from slowest to fastest:\n", [Name]), + print_times_1(Fam, Total) + end. + +print_times_1([{Name,T}|Ts], Total) -> + Elapsed = erlang:convert_time_unit(T, native, microsecond), + io:format(" ~-27s: ~10.3f s ~3w %\n", + [Name,Elapsed/1000000,round(100*T/Total)]), + print_times_1(Ts, Total); +print_times_1([], _Total) -> ok. + run_eprof({Name,Fun}, Code, Name, St) -> io:format("~p: Running eprof\n", [Name]), c:appcall(tools, eprof, start_profiling, [[self()]]), @@ -731,8 +818,6 @@ kernel_passes() -> %% Optimizations that must be done after all other optimizations. [{pass,sys_core_bsm}, {iff,dcbsm,{listing,"core_bsm"}}, - {pass,sys_core_dsetel}, - {iff,dsetel,{listing,"dsetel"}}, {iff,clint,?pass(core_lint_module)}, {iff,core,?pass(save_core_code)}, @@ -741,8 +826,30 @@ kernel_passes() -> ?pass(v3_kernel), {iff,dkern,{listing,"kernel"}}, {iff,'to_kernel',{done,"kernel"}}, - {pass,v3_codegen}, - {iff,dcg,{listing,"codegen"}} + {pass,beam_kernel_to_ssa}, + {iff,dssa,{listing,"ssa"}}, + {iff,ssalint,{pass,beam_ssa_lint}}, + {delay, + [{unless,no_share_opt,{pass,beam_ssa_share}}, + {iff,dssashare,{listing,"ssashare"}}, + {iff,ssalint,{pass,beam_ssa_lint}}, + {unless,no_bsm_opt,{pass,beam_ssa_bsm}}, + {iff,dssabsm,{listing,"ssabsm"}}, + {iff,ssalint,{pass,beam_ssa_lint}}, + {unless,no_fun_opt,{pass,beam_ssa_funs}}, + {iff,dssafuns,{listing,"ssafuns"}}, + {iff,ssalint,{pass,beam_ssa_lint}}, + {unless,no_ssa_opt,{pass,beam_ssa_opt}}, + {iff,dssaopt,{listing,"ssaopt"}}, + {iff,ssalint,{pass,beam_ssa_lint}}, + {unless,no_recv_opt,{pass,beam_ssa_recv}}, + {iff,drecv,{listing,"recv"}}]}, + {pass,beam_ssa_pre_codegen}, + {iff,dprecg,{listing,"precodegen"}}, + {iff,ssalint,{pass,beam_ssa_lint}}, + {pass,beam_ssa_codegen}, + {iff,dcg,{listing,"codegen"}}, + {iff,doldcg,{listing,"codegen"}} | asm_passes()]. asm_passes() -> @@ -751,34 +858,16 @@ asm_passes() -> [{pass,beam_a}, {iff,da,{listing,"a"}}, {unless,no_postopt, - [{unless,no_reorder,{pass,beam_reorder}}, - {iff,dre,{listing,"reorder"}}, - {pass,beam_block}, + [{pass,beam_block}, {iff,dblk,{listing,"block"}}, {unless,no_except,{pass,beam_except}}, {iff,dexcept,{listing,"except"}}, - {unless,no_bs_opt,{pass,beam_bs}}, - {iff,dbs,{listing,"bs"}}, - {unless,no_topt,{pass,beam_type}}, - {iff,dtype,{listing,"type"}}, - {pass,beam_split}, - {iff,dsplit,{listing,"split"}}, - {unless,no_dead,{pass,beam_dead}}, - {iff,ddead,{listing,"dead"}}, {unless,no_jopt,{pass,beam_jump}}, {iff,djmp,{listing,"jump"}}, {unless,no_peep_opt,{pass,beam_peep}}, {iff,dpeep,{listing,"peep"}}, {pass,beam_clean}, {iff,dclean,{listing,"clean"}}, - {unless,no_bsm_opt,{pass,beam_bsm}}, - {iff,dbsm,{listing,"bsm"}}, - {unless,no_recv_opt,{pass,beam_receive}}, - {iff,drecv,{listing,"recv"}}, - {unless,no_record_opt,{pass,beam_record}}, - {iff,drecord,{listing,"record"}}, - {unless,no_blk2,?pass(block2)}, - {iff,dblk2,{listing,"block2"}}, {unless,no_stack_trimming,{pass,beam_trim}}, {iff,dtrim,{listing,"trim"}}, {pass,beam_flatten}]}, @@ -787,7 +876,9 @@ asm_passes() -> %% need to do a few clean-ups to code. {iff,no_postopt,[{pass,beam_clean}]}, + {iff,diffable,?pass(diffable)}, {pass,beam_z}, + {iff,diffable,{listing,"S"}}, {iff,dz,{listing,"z"}}, {iff,dopt,{listing,"optimize"}}, {iff,'S',{listing,"S"}}, @@ -1360,10 +1451,6 @@ v3_kernel(Code0, #compile{options=Opts,warnings=Ws0}=St) -> {ok,Code,St} end. -block2(Code0, #compile{options=Opts}=St) -> - {ok,Code} = beam_block:module(Code0, [no_blockify|Opts]), - {ok,Code,St}. - test_old_inliner(#compile{options=Opts}) -> %% The point of this test is to avoid loading the old inliner %% if we know that it will not be used. @@ -1568,18 +1655,22 @@ native_compile_1(Code, St) -> case IgnoreErrors of true -> Ws = [{St#compile.ifile,[{none,?MODULE,{native,R}}]}], - {ok,St#compile{warnings=St#compile.warnings ++ Ws}}; + {ok,Code,St#compile{warnings=St#compile.warnings ++ Ws}}; false -> Es = [{St#compile.ifile,[{none,?MODULE,{native,R}}]}], {error,St#compile{errors=St#compile.errors ++ Es}} end catch + exit:{unimplemented_instruction,_}=Unimplemented -> + Ws = [{St#compile.ifile, + [{none,?MODULE,Unimplemented}]}], + {ok,Code,St#compile{warnings=St#compile.warnings ++ Ws}}; Class:R:Stack -> case IgnoreErrors of true -> Ws = [{St#compile.ifile, [{none,?MODULE,{native_crash,R,Stack}}]}], - {ok,St#compile{warnings=St#compile.warnings ++ Ws}}; + {ok,Code,St#compile{warnings=St#compile.warnings ++ Ws}}; false -> erlang:raise(Class, R, Stack) end @@ -1849,6 +1940,39 @@ restore_expand_module([F|Fs]) -> [F|restore_expand_module(Fs)]; restore_expand_module([]) -> []. +%%% +%%% Transform the BEAM code to make it more friendly for +%%% diffing: using function names instead of labels for +%%% local calls and number labels relative to each function. +%%% + +diffable(Code0, St) -> + {Mod,Exp,Attr,Fs0,NumLabels} = Code0, + EntryLabels0 = [{Entry,{Name,Arity}} || + {function,Name,Arity,Entry,_} <- Fs0], + EntryLabels = maps:from_list(EntryLabels0), + Fs = [diffable_fix_function(F, EntryLabels) || F <- Fs0], + Code = {Mod,Exp,Attr,Fs,NumLabels}, + {ok,Code,St}. + +diffable_fix_function({function,Name,Arity,Entry0,Is0}, LabelMap0) -> + Entry = maps:get(Entry0, LabelMap0), + {Is1,LabelMap} = diffable_label_map(Is0, 1, LabelMap0, []), + Fb = fun(Old) -> error({no_fb,Old}) end, + Is = beam_utils:replace_labels(Is1, [], LabelMap, Fb), + {function,Name,Arity,Entry,Is}. + +diffable_label_map([{label,Old}|Is], New, Map, Acc) -> + case Map of + #{Old:=NewLabel} -> + diffable_label_map(Is, New, Map, [{label,NewLabel}|Acc]); + #{} -> + diffable_label_map(Is, New+1, Map#{Old=>New}, [{label,New}|Acc]) + end; +diffable_label_map([I|Is], New, Map, Acc) -> + diffable_label_map(Is, New, Map, [I|Acc]); +diffable_label_map([], _New, Map, Acc) -> + {Acc,Map}. -spec options() -> 'ok'. @@ -1969,22 +2093,25 @@ pre_load() -> L = [beam_a, beam_asm, beam_block, - beam_bs, - beam_bsm, beam_clean, - beam_dead, beam_dict, beam_except, beam_flatten, beam_jump, + beam_kernel_to_ssa, beam_opcodes, beam_peep, - beam_receive, - beam_record, - beam_reorder, - beam_split, + beam_ssa, + beam_ssa_bsm, + beam_ssa_codegen, + beam_ssa_dead, + beam_ssa_funs, + beam_ssa_opt, + beam_ssa_pre_codegen, + beam_ssa_recv, + beam_ssa_share, + beam_ssa_type, beam_trim, - beam_type, beam_utils, beam_validator, beam_z, @@ -2001,9 +2128,7 @@ pre_load() -> erl_scan, sys_core_alias, sys_core_bsm, - sys_core_dsetel, sys_core_fold, - v3_codegen, v3_core, v3_kernel], _ = code:ensure_modules_loaded(L), diff --git a/lib/compiler/src/compiler.app.src b/lib/compiler/src/compiler.app.src index cf32fd251c..a086a3a8d3 100644 --- a/lib/compiler/src/compiler.app.src +++ b/lib/compiler/src/compiler.app.src @@ -24,24 +24,29 @@ beam_a, beam_asm, beam_block, - beam_bs, - beam_bsm, beam_clean, - beam_dead, beam_dict, beam_disasm, beam_except, beam_flatten, beam_jump, + beam_kernel_to_ssa, beam_listing, beam_opcodes, beam_peep, - beam_receive, - beam_reorder, - beam_record, - beam_split, + beam_ssa, + beam_ssa_bsm, + beam_ssa_codegen, + beam_ssa_dead, + beam_ssa_funs, + beam_ssa_lint, + beam_ssa_opt, + beam_ssa_pp, + beam_ssa_pre_codegen, + beam_ssa_recv, + beam_ssa_share, + beam_ssa_type, beam_trim, - beam_type, beam_utils, beam_validator, beam_z, @@ -60,12 +65,10 @@ rec_env, sys_core_alias, sys_core_bsm, - sys_core_dsetel, sys_core_fold, sys_core_fold_lists, sys_core_inline, sys_pre_attributes, - v3_codegen, v3_core, v3_kernel, v3_kernel_pp diff --git a/lib/compiler/src/core_parse.hrl b/lib/compiler/src/core_parse.hrl index 83a6f0179c..90c796d3d9 100644 --- a/lib/compiler/src/core_parse.hrl +++ b/lib/compiler/src/core_parse.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2016. All Rights Reserved. +%% Copyright Ericsson AB 1999-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -29,81 +29,82 @@ %% The record definitions appear alphabetically --record(c_alias, {anno=[], var, % var :: Tree, - pat}). % pat :: Tree +-record(c_alias, {anno=[] :: list(), var :: cerl:cerl(), + pat :: cerl:cerl()}). --record(c_apply, {anno=[], op, % op :: Tree, - args}). % args :: [Tree] +-record(c_apply, {anno=[] :: list(), op :: cerl:cerl(), + args :: [cerl:cerl()]}). --record(c_binary, {anno=[], segments :: [cerl:c_bitstr()]}). +-record(c_binary, {anno=[] :: list(), segments :: [cerl:c_bitstr()]}). --record(c_bitstr, {anno=[], val, % val :: Tree, - size, % size :: Tree, - unit, % unit :: Tree, - type, % type :: Tree, - flags}). % flags :: Tree +-record(c_bitstr, {anno=[] :: list(), val :: cerl:cerl(), + size :: cerl:cerl(), + unit :: cerl:cerl(), + type :: cerl:cerl(), + flags :: cerl:cerl()}). --record(c_call, {anno=[], module, % module :: Tree, - name, % name :: Tree, - args}). % args :: [Tree] +-record(c_call, {anno=[] :: list(), module :: cerl:cerl(), + name :: cerl:cerl(), + args :: [cerl:cerl()]}). --record(c_case, {anno=[], arg, % arg :: Tree, - clauses}). % clauses :: [Tree] +-record(c_case, {anno=[] :: list(), arg :: cerl:cerl(), + clauses :: [cerl:cerl()]}). --record(c_catch, {anno=[], body}). % body :: Tree +-record(c_catch, {anno=[] :: list(), body :: cerl:cerl()}). --record(c_clause, {anno=[], pats, % pats :: [Tree], - guard, % guard :: Tree, - body}). % body :: Tree +-record(c_clause, {anno=[] :: list(), pats :: [cerl:cerl()], + guard :: cerl:cerl(), + body :: cerl:cerl() | any()}). % TODO --record(c_cons, {anno=[], hd, % hd :: Tree, - tl}). % tl :: Tree +-record(c_cons, {anno=[] :: list(), hd :: cerl:cerl(), + tl :: cerl:cerl()}). --record(c_fun, {anno=[], vars, % vars :: [Tree], - body}). % body :: Tree +-record(c_fun, {anno=[] :: list(), vars :: [cerl:cerl()], + body :: cerl:cerl()}). --record(c_let, {anno=[], vars, % vars :: [Tree], - arg, % arg :: Tree, - body}). % body :: Tree +-record(c_let, {anno=[] :: list(), vars :: [cerl:cerl()], + arg :: cerl:cerl(), + body :: cerl:cerl()}). --record(c_letrec, {anno=[], defs, % defs :: [#c_def{}], - body}). % body :: Tree +-record(c_letrec, {anno=[] :: list(), + defs :: [{cerl:cerl(), cerl:cerl()}], + body :: cerl:cerl()}). --record(c_literal, {anno=[], val}). % val :: literal() +-record(c_literal, {anno=[] :: list(), val :: any()}). --record(c_map, {anno=[], +-record(c_map, {anno=[] :: list(), arg=#c_literal{val=#{}} :: cerl:c_var() | cerl:c_literal(), es :: [cerl:c_map_pair()], is_pat=false :: boolean()}). --record(c_map_pair, {anno=[], +-record(c_map_pair, {anno=[] :: list(), op :: #c_literal{val::'assoc'} | #c_literal{val::'exact'}, - key, - val}). + key :: any(), % TODO + val :: any()}). % TODO --record(c_module, {anno=[], name, % name :: Tree, - exports, % exports :: [Tree], - attrs, % attrs :: [#c_def{}], - defs}). % defs :: [#c_def{}] +-record(c_module, {anno=[] :: list(), name :: cerl:cerl(), + exports :: [cerl:cerl()], + attrs :: [{cerl:cerl(), cerl:cerl()}], + defs :: [{cerl:cerl(), cerl:cerl()}]}). --record(c_primop, {anno=[], name, % name :: Tree, - args}). % args :: [Tree] +-record(c_primop, {anno=[] :: list(), name :: cerl:cerl(), + args :: [cerl:cerl()]}). --record(c_receive, {anno=[], clauses, % clauses :: [Tree], - timeout, % timeout :: Tree, - action}). % action :: Tree +-record(c_receive, {anno=[] :: list(), clauses :: [cerl:cerl()], + timeout :: cerl:cerl(), + action :: cerl:cerl()}). --record(c_seq, {anno=[], arg, % arg :: Tree, - body}). % body :: Tree +-record(c_seq, {anno=[] :: list(), arg :: cerl:cerl() | any(), % TODO + body :: cerl:cerl()}). --record(c_try, {anno=[], arg, % arg :: Tree, - vars, % vars :: [Tree], - body, % body :: Tree - evars, % evars :: [Tree], - handler}). % handler :: Tree +-record(c_try, {anno=[] :: list(), arg :: cerl:cerl(), + vars :: [cerl:cerl()], + body :: cerl:cerl(), + evars :: [cerl:cerl()], + handler :: cerl:cerl()}). --record(c_tuple, {anno=[], es}). % es :: [Tree] +-record(c_tuple, {anno=[] :: list(), es :: [cerl:cerl()]}). --record(c_values, {anno=[], es}). % es :: [Tree] +-record(c_values, {anno=[] :: list(), es :: [cerl:cerl()]}). --record(c_var, {anno=[], name :: cerl:var_name()}). +-record(c_var, {anno=[] :: list(), name :: cerl:var_name()}). diff --git a/lib/compiler/src/erl_bifs.erl b/lib/compiler/src/erl_bifs.erl index 68489a0122..94a5dfe012 100644 --- a/lib/compiler/src/erl_bifs.erl +++ b/lib/compiler/src/erl_bifs.erl @@ -32,6 +32,22 @@ %% Returns `true' if the function `Module:Name/Arity' does not %% affect the state, nor depend on the state, although its %% evaluation is not guaranteed to complete normally for all input. +%% +%% NOTE: There is no need to include every new pure BIF +%% here. Including it here means that the value of the function +%% will be evaluated at compile-time if the arguments are +%% constant. If that optimization is not useful/desired, there is +%% no need to include the new BIF here. +%% +%% Functions whose return value could conceivably change in a +%% future version of the runtime system must NOT be included here. +%% +%% Here are some example of functions that should not be +%% included: `term_to_binary/1', hashing functions, non-trivial +%% encode/decode functions. +%% +%% When unsure whether a new BIF should be included here, the +%% conservative safe choice is NOT to include it. -spec is_pure(atom(), atom(), arity()) -> boolean(). @@ -91,6 +107,7 @@ is_pure(erlang, is_bitstring, 1) -> true; %% erlang:is_builtin/3 depends on the state (i.e. the version of the emulator). is_pure(erlang, is_float, 1) -> true; is_pure(erlang, is_function, 1) -> true; +is_pure(erlang, is_function, 2) -> true; is_pure(erlang, is_integer, 1) -> true; is_pure(erlang, is_list, 1) -> true; is_pure(erlang, is_map, 1) -> true; @@ -107,6 +124,7 @@ is_pure(erlang, list_to_atom, 1) -> true; is_pure(erlang, list_to_binary, 1) -> true; is_pure(erlang, list_to_float, 1) -> true; is_pure(erlang, list_to_integer, 1) -> true; +is_pure(erlang, list_to_integer, 2) -> true; is_pure(erlang, list_to_pid, 1) -> true; is_pure(erlang, list_to_tuple, 1) -> true; is_pure(erlang, max, 2) -> true; @@ -193,6 +211,7 @@ is_safe(erlang, is_float, 1) -> true; is_safe(erlang, is_function, 1) -> true; is_safe(erlang, is_integer, 1) -> true; is_safe(erlang, is_list, 1) -> true; +is_safe(erlang, is_map, 1) -> true; is_safe(erlang, is_number, 1) -> true; is_safe(erlang, is_pid, 1) -> true; is_safe(erlang, is_port, 1) -> true; diff --git a/lib/compiler/src/genop.tab b/lib/compiler/src/genop.tab index 02dead9e92..86590fad87 100755 --- a/lib/compiler/src/genop.tab +++ b/lib/compiler/src/genop.tab @@ -142,8 +142,7 @@ BEAM_FORMAT_NUMBER=0 20: send/0 ## @spec remove_message -## @doc Unlink the current message from the message queue and store a -## pointer to the message in x(0). Remove any timeout. +## @doc Unlink the current message from the message queue. Remove any timeout. 21: remove_message/0 ## @spec timeout @@ -574,3 +573,26 @@ BEAM_FORMAT_NUMBER=0 ## @doc Get the tail (or cdr) part of a list (a cons cell) from Source and ## put it into the register Tail. 163: get_tl/2 + +# OTP 22 + +## @spec put_tuple2 Destination Elements +## @doc Build a tuple with the elements in the list Elements and put it +## put into register Destination. +164: put_tuple2/2 + +## @spec bs_get_tail Ctx Dst Live +## @doc Sets Dst to the tail of Ctx at the current position +165: bs_get_tail/3 + +## @spec bs_start_match3 Fail Bin Live Dst +## @doc Starts a binary match sequence +166: bs_start_match3/4 + +## @spec bs_get_position Ctx Dst Live +## @doc Sets Dst to the current position of Ctx +167: bs_get_position/3 + +## @spec bs_set_positon Ctx Pos +## @doc Sets the current position of Ctx to Pos +168: bs_set_position/2 diff --git a/lib/compiler/src/sys_core_bsm.erl b/lib/compiler/src/sys_core_bsm.erl index 62657933ee..685e807e65 100644 --- a/lib/compiler/src/sys_core_bsm.erl +++ b/lib/compiler/src/sys_core_bsm.erl @@ -24,223 +24,52 @@ -export([module/2,format_error/1]). -include("core_parse.hrl"). --import(lists, [member/2,reverse/1,usort/1]). -spec module(cerl:c_module(), [compile:option()]) -> {'ok', cerl:c_module()}. -module(#c_module{defs=Ds0}=Mod, Opts) -> - {Ds,Ws0} = function(Ds0, [], []), - case member(bin_opt_info, Opts) of - false -> - {ok,Mod#c_module{defs=Ds}}; - true -> - Ws1 = [make_warning(Where, What) || {Where,What} <- Ws0], - Ws = usort(Ws1), - {ok,Mod#c_module{defs=Ds},Ws} - end. +module(#c_module{defs=Ds}=Mod, _Opts) -> + {ok,Mod#c_module{defs=function(Ds)}}. -function([{#c_var{name={F,Arity}}=Name,B0}|Fs], FsAcc, Ws0) -> - try cerl_trees:mapfold(fun bsm_an/2, Ws0, B0) of - {B,Ws} -> - function(Fs, [{Name,B}|FsAcc], Ws) +function([{#c_var{name={F,Arity}}=Name,B0}|Fs]) -> + try cerl_trees:map(fun bsm_reorder/1, B0) of + B -> [{Name,B} | function(Fs)] catch - throw:unsafe_bs_context_to_binary -> - %% Unsafe bs_context_to_binary (in the sense that the - %% contents of the binary will probably be wrong). - %% Disable binary optimizations for the entire function. - %% We don't generate an INFO message, because this happens - %% very infrequently and it would be hard to explain in - %% a comprehensible way in an INFO message. - function(Fs, [{Name,B0}|FsAcc], Ws0); Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [F,Arity]), - erlang:raise(Class, Error, Stack) + io:fwrite("Function: ~w/~w\n", [F,Arity]), + erlang:raise(Class, Error, Stack) end; -function([], Fs, Ws) -> - {reverse(Fs),Ws}. +function([]) -> + []. -type error() :: atom(). -spec format_error(error()) -> nonempty_string(). -format_error(bin_opt_alias) -> - "INFO: the '=' operator will prevent delayed sub binary optimization"; -format_error(bin_partition) -> - "INFO: matching non-variables after a previous clause matching a variable " - "will prevent delayed sub binary optimization"; -format_error(bin_var_used) -> - "INFO: using a matched out sub binary will prevent " - "delayed sub binary optimization"; -format_error(orig_bin_var_used_in_guard) -> - "INFO: using the original binary variable in a guard will prevent " - "delayed sub binary optimization"; -format_error(bin_var_used_in_guard) -> - "INFO: using a matched out sub binary in a guard will prevent " - "delayed sub binary optimization". - +format_error(_) -> error(badarg). -%%% -%%% Annotate bit syntax matching to faciliate optimization in further passes. -%%% +%%% Reorder bit syntax matching to faciliate optimization in further passes. -bsm_an(Core0, Ws0) -> - case bsm_an(Core0) of - {ok,Core} -> - {Core,Ws0}; - {ok,Core,W} -> - {Core,[W|Ws0]} - end. +bsm_reorder(#c_case{arg=#c_var{}=V}=Case) -> + bsm_reorder_1([V], Case); +bsm_reorder(#c_case{arg=#c_values{es=Es}}=Case) -> + bsm_reorder_1(Es, Case); +bsm_reorder(Core) -> + Core. -bsm_an(#c_case{arg=#c_var{}=V}=Case) -> - bsm_an_1([V], Case); -bsm_an(#c_case{arg=#c_values{es=Es}}=Case) -> - bsm_an_1(Es, Case); -bsm_an(Other) -> - {ok,Other}. - -bsm_an_1(Vs0, #c_case{clauses=Cs0}=Case) -> +bsm_reorder_1(Vs0, #c_case{clauses=Cs0}=Case) -> case bsm_leftmost(Cs0) of - none -> - {ok,Case}; - 1 -> - bsm_an_2(Vs0, Cs0, Case); - Pos -> - Vs = move_from_col(Pos, Vs0), - Cs = [C#c_clause{pats=move_from_col(Pos, Ps)} || - #c_clause{pats=Ps}=C <- Cs0], - bsm_an_2(Vs, Cs, Case) - end. - -bsm_an_2(Vs, Cs, Case) -> - try - bsm_ensure_no_partition(Cs), - {ok,bsm_do_an(Vs, Cs, Case)} - catch - throw:{problem,Where,What} -> - {ok,Case,{Where,What}} + Pos when Pos > 0, Pos =/= none -> + Vs = core_lib:make_values(move_from_col(Pos, Vs0)), + Cs = [C#c_clause{pats=move_from_col(Pos, Ps)} + || #c_clause{pats=Ps}=C <- Cs0], + Case#c_case{arg=Vs,clauses=Cs}; + _ -> + Case end. move_from_col(Pos, L) -> {First,[Col|Rest]} = lists:split(Pos - 1, L), [Col|First] ++ Rest. -bsm_do_an([#c_var{name=Vname}=V0|Vs0], Cs0, Case) -> - bsm_inner_context_to_binary(Cs0), - Cs = bsm_do_an_var(Vname, Cs0), - V = bsm_annotate_for_reuse(V0), - Vs = core_lib:make_values([V|Vs0]), - Case#c_case{arg=Vs,clauses=Cs}; -bsm_do_an(_Vs, _Cs, Case) -> Case. - -bsm_inner_context_to_binary([#c_clause{body=B}|Cs]) -> - %% Consider: - %% - %% foo(<<Length, Data/binary>>) -> %Line 1 - %% case {Data, Length} of %Line 2 - %% {_, 0} -> Data; %Line 3 - %% {<<...>>, 4} -> ... %Line 4 - %% end. - %% - %% No sub binary will be created for Data in line 1. The match - %% context will be passed on to the `case` in line 2. In line 3, - %% this pass inserts a `bs_context_to_binary` instruction to - %% convert the match context representing Data to a binary before - %% returning it. The problem is that the binary created will be - %% the original binary (including the matched out Length field), - %% not the tail of the binary as it is supposed to be. - %% - %% Here follows a heuristic to disable the binary optimizations - %% for the entire function if this code kind of code is found. - - case cerl_trees:free_variables(B) of - [] -> - %% Since there are no free variables in the body of - %% this clause, there can't be any troublesome - %% bs_context_to_binary instructions. - bsm_inner_context_to_binary(Cs); - [_|_]=Free -> - %% One of the free variables could refer to a match context - %% created by the outer binary match. - F = fun(#c_primop{name=#c_literal{val=bs_context_to_binary}, - args=[#c_var{name=V}]}, _) -> - case member(V, Free) of - true -> - %% This bs_context_to_binary instruction will - %% make a binary of the match context from an - %% outer binary match. It is very likely that - %% the contents of the binary will be wrong - %% (the original binary as opposed to only - %% the tail binary). - throw(unsafe_bs_context_to_binary); - false -> - %% Safe. This bs_context_to_binary instruction - %% will make a binary from a match context - %% defined in the body of the clause. - ok - end; - (_, _) -> - ok - end, - cerl_trees:fold(F, ok, B) - end; -bsm_inner_context_to_binary([]) -> ok. - -bsm_do_an_var(V, [#c_clause{pats=[P|_],guard=G,body=B0}=C0|Cs]) -> - case P of - #c_var{name=VarName} -> - case core_lib:is_var_used(V, G) of - true -> bsm_problem(C0, orig_bin_var_used_in_guard); - false -> ok - end, - case core_lib:is_var_used(VarName, G) of - true -> bsm_problem(C0, bin_var_used_in_guard); - false -> ok - end, - B1 = bsm_maybe_ctx_to_binary(VarName, B0), - B = bsm_maybe_ctx_to_binary(V, B1), - C = C0#c_clause{body=B}, - [C|bsm_do_an_var(V, Cs)]; - #c_alias{} -> - case bsm_could_match_binary(P) of - false -> - [C0|bsm_do_an_var(V, Cs)]; - true -> - bsm_problem(C0, bin_opt_alias) - end; - _ -> - case bsm_could_match_binary(P) andalso bsm_is_var_used(V, G, B0) of - false -> - [C0|bsm_do_an_var(V, Cs)]; - true -> - bsm_problem(C0, bin_var_used) - end - end; -bsm_do_an_var(_, []) -> []. - -bsm_annotate_for_reuse(#c_var{anno=Anno}=Var) -> - Var#c_var{anno=[reuse_for_context|Anno]}. - -bsm_is_var_used(V, G, B) -> - core_lib:is_var_used(V, G) orelse core_lib:is_var_used(V, B). - -bsm_maybe_ctx_to_binary(V, B) -> - case core_lib:is_var_used(V, B) andalso not previous_ctx_to_binary(V, B) of - false -> - B; - true -> - #c_seq{arg=#c_primop{name=#c_literal{val=bs_context_to_binary}, - args=[#c_var{name=V}]}, - body=B} - end. - -previous_ctx_to_binary(V, Core) -> - case Core of - #c_seq{arg=#c_primop{name=#c_literal{val=bs_context_to_binary}, - args=[#c_var{name=V}]}} -> - true; - _ -> - false - end. - %% bsm_leftmost(Cs) -> none | ArgumentNumber %% Find the leftmost argument that matches a nonempty binary. %% Return either 'none' or the argument number (1-N). @@ -262,94 +91,3 @@ bsm_leftmost_2([_|Ps], Cs, N, Pos) -> bsm_leftmost_2(Ps, Cs, N+1, Pos); bsm_leftmost_2([], Cs, _, Pos) -> bsm_leftmost_1(Cs, Pos). - -%% bsm_ensure_no_partition(Cs) -> ok (exception if problem) -%% There must only be a single bs_start_match2 instruction if we -%% are to reuse the binary variable for the match context. -%% -%% To make sure that there is only a single bs_start_match2 -%% instruction, we will check for partitions such as: -%% -%% foo(<<...>>) -> ... -%% foo(<Variable>) when ... -> ... -%% foo(<Non-variable pattern>) -> -%% -%% If there is such partition, we reject the optimization. - -bsm_ensure_no_partition(Cs) -> - bsm_ensure_no_partition_1(Cs, before). - -%% Loop through each clause. -bsm_ensure_no_partition_1([#c_clause{pats=Ps,guard=G}|Cs], State0) -> - State = bsm_ensure_no_partition_2(Ps, G, State0), - case State of - 'after' -> - bsm_ensure_no_partition_after(Cs); - _ -> - ok - end, - bsm_ensure_no_partition_1(Cs, State); -bsm_ensure_no_partition_1([], _) -> ok. - -bsm_ensure_no_partition_2([#c_binary{}|_], _, _State) -> - within; -bsm_ensure_no_partition_2([#c_alias{}=Alias|_], N, State) -> - %% Retrieve the real pattern that the alias refers to and check that. - P = bsm_real_pattern(Alias), - bsm_ensure_no_partition_2([P], N, State); -bsm_ensure_no_partition_2([_|_], _, before=State) -> - %% No binary matching yet - therefore no partition. - State; -bsm_ensure_no_partition_2([P|_], _, State) -> - case bsm_could_match_binary(P) of - false -> - State; - true -> - %% The pattern P *may* match a binary, so we must update the state. - %% (P must be a variable.) - 'after' - end. - -bsm_ensure_no_partition_after([#c_clause{pats=Ps}=C|Cs]) -> - case Ps of - [#c_var{}|_] -> - bsm_ensure_no_partition_after(Cs); - _ -> - bsm_problem(C, bin_partition) - end; -bsm_ensure_no_partition_after([]) -> ok. - -bsm_could_match_binary(#c_alias{pat=P}) -> bsm_could_match_binary(P); -bsm_could_match_binary(#c_cons{}) -> false; -bsm_could_match_binary(#c_tuple{}) -> false; -bsm_could_match_binary(#c_literal{val=Lit}) -> is_bitstring(Lit); -bsm_could_match_binary(_) -> true. - -bsm_real_pattern(#c_alias{pat=P}) -> bsm_real_pattern(P); -bsm_real_pattern(P) -> P. - -bsm_problem(Where, What) -> - throw({problem,Where,What}). - -make_warning(Core, Term) -> - case should_suppress_warning(Core) of - true -> - ok; - false -> - Anno = cerl:get_ann(Core), - Line = get_line(Anno), - File = get_file(Anno), - {File,[{Line,?MODULE,Term}]} - end. - -should_suppress_warning(Core) -> - Ann = cerl:get_ann(Core), - member(compiler_generated, Ann). - -get_line([Line|_]) when is_integer(Line) -> Line; -get_line([_|T]) -> get_line(T); -get_line([]) -> none. - -get_file([{file,File}|_]) -> File; -get_file([_|T]) -> get_file(T); -get_file([]) -> "no_file". % should not happen diff --git a/lib/compiler/src/sys_core_dsetel.erl b/lib/compiler/src/sys_core_dsetel.erl deleted file mode 100644 index 9ab83c210f..0000000000 --- a/lib/compiler/src/sys_core_dsetel.erl +++ /dev/null @@ -1,360 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2002-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% -%% Purpose : Using dsetelement to make multiple-field record updates -%% faster. - -%% The expansion of record field updates, when more than one field is -%% updated, but not a majority of the fields, will create a sequence of -%% calls to 'erlang:setelement(Index, Value, Tuple)' where Tuple in the -%% first call is the original record tuple, and in the subsequent calls -%% Tuple is the result of the previous call. Furthermore, all Index -%% values are constant positive integers, and the first call to -%% 'setelement' will have the greatest index. Thus all the following -%% calls do not actually need to test at run-time whether Tuple has type -%% tuple, nor that the index is within the tuple bounds. -%% -%% Since this introduces destructive updates in the Core Erlang code, it -%% must be done as a last stage before going to lower-level code. -%% -%% NOTE: Because there are currently no write barriers in the system, -%% this kind of optimization can only be done when we are sure that -%% garbage collection will not be triggered between the creation of the -%% tuple and the destructive updates - otherwise we might insert -%% pointers from an older generation to a newer. -%% -%% The rewriting is done as follows: -%% -%% let X1 = call 'erlang':'setelement(5, Tuple, Value1) -%% in call 'erlang':'setelement(3, X1, Value2) -%% => -%% let X1 = call 'erlang':'setelement(5, Tuple, Value1) -%% in do primop dsetelement(3, X1, Value2) -%% X1 -%% and -%% let X1 = call 'erlang':'setelement(5, Tuple, Value1) -%% in let X2 = call 'erlang':'setelement(3, X1, Value2) -%% in ... -%% => -%% let X2 = call 'erlang':'setelement(5, Tuple, Value1) -%% in do primop 'dsetelement(3, X2, Value2) -%% ... -%% if X1 is used exactly once. -%% Thus, we need to track variable usage. -%% - --module(sys_core_dsetel). - --export([module/2]). - --include("core_parse.hrl"). - --spec module(cerl:c_module(), [compile:option()]) -> {'ok', cerl:c_module()}. - -module(M0, _Options) -> - M = visit_module(M0), - {ok,M}. - -visit_module(#c_module{defs=Ds0}=R) -> - Env = #{}, - Ds = visit_module_1(Ds0, Env, []), - R#c_module{defs=Ds}. - -visit_module_1([{Name,F0}|Fs], Env, Acc) -> - try visit(Env, F0) of - {F,_} -> - visit_module_1(Fs, Env, [{Name,F}|Acc]) - catch - Class:Error:Stack -> - #c_var{name={Func,Arity}} = Name, - io:fwrite("Function: ~w/~w\n", [Func,Arity]), - erlang:raise(Class, Error, Stack) - end; -visit_module_1([], _, Acc) -> - lists:reverse(Acc). - -visit(Env, #c_var{name={_,_}}=R) -> - %% Ignore local function name. - {R, Env}; -visit(Env0, #c_var{name=X}=R) -> - %% There should not be any free variables. If there are, - %% the case will fail with an exception. - case Env0 of - #{X:=N} -> - {R, Env0#{X:=N+1}} - end; -visit(Env, #c_literal{}=R) -> - {R, Env}; -visit(Env0, #c_tuple{es=Es0}=R) -> - {Es1,Env1} = visit_list(Env0, Es0), - {R#c_tuple{es=Es1}, Env1}; -visit(Env0, #c_map{es=Es0}=R) -> - {Es1,Env1} = visit_list(Env0, Es0), - {R#c_map{es=Es1}, Env1}; -visit(Env0, #c_map_pair{key=K0,val=V0}=R) -> - {K,Env1} = visit(Env0, K0), - {V,Env2} = visit(Env1, V0), - {R#c_map_pair{key=K,val=V}, Env2}; -visit(Env0, #c_cons{hd=H0,tl=T0}=R) -> - {H1,Env1} = visit(Env0, H0), - {T1,Env2} = visit(Env1, T0), - {R#c_cons{hd=H1,tl=T1}, Env2}; -visit(Env0, #c_binary{segments=Segs}=R) -> - Env = visit_bin_segs(Env0, Segs), - {R, Env}; -visit(Env0, #c_values{es=Es0}=R) -> - {Es1,Env1} = visit_list(Env0, Es0), - {R#c_values{es=Es1}, Env1}; -visit(Env0, #c_fun{vars=Vs, body=B0}=R) -> - {Xs, Env1} = bind_vars(Vs, Env0), - {B1,Env2} = visit(Env1, B0), - {R#c_fun{body=B1}, restore_vars(Xs, Env0, Env2)}; -visit(Env0, #c_let{vars=Vs, arg=A0, body=B0}=R) -> - {A1,Env1} = visit(Env0, A0), - {Xs,Env2} = bind_vars(Vs, Env1), - {B1,Env3} = visit(Env2, B0), - rewrite(R#c_let{arg=A1,body=B1}, Env3, restore_vars(Xs, Env1, Env3)); -visit(Env0, #c_seq{arg=A0, body=B0}=R) -> - {A1,Env1} = visit(Env0, A0), - {B1,Env2} = visit(Env1, B0), - {R#c_seq{arg=A1,body=B1}, Env2}; -visit(Env0, #c_case{arg=A0,clauses=Cs0}=R) -> - {A1,Env1} = visit(Env0, A0), - {Cs1,Env2} = visit_list(Env1, Cs0), - {R#c_case{arg=A1,clauses=Cs1}, Env2}; -visit(Env0, #c_clause{pats=Ps,guard=G0,body=B0}=R) -> - {Vs, Env1} = visit_pats(Ps, Env0), - {G1,Env2} = visit(Env1, G0), - {B1,Env3} = visit(Env2, B0), - {R#c_clause{guard=G1,body=B1}, restore_vars(Vs, Env0, Env3)}; -visit(Env0, #c_receive{clauses=Cs0,timeout=T0,action=A0}=R) -> - {T1,Env1} = visit(Env0, T0), - {Cs1,Env2} = visit_list(Env1, Cs0), - {A1,Env3} = visit(Env2, A0), - {R#c_receive{clauses=Cs1,timeout=T1,action=A1}, Env3}; -visit(Env0, #c_apply{op=Op0, args=As0}=R) -> - {Op1,Env1} = visit(Env0, Op0), - {As1,Env2} = visit_list(Env1, As0), - {R#c_apply{op=Op1,args=As1}, Env2}; -visit(Env0, #c_call{module=M0,name=N0,args=As0}=R) -> - {M1,Env1} = visit(Env0, M0), - {N1,Env2} = visit(Env1, N0), - {As1,Env3} = visit_list(Env2, As0), - {R#c_call{module=M1,name=N1,args=As1}, Env3}; -visit(Env0, #c_primop{name=N0, args=As0}=R) -> - {N1,Env1} = visit(Env0, N0), - {As1,Env2} = visit_list(Env1, As0), - {R#c_primop{name=N1,args=As1}, Env2}; -visit(Env0, #c_try{arg=E0, vars=Vs, body=B0, evars=Evs, handler=H0}=R) -> - {E1,Env1} = visit(Env0, E0), - {Xs, Env2} = bind_vars(Vs, Env1), - {B1,Env3} = visit(Env2, B0), - Env4 = restore_vars(Xs, Env1, Env3), - {Ys, Env5} = bind_vars(Evs, Env4), - {H1,Env6} = visit(Env5, H0), - {R#c_try{arg=E1,body=B1,handler=H1}, restore_vars(Ys, Env4, Env6)}; -visit(Env0, #c_catch{body=B0}=R) -> - {B1,Env1} = visit(Env0, B0), - {R#c_catch{body=B1}, Env1}; -visit(Env0, #c_letrec{defs=Ds0,body=B0}=R) -> - {Xs, Env1} = bind_vars([V || {V,_} <- Ds0], Env0), - {Ds1,Env2} = visit_def_list(Env1, Ds0), - {B1,Env3} = visit(Env2, B0), - {R#c_letrec{defs=Ds1,body=B1}, restore_vars(Xs, Env0, Env3)}. -%% The following general code for handling modules is slow if a module -%% contains very many functions. There is special code in visit_module/1 -%% which is much faster. -%% visit(Env0, #c_module{defs=D0}=R) -> -%% {R1,Env1} = visit(Env0, #c_letrec{defs=D0,body=#c_nil{}}), -%% {R#c_module{defs=R1#c_letrec.defs}, Env1}; - -visit_list(Env, L) -> - lists:mapfoldl(fun (E, A) -> visit(A, E) end, Env, L). - -visit_def_list(Env, L) -> - lists:mapfoldl(fun ({Name,V0}, E0) -> - {V1,E1} = visit(E0, V0), - {{Name,V1}, E1} - end, Env, L). - -visit_bin_segs(Env, Segs) -> - lists:foldl(fun (#c_bitstr{val=Val,size=Sz}, E0) -> - {_, E1} = visit(E0, Val), - {_, E2} = visit(E1, Sz), - E2 - end, Env, Segs). - -bind_vars(Vs, Env) -> - bind_vars(Vs, Env, []). - -bind_vars([#c_var{name=X}|Vs], Env0, Xs)-> - bind_vars(Vs, Env0#{X=>0}, [X|Xs]); -bind_vars([], Env,Xs) -> - {Xs, Env}. - -visit_pats(Ps, Env) -> - visit_pats(Ps, Env, []). - -visit_pats([P|Ps], Env0, Vs0) -> - {Vs1, Env1} = visit_pat(Env0, P, Vs0), - visit_pats(Ps, Env1, Vs1); -visit_pats([], Env, Vs) -> - {Vs, Env}. - -visit_pat(Env0, #c_var{name=V}, Vs) -> - {[V|Vs], Env0#{V=>0}}; -visit_pat(Env0, #c_tuple{es=Es}, Vs) -> - visit_pats(Es, Env0, Vs); -visit_pat(Env0, #c_map{es=Es}, Vs) -> - visit_pats(Es, Env0, Vs); -visit_pat(Env0, #c_map_pair{op=#c_literal{val=exact},key=V,val=K}, Vs0) -> - {Vs1, Env1} = visit_pat(Env0, V, Vs0), - visit_pat(Env1, K, Vs1); -visit_pat(Env0, #c_cons{hd=H,tl=T}, Vs0) -> - {Vs1, Env1} = visit_pat(Env0, H, Vs0), - visit_pat(Env1, T, Vs1); -visit_pat(Env0, #c_binary{segments=Segs}, Vs) -> - visit_pats(Segs, Env0, Vs); -visit_pat(Env0, #c_bitstr{val=Val,size=Sz}, Vs0) -> - {Vs1, Env1} = - case Sz of - #c_var{name=V} -> - %% We don't tolerate free variables. - case Env0 of - #{V:=N} -> - {Vs0, Env0#{V:=N+1}} - end; - _ -> - visit_pat(Env0, Sz, Vs0) - end, - visit_pat(Env1, Val, Vs1); -visit_pat(Env0, #c_alias{pat=P,var=#c_var{name=V}}, Vs) -> - visit_pat(Env0#{V=>0}, P, [V|Vs]); -visit_pat(Env, #c_literal{}, Vs) -> - {Vs, Env}. - -restore_vars([V|Vs], Env0, Env1) -> - case Env0 of - #{V:=N} -> - restore_vars(Vs, Env0, Env1#{V=>N}); - _ -> - restore_vars(Vs, Env0, maps:remove(V, Env1)) - end; -restore_vars([], _, Env1) -> - Env1. - - -%% let X1 = call 'erlang':'setelement(5, Tuple, Value1) -%% in call 'erlang':'setelement(3, X1, Value2) -%% => -%% let X1 = call 'erlang':'setelement(5, Tuple, Value1) -%% in do primop dsetelement(3, X1, Value2) -%% X1 - -rewrite(#c_let{vars=[#c_var{name=X}=V]=Vs, - arg=#c_call{module=#c_literal{val='erlang'}, - name=#c_literal{val='setelement'}, - args=[#c_literal{val=Index1}, _Tuple, _Val1] - }=A, - body=#c_call{anno=Banno,module=#c_literal{val='erlang'}, - name=#c_literal{val='setelement'}, - args=[#c_literal{val=Index2}, - #c_var{name=X}, - Val2] - } - }=R, - _BodyEnv, FinalEnv) - when is_integer(Index1), is_integer(Index2), Index2 > 0, Index1 > Index2 -> - case is_safe(Val2) of - true -> - {R#c_let{vars=Vs, - arg=A, - body=#c_seq{arg=#c_primop{ - anno=Banno, - name=#c_literal{val='dsetelement'}, - args=[#c_literal{val=Index2}, - V, - Val2]}, - body=V} - }, - FinalEnv}; - false -> - {R, FinalEnv} - end; - -%% let X1 = call 'erlang':'setelement(5, Tuple, Value1) -%% in let X2 = 'erlang':'setelement(3, X1, Value2) -%% in ... -%% => -%% let X2 = call 'erlang':'setelement(5, Tuple, Value1) -%% in do primop dsetelement(3, X2, Value2) -%% ... -%% if X1 is used exactly once. - -rewrite(#c_let{vars=[#c_var{name=X1}], - arg=#c_call{module=#c_literal{val='erlang'}, - name=#c_literal{val='setelement'}, - args=[#c_literal{val=Index1}, _Tuple, _Val1] - }=A, - body=#c_let{vars=[#c_var{}=V]=Vs, - arg=#c_call{anno=Banno, - module=#c_literal{val='erlang'}, - name=#c_literal{val='setelement'}, - args=[#c_literal{val=Index2}, - #c_var{name=X1}, - Val2]}, - body=B} - }=R, - BodyEnv, FinalEnv) - when is_integer(Index1), is_integer(Index2), Index2 > 0, Index1 > Index2 -> - case is_single_use(X1, BodyEnv) andalso is_safe(Val2) of - true -> - {R#c_let{vars=Vs, - arg=A, - body=#c_seq{arg=#c_primop{ - anno=Banno, - name=#c_literal{val='dsetelement'}, - args=[#c_literal{val=Index2}, - V, - Val2]}, - body=B} - }, - FinalEnv}; - false -> - {R, FinalEnv} - end; - -rewrite(R, _, FinalEnv) -> - {R, FinalEnv}. - -%% is_safe(CoreExpr) -> true|false -%% Determines whether the Core expression can cause a GC collection at run-time. -%% Note: Assumes that the constant pool is turned on. - -is_safe(#c_var{}) -> true; -is_safe(#c_literal{}) -> true; -is_safe(_) -> false. - -is_single_use(V, Env) -> - case Env of - #{V:=1} -> - true; - _ -> - false - end. diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index 1681d97efb..4939a94a92 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -99,7 +99,7 @@ t=#{} :: map(), %Types in_guard=false}). %In guard or not. --type type_info() :: cerl:cerl() | 'bool' | 'integer'. +-type type_info() :: cerl:cerl() | 'bool' | 'integer' | {'fun', pos_integer()}. -type yes_no_maybe() :: 'yes' | 'no' | 'maybe'. -type sub() :: #sub{}. @@ -478,9 +478,20 @@ expr(#c_try{anno=A,arg=E0,vars=Vs0,body=B0,evars=Evs0,handler=H0}=Try, _, Sub0) false -> {Evs1,Sub2} = var_list(Evs0, Sub0), H1 = body(H0, value, Sub2), - Try#c_try{arg=E1,vars=Vs1,body=B1,evars=Evs1,handler=H1} + H2 = opt_try_handler(H1, lists:last(Evs1)), + Try#c_try{arg=E1,vars=Vs1,body=B1,evars=Evs1,handler=H2} end. +%% Attempts to convert old erlang:get_stacktrace/0 calls into the new +%% three-argument catch, with possibility of further optimisations. +opt_try_handler(#c_call{anno=A,module=#c_literal{val=erlang},name=#c_literal{val=get_stacktrace},args=[]}, Var) -> + #c_primop{anno=A,name=#c_literal{val=build_stacktrace},args=[Var]}; +opt_try_handler(#c_case{clauses=Cs0} = Case, Var) -> + Cs = [C#c_clause{body=opt_try_handler(B, Var)} || #c_clause{body=B} = C <- Cs0], + Case#c_case{clauses=Cs}; +opt_try_handler(#c_let{arg=Arg} = Let, Var) -> + Let#c_let{arg=opt_try_handler(Arg, Var)}; +opt_try_handler(X, _) -> X. %% If a fun or its application is used as an argument, then it's unsafe to %% handle it in effect context as the side-effects may rely on its return @@ -561,6 +572,7 @@ ifes_list(FVar, [E|Es], Safe) -> ifes_list(_FVar, [], _Safe) -> true. + expr_list(Es, Ctxt, Sub) -> [expr(E, Ctxt, Sub) || E <- Es]. @@ -949,18 +961,16 @@ fold_lit_args(Call, Module, Name, Args0) -> %% fold_non_lit_args(Call, erlang, is_boolean, [Arg], Sub) -> eval_is_boolean(Call, Arg, Sub); -fold_non_lit_args(Call, erlang, element, [Arg1,Arg2], Sub) -> - eval_element(Call, Arg1, Arg2, Sub); fold_non_lit_args(Call, erlang, length, [Arg], _) -> eval_length(Call, Arg); fold_non_lit_args(Call, erlang, '++', [Arg1,Arg2], _) -> eval_append(Call, Arg1, Arg2); fold_non_lit_args(Call, lists, append, [Arg1,Arg2], _) -> eval_append(Call, Arg1, Arg2); -fold_non_lit_args(Call, erlang, setelement, [Arg1,Arg2,Arg3], _) -> - eval_setelement(Call, Arg1, Arg2, Arg3); -fold_non_lit_args(Call, erlang, is_record, [Arg1,Arg2,Arg3], Sub) -> - eval_is_record(Call, Arg1, Arg2, Arg3, Sub); +fold_non_lit_args(Call, erlang, is_function, [Arg1], Sub) -> + eval_is_function_1(Call, Arg1, Sub); +fold_non_lit_args(Call, erlang, is_function, [Arg1,Arg2], Sub) -> + eval_is_function_2(Call, Arg1, Arg2, Sub); fold_non_lit_args(Call, erlang, N, Args, Sub) -> NumArgs = length(Args), case erl_internal:comp_op(N, NumArgs) of @@ -976,6 +986,22 @@ fold_non_lit_args(Call, erlang, N, Args, Sub) -> end; fold_non_lit_args(Call, _, _, _, _) -> Call. +eval_is_function_1(Call, Arg1, Sub) -> + case get_type(Arg1, Sub) of + none -> Call; + {'fun',_} -> #c_literal{anno=cerl:get_ann(Call),val=true}; + _ -> #c_literal{anno=cerl:get_ann(Call),val=false} + end. + +eval_is_function_2(Call, Arg1, #c_literal{val=Arity}, Sub) + when is_integer(Arity), Arity > 0 -> + case get_type(Arg1, Sub) of + none -> Call; + {'fun',Arity} -> #c_literal{anno=cerl:get_ann(Call),val=true}; + _ -> #c_literal{anno=cerl:get_ann(Call),val=false} + end; +eval_is_function_2(Call, _Arg1, _Arg2, _Sub) -> Call. + %% Evaluate a relational operation using type information. eval_rel_op(Call, Op, [#c_var{name=V},#c_var{name=V}], _) -> Bool = erlang:Op(same, same), @@ -1109,96 +1135,6 @@ eval_append(Call, #c_cons{anno=Anno,hd=H,tl=T}, List) -> eval_append(Call, X, Y) -> Call#c_call{args=[X,Y]}. %Rebuild call arguments. -%% eval_element(Call, Pos, Tuple, Types) -> Val. -%% Evaluates element/2 if the position Pos is a literal and -%% the shape of the tuple Tuple is known. -%% -eval_element(Call, #c_literal{val=Pos}, Tuple, Types) - when is_integer(Pos) -> - case get_type(Tuple, Types) of - none -> - Call; - Type -> - Es = case cerl:is_c_tuple(Type) of - false -> []; - true -> cerl:tuple_es(Type) - end, - if - 1 =< Pos, Pos =< length(Es) -> - El = lists:nth(Pos, Es), - try - cerl:set_ann(pat_to_expr(El), [compiler_generated]) - catch - throw:impossible -> - Call - end; - true -> - %% Index outside tuple or not a tuple. - eval_failure(Call, badarg) - end - end; -eval_element(Call, Pos, Tuple, Sub) -> - case is_int_type(Pos, Sub) =:= no orelse - is_tuple_type(Tuple, Sub) =:= no of - true -> - eval_failure(Call, badarg); - false -> - Call - end. - -%% eval_is_record(Call, Var, Tag, Size, Types) -> Val. -%% Evaluates is_record/3 using type information. -%% -eval_is_record(Call, Term, #c_literal{val=NeededTag}, - #c_literal{val=Size}, Types) -> - case get_type(Term, Types) of - none -> - Call; - Type -> - Es = case cerl:is_c_tuple(Type) of - false -> []; - true -> cerl:tuple_es(Type) - end, - case Es of - [#c_literal{val=Tag}|_] -> - Bool = Tag =:= NeededTag andalso - length(Es) =:= Size, - #c_literal{val=Bool}; - _ -> - #c_literal{val=false} - end - end; -eval_is_record(Call, _, _, _, _) -> Call. - -%% eval_setelement(Call, Pos, Tuple, NewVal) -> Core. -%% Evaluates setelement/3 if position Pos is an integer -%% and the shape of the tuple Tuple is known. -%% -eval_setelement(Call, #c_literal{val=Pos}, Tuple, NewVal) - when is_integer(Pos) -> - case cerl:is_data(Tuple) of - false -> - Call; - true -> - Es0 = case cerl:is_c_tuple(Tuple) of - false -> []; - true -> cerl:tuple_es(Tuple) - end, - if - 1 =< Pos, Pos =< length(Es0) -> - Es = eval_setelement_1(Pos, Es0, NewVal), - cerl:update_c_tuple(Tuple, Es); - true -> - eval_failure(Call, badarg) - end - end; -eval_setelement(Call, _, _, _) -> Call. - -eval_setelement_1(1, [_|T], NewVal) -> - [NewVal|T]; -eval_setelement_1(Pos, [H|T], NewVal) when Pos > 1 -> - [H|eval_setelement_1(Pos-1, T, NewVal)]. - %% eval_failure(Call, Reason) -> Core. %% Warn for a call that will fail and replace the call with %% a call to erlang:error(Reason). @@ -1258,16 +1194,15 @@ clause(#c_clause{pats=Ps0}=Cl, Cexpr, Ctxt, Sub0) -> end. clause_1(#c_clause{guard=G0,body=B0}=Cl, Ps1, Cexpr, Ctxt, Sub1) -> - Sub2 = update_types(Cexpr, Ps1, Sub1), GSub = case {Cexpr,Ps1,G0} of {_,_,#c_literal{}} -> %% No need for substitution tricks when the guard %% does not contain any variables. - Sub2; + Sub1; {#c_var{name='_'},_,_} -> %% In a 'receive', Cexpr is the variable '_', which represents the %% message being matched. We must NOT do any extra substiutions. - Sub2; + Sub1; {#c_var{},[#c_var{}=Var],_} -> %% The idea here is to optimize expressions such as %% @@ -1289,16 +1224,16 @@ clause_1(#c_clause{guard=G0,body=B0}=Cl, Ps1, Cexpr, Ctxt, Sub1) -> %% case cerl:is_c_fname(Cexpr) of false -> - sub_set_var(Var, Cexpr, Sub2); + sub_set_var(Var, Cexpr, Sub1); true -> %% We must not copy funs, and especially not into guards. - Sub2 + Sub1 end; _ -> - Sub2 + Sub1 end, G1 = guard(G0, GSub), - B1 = body(B0, Ctxt, Sub2), + B1 = body(B0, Ctxt, Sub1), Cl#c_clause{pats=Ps1,guard=G1,body=B1}. %% let_substs(LetVars, LetArg, Sub) -> {[Var],[Val],Sub}. @@ -1382,8 +1317,7 @@ pattern(#c_binary{segments=V0}=Pat, Isub, Osub0) -> {Pat#c_binary{segments=V1},Osub1}; pattern(#c_alias{var=V0,pat=P0}=Pat, Isub, Osub0) -> {V1,Osub1} = pattern(V0, Isub, Osub0), - {P1,Osub2} = pattern(P0, Isub, Osub1), - Osub = update_types(V1, [P1], Osub2), + {P1,Osub} = pattern(P0, Isub, Osub1), {Pat#c_alias{var=V1,pat=P1},Osub}. map_pair_pattern_list(Ps0, Isub, Osub0) -> @@ -2063,58 +1997,22 @@ case_opt_compiler_generated(Core) -> %% case_expand_var(Expr0, Sub) -> Expr -%% If Expr0 is a variable that has been previously matched and -%% is known to be a tuple, return the tuple instead. Otherwise +%% If Expr0 is a variable that is known to be bound to a +%% constructed tuple, return the tuple instead. Otherwise %% return Expr0 unchanged. -%% + case_expand_var(E, #sub{t=Tdb}) -> Key = cerl:var_name(E), case Tdb of - #{Key:=T0} -> - case cerl:is_c_tuple(T0) of - false -> - E; - true -> - %% The pattern was a tuple. Now we must make sure - %% that the elements of the tuple are suitable. In - %% particular, we don't want binary or map - %% construction here, since that means that the - %% binary or map will be constructed in the 'case' - %% argument. That is wasteful for binaries. Even - %% worse is that any map pattern that use the ':=' - %% operator will fail when used in map - %% construction (only the '=>' operator is allowed - %% when constructing a map from scratch). - try - cerl_trees:map(fun coerce_to_data/1, T0) - catch - throw:impossible -> - %% Something unsuitable was found (map or - %% or binary). Keep the variable. - E - end + #{Key:=T} -> + case cerl:is_c_tuple(T) of + false -> E; + true -> T end; _ -> E end. -%% coerce_to_data(Core) -> Core' -%% Coerce an element originally from a pattern to an data item or or -%% variable. Throw an 'impossible' exception if non-data Core Erlang -%% terms such as binary construction or map construction are -%% encountered. - -coerce_to_data(C) -> - case cerl:is_c_alias(C) of - false -> - case cerl:is_data(C) orelse cerl:is_c_var(C) of - true -> C; - false -> throw(impossible) - end; - true -> - coerce_to_data(cerl:alias_pat(C)) - end. - %% case_opt_nomatch(E, Clauses, LitExpr) -> Clauses' %% Remove all clauses that cannot possibly match. @@ -3108,14 +3006,6 @@ is_int_type(Var, Sub) -> C -> yes_no(cerl:is_c_int(C)) end. --spec is_tuple_type(cerl:cerl(), sub()) -> yes_no_maybe(). - -is_tuple_type(Var, Sub) -> - case get_type(Var, Sub) of - none -> maybe; - C -> yes_no(cerl:is_c_tuple(C)) - end. - yes_no(true) -> yes; yes_no(false) -> no. @@ -3177,23 +3067,23 @@ returns_integer(_, _) -> false. %% update_types(Expr, Pattern, Sub) -> Sub' %% Update the type database. --spec update_types(cerl:cerl(), [type_info()], sub()) -> sub(). +-spec update_types(cerl:c_var(), [type_info()], sub()) -> sub(). -update_types(Expr, Pat, #sub{t=Tdb0}=Sub) -> - Tdb = update_types_1(Expr, Pat, Tdb0), +update_types(#c_var{name=V}, Pat, #sub{t=Tdb0}=Sub) -> + Tdb = update_types_1(V, Pat, Tdb0), Sub#sub{t=Tdb}. -update_types_1(#c_var{name=V}, Pat, Types) -> - update_types_2(V, Pat, Types); -update_types_1(_, _, Types) -> Types. - -update_types_2(V, [#c_tuple{}=P], Types) -> +update_types_1(V, [#c_tuple{}=P], Types) -> Types#{V=>P}; -update_types_2(V, [#c_literal{val=Bool}], Types) when is_boolean(Bool) -> +update_types_1(V, [#c_literal{val=Bool}], Types) when is_boolean(Bool) -> Types#{V=>bool}; -update_types_2(V, [Type], Types) when is_atom(Type) -> +update_types_1(V, [#c_fun{vars=Vars}], Types) -> + Types#{V=>{'fun',length(Vars)}}; +update_types_1(V, [#c_var{name={_,Arity}}], Types) -> + Types#{V=>{'fun',Arity}}; +update_types_1(V, [Type], Types) when is_atom(Type) -> Types#{V=>Type}; -update_types_2(_, _, Types) -> Types. +update_types_1(_, _, Types) -> Types. %% kill_types(V, Tdb) -> Tdb' %% Kill any entries that references the variable, @@ -3209,6 +3099,8 @@ kill_types2(V, [{_,#c_tuple{}=Tuple}=Entry|Tdb]) -> false -> [Entry|kill_types2(V, Tdb)]; true -> kill_types2(V, Tdb) end; +kill_types2(V, [{_, {'fun',_}}=Entry|Tdb]) -> + [Entry|kill_types2(V, Tdb)]; kill_types2(V, [{_,Atom}=Entry|Tdb]) when is_atom(Atom) -> [Entry|kill_types2(V, Tdb)]; kill_types2(_, []) -> []. diff --git a/lib/compiler/src/sys_core_fold_lists.erl b/lib/compiler/src/sys_core_fold_lists.erl index 9867fab46a..e93b435011 100644 --- a/lib/compiler/src/sys_core_fold_lists.erl +++ b/lib/compiler/src/sys_core_fold_lists.erl @@ -37,22 +37,27 @@ call(#c_call{anno=Anno}, lists, all, [Arg1,Arg2]) -> Xs = #c_var{name='Xs'}, X = #c_var{name='X'}, Err1 = #c_tuple{es=[#c_literal{val='case_clause'}, X]}, - CC1 = #c_clause{pats=[#c_literal{val=true}], guard=#c_literal{val=true}, + CC1 = #c_clause{anno=Anno, + pats=[#c_literal{val=true}], guard=#c_literal{val=true}, body=#c_apply{anno=Anno, op=Loop, args=[Xs]}}, - CC2 = #c_clause{pats=[#c_literal{val=false}], guard=#c_literal{val=true}, + CC2 = #c_clause{anno=Anno, + pats=[#c_literal{val=false}], guard=#c_literal{val=true}, body=#c_literal{val=false}}, - CC3 = #c_clause{pats=[X], guard=#c_literal{val=true}, + CC3 = #c_clause{anno=Anno, + pats=[X], guard=#c_literal{val=true}, body=match_fail(Anno, Err1)}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + C1 = #c_clause{anno=Anno, + pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, body=#c_case{arg=#c_apply{anno=Anno, op=F, args=[X]}, clauses = [CC1, CC2, CC3]}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], + C2 = #c_clause{anno=Anno, + pats=[#c_literal{val=[]}], guard=#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}, args=[F, #c_literal{val=1}]}, body=#c_literal{val=true}}, Err2 = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + C3 = #c_clause{anno=Anno, pats=[Xs], guard=#c_literal{val=true}, body=match_fail([{function_name,{'lists^all',1}}|Anno], Err2)}, Fun = #c_fun{vars=[Xs], body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, @@ -66,16 +71,21 @@ call(#c_call{anno=Anno}, lists, any, [Arg1,Arg2]) -> Xs = #c_var{name='Xs'}, X = #c_var{name='X'}, Err1 = #c_tuple{es=[#c_literal{val='case_clause'}, X]}, - CC1 = #c_clause{pats=[#c_literal{val=true}], guard=#c_literal{val=true}, + CC1 = #c_clause{anno=Anno, + pats=[#c_literal{val=true}], guard=#c_literal{val=true}, body=#c_literal{val=true}}, - CC2 = #c_clause{pats=[#c_literal{val=false}], guard=#c_literal{val=true}, + CC2 = #c_clause{anno=Anno, + pats=[#c_literal{val=false}], guard=#c_literal{val=true}, body=#c_apply{anno=Anno, op=Loop, args=[Xs]}}, - CC3 = #c_clause{pats=[X], guard=#c_literal{val=true}, + CC3 = #c_clause{anno=Anno, + pats=[X], guard=#c_literal{val=true}, body=match_fail(Anno, Err1)}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + C1 = #c_clause{anno=Anno, + pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, body=#c_case{arg=#c_apply{anno=Anno, op=F, args=[X]}, clauses = [CC1, CC2, CC3]}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], + C2 = #c_clause{anno=Anno, + pats=[#c_literal{val=[]}], guard=#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}, args=[F, #c_literal{val=1}]}, @@ -94,16 +104,17 @@ call(#c_call{anno=Anno}, lists, foreach, [Arg1,Arg2]) -> F = #c_var{name='F'}, Xs = #c_var{name='Xs'}, X = #c_var{name='X'}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + C1 = #c_clause{anno=Anno, + pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, body=#c_seq{arg=#c_apply{anno=Anno, op=F, args=[X]}, body=#c_apply{anno=Anno, op=Loop, args=[Xs]}}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], + C2 = #c_clause{anno=Anno, pats=[#c_literal{val=[]}], guard=#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}, args=[F, #c_literal{val=1}]}, body=#c_literal{val=ok}}, Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + C3 = #c_clause{anno=Anno, pats=[Xs], guard=#c_literal{val=true}, body=match_fail([{function_name,{'lists^foreach',1}}|Anno], Err)}, Fun = #c_fun{vars=[Xs], body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, @@ -117,7 +128,8 @@ call(#c_call{anno=Anno}, lists, map, [Arg1,Arg2]) -> Xs = #c_var{name='Xs'}, X = #c_var{name='X'}, H = #c_var{name='H'}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + C1 = #c_clause{anno=Anno, + pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, body=#c_let{vars=[H], arg=#c_apply{anno=Anno, op=F, args=[X]}, @@ -126,7 +138,7 @@ call(#c_call{anno=Anno}, lists, map, [Arg1,Arg2]) -> tl=#c_apply{anno=Anno, op=Loop, args=[Xs]}}}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], + C2 = #c_clause{anno=Anno, pats=[#c_literal{val=[]}], guard=#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}, args=[F, #c_literal{val=1}]}, @@ -146,7 +158,8 @@ call(#c_call{anno=Anno}, lists, flatmap, [Arg1,Arg2]) -> Xs = #c_var{name='Xs'}, X = #c_var{name='X'}, H = #c_var{name='H'}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + C1 = #c_clause{anno=Anno, + pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, body=#c_let{vars=[H], arg=#c_apply{anno=Anno, op=F, args=[X]}, body=#c_call{anno=[compiler_generated|Anno], @@ -156,13 +169,13 @@ call(#c_call{anno=Anno}, lists, flatmap, [Arg1,Arg2]) -> #c_apply{anno=Anno, op=Loop, args=[Xs]}]}}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], + C2 = #c_clause{anno=Anno, pats=[#c_literal{val=[]}], guard=#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}, args=[F, #c_literal{val=1}]}, body=#c_literal{val=[]}}, Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + C3 = #c_clause{anno=Anno, pats=[Xs], guard=#c_literal{val=true}, body=match_fail([{function_name,{'lists^flatmap',1}}|Anno], Err)}, Fun = #c_fun{vars=[Xs], body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, @@ -177,11 +190,13 @@ call(#c_call{anno=Anno}, lists, filter, [Arg1,Arg2]) -> X = #c_var{name='X'}, B = #c_var{name='B'}, Err1 = #c_tuple{es=[#c_literal{val='case_clause'}, X]}, - CC1 = #c_clause{pats=[#c_literal{val=true}], guard=#c_literal{val=true}, + CC1 = #c_clause{anno=Anno, + pats=[#c_literal{val=true}], guard=#c_literal{val=true}, body=#c_cons{anno=[compiler_generated], hd=X, tl=Xs}}, - CC2 = #c_clause{pats=[#c_literal{val=false}], guard=#c_literal{val=true}, + CC2 = #c_clause{anno=Anno, + pats=[#c_literal{val=false}], guard=#c_literal{val=true}, body=Xs}, - CC3 = #c_clause{pats=[X], guard=#c_literal{val=true}, + CC3 = #c_clause{anno=Anno, pats=[X], guard=#c_literal{val=true}, body=match_fail(Anno, Err1)}, Case = #c_case{arg=B, clauses = [CC1, CC2, CC3]}, C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, @@ -192,13 +207,15 @@ call(#c_call{anno=Anno}, lists, filter, [Arg1,Arg2]) -> op=Loop, args=[Xs]}, body=Case}}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], + C2 = #c_clause{anno=Anno, + pats=[#c_literal{val=[]}], guard=#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}, args=[F, #c_literal{val=1}]}, body=#c_literal{val=[]}}, Err2 = #c_tuple{es=[#c_literal{val='function_clause'}, F, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + C3 = #c_clause{anno=Anno, + pats=[Xs], guard=#c_literal{val=true}, body=match_fail([{function_name,{'lists^filter',1}}|Anno], Err2)}, Fun = #c_fun{vars=[Xs], body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, @@ -212,19 +229,20 @@ call(#c_call{anno=Anno}, lists, foldl, [Arg1,Arg2,Arg3]) -> Xs = #c_var{name='Xs'}, X = #c_var{name='X'}, A = #c_var{name='A'}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + C1 = #c_clause{anno=Anno, + pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, body=#c_apply{anno=Anno, op=Loop, args=[Xs, #c_apply{anno=Anno, op=F, args=[X, A]}]}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], + C2 = #c_clause{anno=Anno, pats=[#c_literal{val=[]}], guard=#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}, args=[F, #c_literal{val=2}]}, body=A}, Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, A, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + C3 = #c_clause{anno=Anno, pats=[Xs], guard=#c_literal{val=true}, body=match_fail([{function_name,{'lists^foldl',2}}|Anno], Err)}, Fun = #c_fun{vars=[Xs, A], body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, @@ -238,19 +256,20 @@ call(#c_call{anno=Anno}, lists, foldr, [Arg1,Arg2,Arg3]) -> Xs = #c_var{name='Xs'}, X = #c_var{name='X'}, A = #c_var{name='A'}, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + C1 = #c_clause{anno=Anno, + pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, body=#c_apply{anno=Anno, op=F, args=[X, #c_apply{anno=Anno, op=Loop, args=[Xs, A]}]}}, - C2 = #c_clause{pats=[#c_literal{val=[]}], + C2 = #c_clause{anno=Anno, pats=[#c_literal{val=[]}], guard=#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}, args=[F, #c_literal{val=2}]}, body=A}, Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, A, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + C3 = #c_clause{anno=Anno, pats=[Xs], guard=#c_literal{val=true}, body=match_fail([{function_name,{'lists^foldr',2}}|Anno], Err)}, Fun = #c_fun{vars=[Xs, A], body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, @@ -266,13 +285,14 @@ call(#c_call{anno=Anno}, lists, mapfoldl, [Arg1,Arg2,Arg3]) -> Avar = #c_var{name='A'}, Match = fun (A, P, E) -> - C1 = #c_clause{pats=[P], guard=#c_literal{val=true}, body=E}, + C1 = #c_clause{anno=Anno, pats=[P], guard=#c_literal{val=true}, body=E}, Err = #c_tuple{es=[#c_literal{val='badmatch'}, X]}, - C2 = #c_clause{pats=[X], guard=#c_literal{val=true}, + C2 = #c_clause{anno=Anno, pats=[X], guard=#c_literal{val=true}, body=match_fail(Anno, Err)}, #c_case{arg=A, clauses=[C1, C2]} end, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + C1 = #c_clause{anno=Anno, + pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, body=Match(#c_apply{anno=Anno, op=F, args=[X, Avar]}, #c_tuple{es=[X, Avar]}, %%% Tuple passing version @@ -292,7 +312,7 @@ call(#c_call{anno=Anno}, lists, mapfoldl, [Arg1,Arg2,Arg3]) -> %%% body=#c_values{es=[#c_cons{hd=X, tl=Xs}, %%% A]}} )}, - C2 = #c_clause{pats=[#c_literal{val=[]}], + C2 = #c_clause{anno=Anno, pats=[#c_literal{val=[]}], guard=#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}, args=[F, #c_literal{val=2}]}, @@ -302,7 +322,7 @@ call(#c_call{anno=Anno}, lists, mapfoldl, [Arg1,Arg2,Arg3]) -> %%% Multiple-value version %%% body=#c_values{es=[#c_literal{val=[]}, A]}}, Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Avar, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + C3 = #c_clause{anno=Anno, pats=[Xs], guard=#c_literal{val=true}, body=match_fail([{function_name,{'lists^mapfoldl',2}}|Anno], Err)}, Fun = #c_fun{vars=[Xs, Avar], body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, @@ -326,13 +346,13 @@ call(#c_call{anno=Anno}, lists, mapfoldr, [Arg1,Arg2,Arg3]) -> Avar = #c_var{name='A'}, Match = fun (A, P, E) -> - C1 = #c_clause{pats=[P], guard=#c_literal{val=true}, body=E}, + C1 = #c_clause{anno=Anno, pats=[P], guard=#c_literal{val=true}, body=E}, Err = #c_tuple{es=[#c_literal{val='badmatch'}, X]}, - C2 = #c_clause{pats=[X], guard=#c_literal{val=true}, + C2 = #c_clause{anno=Anno, pats=[X], guard=#c_literal{val=true}, body=match_fail(Anno, Err)}, #c_case{arg=A, clauses=[C1, C2]} end, - C1 = #c_clause{pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, + C1 = #c_clause{anno=Anno, pats=[#c_cons{hd=X, tl=Xs}], guard=#c_literal{val=true}, %%% Tuple passing version body=Match(#c_apply{anno=Anno, op=Loop, @@ -352,7 +372,8 @@ call(#c_call{anno=Anno}, lists, mapfoldr, [Arg1,Arg2,Arg3]) -> %%% #c_values{es=[#c_cons{hd=X, tl=Xs}, %%% A]})} }, - C2 = #c_clause{pats=[#c_literal{val=[]}], + C2 = #c_clause{anno=Anno, + pats=[#c_literal{val=[]}], guard=#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}, args=[F, #c_literal{val=2}]}, @@ -362,7 +383,7 @@ call(#c_call{anno=Anno}, lists, mapfoldr, [Arg1,Arg2,Arg3]) -> %%% Multiple-value version %%% body=#c_values{es=[#c_literal{val=[]}, A]}}, Err = #c_tuple{es=[#c_literal{val='function_clause'}, F, Avar, Xs]}, - C3 = #c_clause{pats=[Xs], guard=#c_literal{val=true}, + C3 = #c_clause{anno=Anno, pats=[Xs], guard=#c_literal{val=true}, body=match_fail([{function_name,{'lists^mapfoldr',2}}|Anno], Err)}, Fun = #c_fun{vars=[Xs, Avar], body=#c_case{arg=Xs, clauses=[C1, C2, C3]}}, diff --git a/lib/compiler/src/sys_core_inline.erl b/lib/compiler/src/sys_core_inline.erl index 5a6cc45e4a..3380e3f1bd 100644 --- a/lib/compiler/src/sys_core_inline.erl +++ b/lib/compiler/src/sys_core_inline.erl @@ -195,6 +195,9 @@ kill_id_anns(Body) -> cerl_trees:map(fun(#c_fun{anno=A0}=CFun) -> A = kill_id_anns_1(A0), CFun#c_fun{anno=A}; + (#c_var{anno=A0}=Var) -> + A = kill_id_anns_1(A0), + Var#c_var{anno=A}; (Expr) -> %% Mark everything as compiler generated to %% suppress bogus warnings. diff --git a/lib/compiler/src/v3_codegen.erl b/lib/compiler/src/v3_codegen.erl deleted file mode 100644 index d7a7778740..0000000000 --- a/lib/compiler/src/v3_codegen.erl +++ /dev/null @@ -1,2791 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1999-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% -%% Purpose : Code generator for Beam. - --module(v3_codegen). - -%% The main interface. --export([module/2]). - --import(lists, [member/2,keymember/3,keysort/2,keydelete/3, - append/1,flatmap/2,filter/2,foldl/3,foldr/3,mapfoldl/3, - sort/1,reverse/1,reverse/2,map/2]). --import(ordsets, [add_element/2,intersection/2,union/2]). - --include("v3_kernel.hrl"). - -%% These are not defined in v3_kernel.hrl. -get_kanno(Kthing) -> element(2, Kthing). -set_kanno(Kthing, Anno) -> setelement(2, Kthing, Anno). - -%% Main codegen structure. --record(cg, {lcount=1, %Label counter - bfail, %Fail label for BIFs - break, %Break label - recv, %Receive label - is_top_block, %Boolean: top block or not - functable=#{}, %Map of local functions: {Name,Arity}=>Label - in_catch=false, %Inside a catch or not. - need_frame, %Need a stack frame. - ultimate_failure, %Label for ultimate match failure. - ctx %Match context. - }). - -%% Stack/register state record. --record(sr, {reg=[], %Register table - stk=[], %Stack table - res=[]}). %Registers to reserve - -%% Internal records. --record(cg_need_heap, {anno=[] :: term(), - h=0 :: integer()}). --record(cg_block, {anno=[] :: term(), - es=[] :: [term()]}). - --type vdb_entry() :: {atom(),non_neg_integer(),non_neg_integer()}. - --record(l, {i=0 :: non_neg_integer(), %Op number - vdb=[] :: [vdb_entry()], %Variable database - a=[] :: [term()]}). %Core annotation - --spec module(#k_mdef{}, [compile:option()]) -> {'ok',beam_asm:module_code()}. - -module(#k_mdef{name=Mod,exports=Es,attributes=Attr,body=Forms}, _Opts) -> - {Asm,St} = functions(Forms, {atom,Mod}), - {ok,{Mod,Es,Attr,Asm,St#cg.lcount}}. - -functions(Forms, AtomMod) -> - mapfoldl(fun (F, St) -> function(F, AtomMod, St) end, #cg{lcount=1}, Forms). - -function(#k_fdef{anno=#k{a=Anno},func=Name,arity=Arity, - vars=As,body=Kb}, AtomMod, St0) -> - try - #k_match{} = Kb, %Assertion. - - %% Annotate kernel records with variable usage. - Vdb0 = init_vars(As), - {Body,_,Vdb} = body(Kb, 1, Vdb0), - - %% Generate the BEAM assembly code. - {Asm,EntryLabel,St} = cg_fun(Body, As, Vdb, AtomMod, - {Name,Arity}, Anno, St0), - Func = {function,Name,Arity,EntryLabel,Asm}, - {Func,St} - catch - Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [Name,Arity]), - erlang:raise(Class, Error, Stack) - end. - -%% This pass creates beam format annotated with variable lifetime -%% information. Each thing is given an index and for each variable we -%% store the first and last index for its occurrence. The variable -%% database, VDB, attached to each thing is only relevant internally -%% for that thing. -%% -%% For nested things like matches the numbering continues locally and -%% the VDB for that thing refers to the variable usage within that -%% thing. Variables which live through a such a thing are internally -%% given a very large last index. Internally the indexes continue -%% after the index of that thing. This creates no problems as the -%% internal variable info never escapes and externally we only see -%% variable which are alive both before or after. -%% -%% This means that variables never "escape" from a thing and the only -%% way to get values from a thing is to "return" them, with 'break' or -%% 'return'. Externally these values become the return values of the -%% thing. This is no real limitation as most nested things have -%% multiple threads so working out a common best variable usage is -%% difficult. - -%% body(Kbody, I, Vdb) -> {[Expr],MaxI,Vdb}. -%% Handle a body. - -body(#k_seq{arg=Ke,body=Kb}, I, Vdb0) -> - %%ok = io:fwrite("life ~w:~p~n", [?LINE,{Ke,I,Vdb0}]), - A = get_kanno(Ke), - Vdb1 = use_vars(union(A#k.us, A#k.ns), I, Vdb0), - {Es,MaxI,Vdb2} = body(Kb, I+1, Vdb1), - E = expr(Ke, I, Vdb2), - {[E|Es],MaxI,Vdb2}; -body(Ke, I, Vdb0) -> - %%ok = io:fwrite("life ~w:~p~n", [?LINE,{Ke,I,Vdb0}]), - A = get_kanno(Ke), - Vdb1 = use_vars(union(A#k.us, A#k.ns), I, Vdb0), - E = expr(Ke, I, Vdb1), - {[E],I,Vdb1}. - -%% expr(Kexpr, I, Vdb) -> Expr. - -expr(#k_test{anno=A}=Test, I, _Vdb) -> - Test#k_test{anno=#l{i=I,a=A#k.a}}; -expr(#k_call{anno=A}=Call, I, _Vdb) -> - Call#k_call{anno=#l{i=I,a=A#k.a}}; -expr(#k_enter{anno=A}=Enter, I, _Vdb) -> - Enter#k_enter{anno=#l{i=I,a=A#k.a}}; -expr(#k_bif{anno=A}=Bif, I, _Vdb) -> - Bif#k_bif{anno=#l{i=I,a=A#k.a}}; -expr(#k_match{anno=A,body=Kb,ret=Rs}, I, Vdb) -> - %% Work out imported variables which need to be locked. - Mdb = vdb_sub(I, I+1, Vdb), - M = match(Kb, A#k.us, I+1, Mdb), - L = #l{i=I,vdb=use_vars(A#k.us, I+1, Mdb),a=A#k.a}, - #k_match{anno=L,body=M,ret=Rs}; -expr(#k_guard_match{anno=A,body=Kb,ret=Rs}, I, Vdb) -> - %% Work out imported variables which need to be locked. - Mdb = vdb_sub(I, I+1, Vdb), - M = match(Kb, A#k.us, I+1, Mdb), - L = #l{i=I,vdb=use_vars(A#k.us, I+1, Mdb),a=A#k.a}, - #k_guard_match{anno=L,body=M,ret=Rs}; -expr(#k_protected{}=Protected, I, Vdb) -> - protected(Protected, I, Vdb); -expr(#k_try{anno=A,arg=Ka,vars=Vs,body=Kb,evars=Evs,handler=Kh}=Try, I, Vdb) -> - %% Lock variables that are alive before the catch and used afterwards. - %% Don't lock variables that are only used inside the try. - Tdb0 = vdb_sub(I, I+1, Vdb), - %% This is the tricky bit. Lock variables in Arg that are used in - %% the body and handler. Add try tag 'variable'. - Ab = get_kanno(Kb), - Ah = get_kanno(Kh), - Tdb1 = use_vars(union(Ab#k.us, Ah#k.us), I+3, Tdb0), - Tdb2 = vdb_sub(I, I+2, Tdb1), - Vnames = fun (Kvar) -> Kvar#k_var.name end, %Get the variable names - {Aes,_,Adb} = body(Ka, I+2, add_var({catch_tag,I+1}, I+1, locked, Tdb2)), - {Bes,_,Bdb} = body(Kb, I+4, new_vars(sort(map(Vnames, Vs)), I+3, Tdb2)), - {Hes,_,Hdb} = body(Kh, I+4, new_vars(sort(map(Vnames, Evs)), I+3, Tdb2)), - L = #l{i=I,vdb=Tdb1,a=A#k.a}, - Try#k_try{anno=L, - arg=#cg_block{es=Aes,anno=#l{i=I+1,vdb=Adb,a=[]}}, - vars=Vs,body=#cg_block{es=Bes,anno=#l{i=I+3,vdb=Bdb,a=[]}}, - evars=Evs,handler=#cg_block{es=Hes,anno=#l{i=I+3,vdb=Hdb,a=[]}}}; -expr(#k_try_enter{anno=A,arg=Ka,vars=Vs,body=Kb,evars=Evs,handler=Kh}, I, Vdb) -> - %% Lock variables that are alive before the catch and used afterwards. - %% Don't lock variables that are only used inside the try. - Tdb0 = vdb_sub(I, I+1, Vdb), - %% This is the tricky bit. Lock variables in Arg that are used in - %% the body and handler. Add try tag 'variable'. - Ab = get_kanno(Kb), - Ah = get_kanno(Kh), - Tdb1 = use_vars(union(Ab#k.us, Ah#k.us), I+3, Tdb0), - Tdb2 = vdb_sub(I, I+2, Tdb1), - Vnames = fun (Kvar) -> Kvar#k_var.name end, %Get the variable names - {Aes,_,Adb} = body(Ka, I+2, add_var({catch_tag,I+1}, I+1, 1000000, Tdb2)), - {Bes,_,Bdb} = body(Kb, I+4, new_vars(sort(map(Vnames, Vs)), I+3, Tdb2)), - {Hes,_,Hdb} = body(Kh, I+4, new_vars(sort(map(Vnames, Evs)), I+3, Tdb2)), - L = #l{i=I,vdb=Tdb1,a=A#k.a}, - #k_try_enter{anno=L, - arg=#cg_block{es=Aes,anno=#l{i=I+1,vdb=Adb,a=[]}}, - vars=Vs,body=#cg_block{es=Bes,anno=#l{i=I+3,vdb=Bdb,a=[]}}, - evars=Evs,handler=#cg_block{es=Hes,anno=#l{i=I+3,vdb=Hdb,a=[]}}}; -expr(#k_catch{anno=A,body=Kb}=Catch, I, Vdb) -> - %% Lock variables that are alive before the catch and used afterwards. - %% Don't lock variables that are only used inside the catch. - %% Add catch tag 'variable'. - Cdb0 = vdb_sub(I, I+1, Vdb), - {Es,_,Cdb1} = body(Kb, I+1, add_var({catch_tag,I}, I, locked, Cdb0)), - L = #l{i=I,vdb=Cdb1,a=A#k.a}, - Catch#k_catch{anno=L,body=#cg_block{es=Es}}; -expr(#k_receive{anno=A,var=V,body=Kb,action=Ka}=Recv, I, Vdb) -> - %% Work out imported variables which need to be locked. - Rdb = vdb_sub(I, I+1, Vdb), - M = match(Kb, add_element(V#k_var.name, A#k.us), I+1, - new_vars([V#k_var.name], I, Rdb)), - {Tes,_,Adb} = body(Ka, I+1, Rdb), - Le = #l{i=I,vdb=use_vars(A#k.us, I+1, Vdb),a=A#k.a}, - Recv#k_receive{anno=Le,body=M, - action=#cg_block{anno=#l{i=I+1,vdb=Adb,a=[]},es=Tes}}; -expr(#k_receive_accept{anno=A}, I, _Vdb) -> - #k_receive_accept{anno=#l{i=I,a=A#k.a}}; -expr(#k_receive_next{anno=A}, I, _Vdb) -> - #k_receive_next{anno=#l{i=I,a=A#k.a}}; -expr(#k_put{anno=A}=Put, I, _Vdb) -> - Put#k_put{anno=#l{i=I,a=A#k.a}}; -expr(#k_break{anno=A}=Break, I, _Vdb) -> - Break#k_break{anno=#l{i=I,a=A#k.a}}; -expr(#k_guard_break{anno=A}=Break, I, _Vdb) -> - Break#k_guard_break{anno=#l{i=I,a=A#k.a}}; -expr(#k_return{anno=A}=Ret, I, _Vdb) -> - Ret#k_return{anno=#l{i=I,a=A#k.a}}. - -%% protected(Kprotected, I, Vdb) -> Protected. -%% Only used in guards. - -protected(#k_protected{anno=A,arg=Ts}=Prot, I, Vdb) -> - %% Lock variables that are alive before try and used afterwards. - %% Don't lock variables that are only used inside the protected - %% expression. - Pdb0 = vdb_sub(I, I+1, Vdb), - {T,MaxI,Pdb1} = body(Ts, I+1, Pdb0), - Pdb2 = use_vars(A#k.ns, MaxI+1, Pdb1), %Save "return" values - Prot#k_protected{arg=T,anno=#l{i=I,a=A#k.a,vdb=Pdb2}}. - -%% match(Kexpr, [LockVar], I, Vdb) -> Expr. -%% Convert match tree to old format. - -match(#k_alt{anno=A,first=Kf,then=Kt}, Ls, I, Vdb0) -> - Vdb1 = use_vars(union(A#k.us, Ls), I, Vdb0), - F = match(Kf, Ls, I+1, Vdb1), - T = match(Kt, Ls, I+1, Vdb1), - #k_alt{anno=[],first=F,then=T}; -match(#k_select{anno=A,types=Kts}=Select, Ls, I, Vdb0) -> - Vdb1 = use_vars(union(A#k.us, Ls), I, Vdb0), - Ts = [type_clause(Tc, Ls, I+1, Vdb1) || Tc <- Kts], - Select#k_select{anno=[],types=Ts}; -match(#k_guard{anno=A,clauses=Kcs}, Ls, I, Vdb0) -> - Vdb1 = use_vars(union(A#k.us, Ls), I, Vdb0), - Cs = [guard_clause(G, Ls, I+1, Vdb1) || G <- Kcs], - #k_guard{anno=[],clauses=Cs}; -match(Other, Ls, I, Vdb0) -> - Vdb1 = use_vars(Ls, I, Vdb0), - {B,_,Vdb2} = body(Other, I+1, Vdb1), - Le = #l{i=I,vdb=Vdb2,a=[]}, - #cg_block{anno=Le,es=B}. - -type_clause(#k_type_clause{anno=A,type=T,values=Kvs}, Ls, I, Vdb0) -> - %%ok = io:format("life ~w: ~p~n", [?LINE,{T,Kvs}]), - Vdb1 = use_vars(union(A#k.us, Ls), I+1, Vdb0), - Vs = [val_clause(Vc, Ls, I+1, Vdb1) || Vc <- Kvs], - #k_type_clause{anno=[],type=T,values=Vs}. - -val_clause(#k_val_clause{anno=A,val=V,body=Kb}, Ls0, I, Vdb0) -> - New = (get_kanno(V))#k.ns, - Bus = (get_kanno(Kb))#k.us, - %%ok = io:format("Ls0 = ~p, Used=~p\n New=~p, Bus=~p\n", [Ls0,Used,New,Bus]), - Ls1 = union(intersection(New, Bus), Ls0), %Lock for safety - Vdb1 = use_vars(union(A#k.us, Ls1), I+1, new_vars(New, I, Vdb0)), - B = match(Kb, Ls1, I+1, Vdb1), - Le = #l{i=I,vdb=use_vars(Bus, I+1, Vdb1),a=A#k.a}, - #k_val_clause{anno=Le,val=V,body=B}. - -guard_clause(#k_guard_clause{anno=A,guard=Kg,body=Kb}, Ls, I, Vdb0) -> - Vdb1 = use_vars(union(A#k.us, Ls), I+2, Vdb0), - Gdb = vdb_sub(I+1, I+2, Vdb1), - G = protected(Kg, I+1, Gdb), - B = match(Kb, Ls, I+2, Vdb1), - Le = #l{i=I,vdb=use_vars((get_kanno(Kg))#k.us, I+2, Vdb1),a=A#k.a}, - #k_guard_clause{anno=Le,guard=G,body=B}. - - -%% Here follows the code generator pass. -%% -%% The following assumptions have been made: -%% -%% 1. Matches, i.e. things with {match,M,Ret} wrappers, only return -%% values; no variables are exported. If the match would have returned -%% extra variables then these have been transformed to multiple return -%% values. -%% -%% 2. All BIF's called in guards are gc-safe so there is no need to -%% put thing on the stack in the guard. While this would in principle -%% work it would be difficult to keep track of the stack depth when -%% trimming. -%% -%% The code generation uses variable lifetime information added by -%% the previous pass to save variables, allocate registers and -%% move registers to the stack when necessary. -%% -%% We try to use a consistent variable name scheme throughout. The -%% StackReg record is always called Bef,Int<n>,Aft. - -%% cg_fun([Lkexpr], [HeadVar], Vdb, State) -> {[Ainstr],State} - -cg_fun(Les, Hvs, Vdb, AtomMod, NameArity, Anno, St0) -> - {Fi,St1} = new_label(St0), %FuncInfo label - {Fl,St2} = local_func_label(NameArity, St1), - - %% - %% The pattern matching compiler (in v3_kernel) no longer - %% provides its own catch-all clause, because the - %% call to erlang:exit/1 caused problem when cases were - %% used in guards. Therefore, there may be tests that - %% cannot fail (providing that there is not a bug in a - %% previous optimzation pass), but still need to provide - %% a label (there are instructions, such as is_tuple/2, - %% that do not allow {f,0}). - %% - %% We will generate an ultimate failure label and put it - %% at the end of function, followed by an 'if_end' instruction. - %% Note that and 'if_end' instruction does not need any - %% live x registers, so it will always be safe to jump to - %% it. (We never ever expect the jump to be taken, and in - %% most functions there will never be any references to - %% the label in the first place.) - %% - - {UltimateMatchFail,St3} = new_label(St2), - - %% Create initial stack/register state, clear unused arguments. - Bef = clear_dead(#sr{reg=foldl(fun (#k_var{name=V}, Reg) -> - put_reg(V, Reg) - end, [], Hvs), - stk=[]}, 0, Vdb), - {B,_Aft,St} = cg_list(Les, Vdb, Bef, - St3#cg{bfail=0, - ultimate_failure=UltimateMatchFail, - is_top_block=true}), - {Name,Arity} = NameArity, - Asm = [{label,Fi},line(Anno),{func_info,AtomMod,{atom,Name},Arity}, - {label,Fl}|B++[{label,UltimateMatchFail},if_end]], - {Asm,Fl,St}. - -%% cg(Lkexpr, Vdb, StackReg, State) -> {[Ainstr],StackReg,State}. -%% Generate code for a kexpr. - -cg(#cg_block{anno=Le,es=Es}, Vdb, Bef, St) -> - block_cg(Es, Le, Vdb, Bef, St); -cg(#k_match{anno=Le,body=M,ret=Rs}, Vdb, Bef, St) -> - match_cg(M, Rs, Le, Vdb, Bef, St); -cg(#k_guard_match{anno=Le,body=M,ret=Rs}, Vdb, Bef, St) -> - guard_match_cg(M, Rs, Le, Vdb, Bef, St); -cg(#k_call{anno=Le,op=Func,args=As,ret=Rs}, Vdb, Bef, St) -> - call_cg(Func, As, Rs, Le, Vdb, Bef, St); -cg(#k_enter{anno=Le,op=Func,args=As}, Vdb, Bef, St) -> - enter_cg(Func, As, Le, Vdb, Bef, St); -cg(#k_bif{anno=Le}=Bif, Vdb, Bef, St) -> - bif_cg(Bif, Le, Vdb, Bef, St); -cg(#k_receive{anno=Le,timeout=Te,var=Rvar,body=Rm,action=Tes,ret=Rs}, - Vdb, Bef, St) -> - recv_loop_cg(Te, Rvar, Rm, Tes, Rs, Le, Vdb, Bef, St); -cg(#k_receive_next{anno=Le}, Vdb, Bef, St) -> - recv_next_cg(Le, Vdb, Bef, St); -cg(#k_receive_accept{}, _Vdb, Bef, St) -> - {[remove_message],Bef,St}; -cg(#k_try{anno=Le,arg=Ta,vars=Vs,body=Tb,evars=Evs,handler=Th,ret=Rs}, - Vdb, Bef, St) -> - try_cg(Ta, Vs, Tb, Evs, Th, Rs, Le, Vdb, Bef, St); -cg(#k_try_enter{anno=Le,arg=Ta,vars=Vs,body=Tb,evars=Evs,handler=Th}, - Vdb, Bef, St) -> - try_enter_cg(Ta, Vs, Tb, Evs, Th, Le, Vdb, Bef, St); -cg(#k_catch{anno=Le,body=Cb,ret=[R]}, Vdb, Bef, St) -> - catch_cg(Cb, R, Le, Vdb, Bef, St); -cg(#k_put{anno=Le,arg=Con,ret=Var}, Vdb, Bef, St) -> - put_cg(Var, Con, Le, Vdb, Bef, St); -cg(#k_return{anno=Le,args=Rs}, Vdb, Bef, St) -> - return_cg(Rs, Le, Vdb, Bef, St); -cg(#k_break{anno=Le,args=Bs}, Vdb, Bef, St) -> - break_cg(Bs, Le, Vdb, Bef, St); -cg(#k_guard_break{anno=Le,args=Bs}, Vdb, Bef, St) -> - guard_break_cg(Bs, Le, Vdb, Bef, St); -cg(#cg_need_heap{h=H}, _Vdb, Bef, St) -> - {[{test_heap,H,max_reg(Bef#sr.reg)}],Bef,St}. - -%% cg_list([Kexpr], FirstI, Vdb, StackReg, St) -> {[Ainstr],StackReg,St}. - -cg_list(Kes, Vdb, Bef, St0) -> - {Keis,{Aft,St1}} = - flatmapfoldl(fun (Ke, {Inta,Sta}) -> - {Keis,Intb,Stb} = cg(Ke, Vdb, Inta, Sta), - {Keis,{Intb,Stb}} - end, {Bef,St0}, need_heap(Kes)), - {Keis,Aft,St1}. - -%% need_heap([Lkexpr], I, St) -> [Lkexpr]. -%% Insert need_heap instructions in Kexpr list. Try to be smart and -%% collect them together as much as possible. - -need_heap(Kes0) -> - {Kes,H} = need_heap_0(reverse(Kes0), 0, []), - - %% Prepend need_heap if necessary. - need_heap_need(H) ++ Kes. - -need_heap_0([Ke|Kes], H0, Acc) -> - {Ns,H} = need_heap_1(Ke, H0), - need_heap_0(Kes, H, [Ke|Ns]++Acc); -need_heap_0([], H, Acc) -> - {Acc,H}. - -need_heap_1(#k_put{arg=#k_binary{}}, H) -> - {need_heap_need(H),0}; -need_heap_1(#k_put{arg=#k_map{}}, H) -> - {need_heap_need(H),0}; -need_heap_1(#k_put{arg=Val}, H) -> - %% Just pass through adding to needed heap. - {[],H + case Val of - #k_cons{} -> 2; - #k_tuple{es=Es} -> 1 + length(Es); - _Other -> 0 - end}; -need_heap_1(#k_bif{}=Bif, H) -> - case is_gc_bif(Bif) of - false -> - {[],H}; - true -> - {need_heap_need(H),0} - end; -need_heap_1(_Ke, H) -> - %% Call or call-like instruction such as set_tuple_element/3. - {need_heap_need(H),0}. - -need_heap_need(0) -> []; -need_heap_need(H) -> [#cg_need_heap{h=H}]. - -%% is_gc_bif(#k_bif{}) -> true|false. -%% is_gc_bif(Name, Arity) -> true|false. -%% Determines whether the BIF Name/Arity might do a GC. - -is_gc_bif(#k_bif{op=#k_remote{name=#k_atom{val=Name}},args=Args}) -> - is_gc_bif(Name, length(Args)); -is_gc_bif(#k_bif{op=#k_internal{}}) -> - true. - -is_gc_bif(hd, 1) -> false; -is_gc_bif(tl, 1) -> false; -is_gc_bif(self, 0) -> false; -is_gc_bif(node, 0) -> false; -is_gc_bif(node, 1) -> false; -is_gc_bif(element, 2) -> false; -is_gc_bif(get, 1) -> false; -is_gc_bif(tuple_size, 1) -> false; -is_gc_bif(map_get, 2) -> false; -is_gc_bif(is_map_key, 2) -> false; -is_gc_bif(Bif, Arity) -> - not (erl_internal:bool_op(Bif, Arity) orelse - erl_internal:new_type_test(Bif, Arity) orelse - erl_internal:comp_op(Bif, Arity)). - -%% match_cg(Matc, [Ret], Le, Vdb, StackReg, State) -> -%% {[Ainstr],StackReg,State}. -%% Generate code for a match. First save all variables on the stack -%% that are to survive after the match. We leave saved variables in -%% their registers as they might actually be in the right place. - -match_cg(M, Rs, Le, Vdb, Bef, St0) -> - I = Le#l.i, - {Sis,Int0} = adjust_stack(Bef, I, I+1, Vdb), - {B,St1} = new_label(St0), - {Mis,Int1,St2} = match_cg(M, St1#cg.ultimate_failure, - Int0, St1#cg{break=B}), - %% Put return values in registers. - Reg = load_vars(Rs, Int1#sr.reg), - {Sis ++ Mis ++ [{label,B}], - clear_dead(Int1#sr{reg=Reg}, I, Vdb), - St2#cg{break=St1#cg.break}}. - -guard_match_cg(M, Rs, Le, Vdb, Bef, St0) -> - I = Le#l.i, - {B,St1} = new_label(St0), - Fail = case St0 of - #cg{bfail=0,ultimate_failure=Fail0} -> Fail0; - #cg{bfail=Fail0} -> Fail0 - end, - {Mis,Aft,St2} = match_cg(M, Fail, Bef, St1#cg{break=B}), - %% Update the register descriptors for the return registers. - Reg = guard_match_regs(Aft#sr.reg, Rs), - {Mis ++ [{label,B}], - clear_dead(Aft#sr{reg=Reg}, I, Vdb), - St2#cg{break=St1#cg.break}}. - -guard_match_regs([{I,gbreakvar}|Rs], [#k_var{name=V}|Vs]) -> - [{I,V}|guard_match_regs(Rs, Vs)]; -guard_match_regs([R|Rs], Vs) -> - [R|guard_match_regs(Rs, Vs)]; -guard_match_regs([], []) -> []. - - -%% match_cg(Match, Fail, StackReg, State) -> {[Ainstr],StackReg,State}. -%% Generate code for a match tree. N.B. there is no need pass Vdb -%% down as each level which uses this takes its own internal Vdb not -%% the outer one. - -match_cg(#k_alt{first=F,then=S}, Fail, Bef, St0) -> - {Tf,St1} = new_label(St0), - {Fis,Faft,St2} = match_cg(F, Tf, Bef, St1), - {Sis,Saft,St3} = match_cg(S, Fail, Bef, St2), - Aft = sr_merge(Faft, Saft), - {Fis ++ [{label,Tf}] ++ Sis,Aft,St3}; -match_cg(#k_select{var=#k_var{anno=Vanno,name=Vname}=V,types=Scs0}, Fail, Bef, St) -> - ReuseForContext = member(reuse_for_context, Vanno) andalso - find_reg(Vname, Bef#sr.reg) =/= error, - Scs = case ReuseForContext of - false -> Scs0; - true -> bsm_rename_ctx(Scs0, Vname) - end, - match_fmf(fun (S, F, Sta) -> - select_cg(S, V, F, Fail, Bef, Sta) end, - Fail, St, Scs); -match_cg(#k_guard{clauses=Gcs}, Fail, Bef, St) -> - match_fmf(fun (G, F, Sta) -> guard_clause_cg(G, F, Bef, Sta) end, - Fail, St, Gcs); -match_cg(#cg_block{anno=Le,es=Es}, _Fail, Bef, St) -> - %% Must clear registers and stack of dead variables. - Int = clear_dead(Bef, Le#l.i, Le#l.vdb), - block_cg(Es, Le, Int, St). - -%% bsm_rename_ctx([Clause], Var) -> [Clause] -%% We know from an annotation that the register for a binary can -%% be reused for the match context because the two are not truly -%% alive at the same time (even though the life time information -%% says so). -%% -%% The easiest way to have those variables share the same register is -%% to rename the variable with the shortest life-span (the match -%% context) to the variable for the binary (which can have a very -%% long life-time because it is locked during matching). We KNOW that -%% the match state variable will only be alive during the matching. -%% -%% We must also remove all information about the match context -%% variable from all life-time information databases (Vdb). - -bsm_rename_ctx([#k_type_clause{type=k_binary,values=Vcs}=TC|Cs], New) -> - [#k_val_clause{val=#k_binary{segs=#k_var{name=Old}}=Bin, - body=Ke0}=VC0] = Vcs, - Ke = bsm_rename_ctx(Ke0, Old, New, false), - VC = VC0#k_val_clause{val=Bin#k_binary{segs=#k_var{name=New}}, - body=Ke}, - [TC#k_type_clause{values=[VC]}|bsm_rename_ctx(Cs, New)]; -bsm_rename_ctx([C|Cs], New) -> - [C|bsm_rename_ctx(Cs, New)]; -bsm_rename_ctx([], _) -> []. - -%% bsm_rename_ctx(Ke, OldName, NewName, InProt) -> Ke' -%% Rename and clear OldName from life-time information. We must -%% recurse into any block contained in a protected, but it would -%% only complicatate things to recurse into blocks not in a protected -%% (the match context variable is not live inside them). - -bsm_rename_ctx(#k_select{var=#k_var{name=V},types=Cs0}=Sel, - Old, New, InProt) -> - Cs = bsm_rename_ctx_list(Cs0, Old, New, InProt), - Sel#k_select{var=#k_var{name=bsm_rename_var(V, Old, New)},types=Cs}; -bsm_rename_ctx(#k_type_clause{values=Cs0}=TC, Old, New, InProt) -> - Cs = bsm_rename_ctx_list(Cs0, Old, New, InProt), - TC#k_type_clause{values=Cs}; -bsm_rename_ctx(#k_val_clause{body=Ke0}=VC, Old, New, InProt) -> - Ke = bsm_rename_ctx(Ke0, Old, New, InProt), - VC#k_val_clause{body=Ke}; -bsm_rename_ctx(#k_alt{first=F0,then=S0}=Alt, Old, New, InProt) -> - F = bsm_rename_ctx(F0, Old, New, InProt), - S = bsm_rename_ctx(S0, Old, New, InProt), - Alt#k_alt{first=F,then=S}; -bsm_rename_ctx(#k_guard{clauses=Gcs0}=Guard, Old, New, InProt) -> - Gcs = bsm_rename_ctx_list(Gcs0, Old, New, InProt), - Guard#k_guard{clauses=Gcs}; -bsm_rename_ctx(#k_guard_clause{guard=G0,body=B0}=GC, Old, New, InProt) -> - G = bsm_rename_ctx(G0, Old, New, InProt), - B = bsm_rename_ctx(B0, Old, New, InProt), - %% A guard clause may cause unsaved variables to be saved on the stack. - %% Since the match state variable Old is an alias for New (uses the - %% same register), it is neither in the stack nor register descriptor - %% lists and we would crash when we didn't find it unless we remove - %% it from the database. - bsm_forget_var(GC#k_guard_clause{guard=G,body=B}, Old); -bsm_rename_ctx(#k_protected{arg=Ts0}=Prot, Old, New, _InProt) -> - InProt = true, - Ts = bsm_rename_ctx_list(Ts0, Old, New, InProt), - bsm_forget_var(Prot#k_protected{arg=Ts}, Old); -bsm_rename_ctx(#k_guard_match{body=Ms0}=Match, Old, New, InProt) -> - Ms = bsm_rename_ctx(Ms0, Old, New, InProt), - Match#k_guard_match{body=Ms}; -bsm_rename_ctx(#k_test{}=Test, _, _, _) -> Test; -bsm_rename_ctx(#k_bif{}=Bif, _, _, _) -> Bif; -bsm_rename_ctx(#k_put{}=Put, _, _, _) -> Put; -bsm_rename_ctx(#k_call{}=Call, _, _, _) -> Call; -bsm_rename_ctx(#cg_block{}=Block, Old, _, false) -> - %% This block is not inside a protected. The match context variable cannot - %% possibly be live inside the block. - bsm_forget_var(Block, Old); -bsm_rename_ctx(#cg_block{es=Es0}=Block, Old, New, true) -> - %% A block in a protected. We must recursively rename the variable - %% inside the block. - Es = bsm_rename_ctx_list(Es0, Old, New, true), - bsm_forget_var(Block#cg_block{es=Es}, Old); -bsm_rename_ctx(#k_guard_break{}=Break, Old, _New, _InProt) -> - bsm_forget_var(Break, Old). - -bsm_rename_ctx_list([C|Cs], Old, New, InProt) -> - [bsm_rename_ctx(C, Old, New, InProt)| - bsm_rename_ctx_list(Cs, Old, New, InProt)]; -bsm_rename_ctx_list([], _, _, _) -> []. - -bsm_rename_var(Old, Old, New) -> New; -bsm_rename_var(V, _, _) -> V. - -%% bsm_forget_var(#l{}, Variable) -> #l{} -%% Remove a variable from the variable life-time database. - -bsm_forget_var(Ke, V) -> - #l{vdb=Vdb} = L0 = get_kanno(Ke), - L = L0#l{vdb=keydelete(V, 1, Vdb)}, - set_kanno(Ke, L). - -%% block_cg([Kexpr], Le, Vdb, StackReg, St) -> {[Ainstr],StackReg,St}. -%% block_cg([Kexpr], Le, StackReg, St) -> {[Ainstr],StackReg,St}. - -block_cg(Es, Le, _Vdb, Bef, St) -> - block_cg(Es, Le, Bef, St). - -block_cg(Es, Le, Bef, #cg{is_top_block=false}=St) -> - cg_block(Es, Le#l.vdb, Bef, St); -block_cg(Es, Le, Bef, #cg{is_top_block=true}=St0) -> - %% No stack frame has been established yet. Do we need one? - case need_stackframe(Es) of - true -> - %% We need a stack frame. Generate the code and add the - %% code for creating and deallocating the stack frame. - {Is0,Aft,St} = cg_block(Es, Le#l.vdb, Bef, - St0#cg{is_top_block=false,need_frame=false}), - Is = top_level_block(Is0, Aft, max_reg(Bef#sr.reg), St), - {Is,Aft,St#cg{is_top_block=true}}; - false -> - %% This sequence of instructions ending in a #k_match{} (a - %% 'case' or 'if') in the Erlang code does not need a - %% stack frame yet. Delay the creation (if a stack frame - %% is needed at all, it will be created inside the - %% #k_match{}). - cg_list(Es, Le#l.vdb, Bef, St0) - end. - -%% need_stackframe([Kexpr]) -> true|false. -%% Does this list of instructions need a stack frame? -%% -%% A sequence of instructions that don't clobber the X registers -%% followed by a single #k_match{} doesn't need a stack frame. - -need_stackframe([H|T]) -> - case H of - #k_bif{op=#k_internal{}} -> true; - #k_put{arg=#k_binary{}} -> true; - #k_bif{} -> need_stackframe(T); - #k_put{} -> need_stackframe(T); - #k_guard_match{} -> need_stackframe(T); - #k_match{} when T =:= [] -> false; - _ -> true - end; -need_stackframe([]) -> false. - -cg_block([], _Vdb, Bef, St0) -> - {[],Bef,St0}; -cg_block(Kes0, Vdb, Bef, St0) -> - {Kes2,Int1,St1} = - case basic_block(Kes0) of - {Kes1,LastI,Args,Rest} -> - cg_basic_block(Kes1, LastI, Args, Vdb, Bef, St0); - {Kes1,Rest} -> - cg_list(Kes1, Vdb, Bef, St0) - end, - {Kes3,Int2,St2} = cg_block(Rest, Vdb, Int1, St1), - {Kes2 ++ Kes3,Int2,St2}. - -basic_block(Kes) -> basic_block(Kes, []). - -basic_block([Ke|Kes], Acc) -> - case collect_block(Ke) of - include -> basic_block(Kes, [Ke|Acc]); - {block_end,As} -> - case Acc of - [] -> - %% If the basic block does not contain any #k_put{} instructions, - %% it serves no useful purpose to do basic block optimizations. - {[Ke],Kes}; - _ -> - #l{i=I} = get_kanno(Ke), - {reverse(Acc, [Ke]),I,As,Kes} - end; - no_block -> {reverse(Acc, [Ke]),Kes} - end. - -collect_block(#k_put{arg=Arg}) -> - %% #k_put{} instructions that may garbage collect are not allowed - %% in basic blocks. - case Arg of - #k_binary{} -> no_block; - #k_map{} -> no_block; - _ -> include - end; -collect_block(#k_call{op=Func,args=As}) -> - {block_end,As++func_vars(Func)}; -collect_block(#k_enter{op=Func,args=As}) -> - {block_end,As++func_vars(Func)}; -collect_block(#k_return{args=Rs}) -> - {block_end,Rs}; -collect_block(#k_break{args=Bs}) -> - {block_end,Bs}; -collect_block(_) -> no_block. - -func_vars(#k_var{}=Var) -> - [Var]; -func_vars(#k_remote{mod=M,name=F}) - when is_record(M, k_var); is_record(F, k_var) -> - [M,F]; -func_vars(_) -> []. - -%% cg_basic_block([Kexpr], FirstI, LastI, Arguments, Vdb, StackReg, State) -> -%% {[Ainstr],StackReg,State}. -%% -%% Do a specialized code generation for a basic block of #put{} -%% instructions (that don't do any garbage collection) followed by a -%% call, break, or return. -%% -%% 'Arguments' is a list of the variables that must be loaded into -%% consecutive X registers before the last instruction in the block. -%% The point of this specialized code generation is to try put the -%% all of the variables in 'Arguments' into the correct X register -%% to begin with, instead of putting them into the first available -%% X register and having to move them to the correct X register -%% later. -%% -%% To achieve that, we attempt to reserve the X registers that the -%% variables in 'Arguments' will need to be in when the block ends. -%% -%% To make it more likely that reservations will be successful, we -%% will try to save variables that need to be saved to the stack as -%% early as possible (if an X register needed by a variable in -%% Arguments is occupied by another variable, the value in the -%% X register can be evicted if it is saved on the stack). -%% -%% We will take care not to increase the size of stack frame compared -%% to what the standard code generator would have done (that is, to -%% save all X registers at the last possible moment). We will do that -%% by extending the stack frame to the minimal size needed to save -%% all that needs to be saved using extend_stack/4, and use -%% save_carefully/4 during code generation to only save the variables -%% that can be saved without growing the stack frame. - -cg_basic_block(Kes, Lf, As, Vdb, Bef, St0) -> - Int0 = reserve_arg_regs(As, Bef), - Int = extend_stack(Int0, Lf, Lf+1, Vdb), - {Keis,{Aft,St1}} = - flatmapfoldl(fun(Ke, St) -> cg_basic_block(Ke, St, Lf, Vdb) end, - {Int,St0}, need_heap(Kes)), - {Keis,Aft,St1}. - -cg_basic_block(#cg_need_heap{}=Ke, {Bef,St0}, _Lf, Vdb) -> - {Keis,Aft,St1} = cg(Ke, Vdb, Bef, St0), - {Keis,{Aft,St1}}; -cg_basic_block(Ke, {Bef,St0}, Lf, Vdb) -> - #l{i=I} = get_kanno(Ke), - - %% Save all we can to increase the possibility that reserving - %% registers will succeed. - {Sis,Int0} = save_carefully(Bef, I, Lf+1, Vdb), - Int1 = reserve(Int0), - {Keis,Aft,St1} = cg(Ke, Vdb, Int1, St0), - {Sis ++ Keis,{Aft,St1}}. - -%% reserve_arg_regs([Argument], Bef) -> Aft. -%% Try to reserve the X registers for all arguments. All registers -%% that we wish to reserve will be saved in Bef#sr.res. - -reserve_arg_regs(As, Bef) -> - Res = reserve_arg_regs_1(As, 0), - reserve(Bef#sr{res=Res}). - -reserve_arg_regs_1([#k_var{name=V}|As], I) -> - [{I,V}|reserve_arg_regs_1(As, I+1)]; -reserve_arg_regs_1([A|As], I) -> - [{I,A}|reserve_arg_regs_1(As, I+1)]; -reserve_arg_regs_1([], _) -> []. - -%% reserve(Bef) -> Aft. -%% Try to reserve more registers. The registers we wish to reserve -%% are found in Bef#sr.res. - -reserve(#sr{reg=Regs,stk=Stk,res=Res}=Sr) -> - Sr#sr{reg=reserve_1(Res, Regs, Stk)}. - -reserve_1([{I,V}|Rs], [free|Regs], Stk) -> - [{reserved,I,V}|reserve_1(Rs, Regs, Stk)]; -reserve_1([{I,V}|Rs], [{I,V}|Regs], Stk) -> - [{I,V}|reserve_1(Rs, Regs, Stk)]; -reserve_1([{I,V}|Rs], [{I,Var}|Regs], Stk) -> - case on_stack(Var, Stk) of - true -> [{reserved,I,V}|reserve_1(Rs, Regs, Stk)]; - false -> [{I,Var}|reserve_1(Rs, Regs, Stk)] - end; -reserve_1([{I,V}|Rs], [{reserved,I,_}|Regs], Stk) -> - [{reserved,I,V}|reserve_1(Rs, Regs, Stk)]; -reserve_1([{I,V}|Rs], [], Stk) -> - [{reserved,I,V}|reserve_1(Rs, [], Stk)]; -reserve_1([], Regs, _) -> Regs. - -%% extend_stack(Bef, FirstBefore, LastFrom, Vdb) -> Aft. -%% Extend the stack enough to fit all variables alive past LastFrom -%% and not already on the stack. - -extend_stack(#sr{stk=Stk0}=Bef, Fb, Lf, Vdb) -> - Stk1 = clear_dead_stk(Stk0, Fb, Vdb), - New = new_not_on_stack(Stk1, Fb, Lf, Vdb), - Stk2 = foldl(fun ({V,_,_}, Stk) -> put_stack(V, Stk) end, Stk1, New), - Stk = Stk0 ++ lists:duplicate(length(Stk2) - length(Stk0), free), - Bef#sr{stk=Stk}. - -%% save_carefully(Bef, FirstBefore, LastFrom, Vdb) -> {[SaveVar],Aft}. -%% Save variables which are used past current point and which are not -%% already on the stack, but only if the variables can be saved without -%% growing the stack frame. - -save_carefully(#sr{stk=Stk}=Bef, Fb, Lf, Vdb) -> - New0 = new_not_on_stack(Stk, Fb, Lf, Vdb), - New = keysort(2, New0), - save_carefully_1(New, Bef, []). - -save_carefully_1([{V,_,_}|Vs], #sr{reg=Regs,stk=Stk0}=Bef, Acc) -> - case put_stack_carefully(V, Stk0) of - error -> - {reverse(Acc),Bef}; - Stk1 -> - SrcReg = fetch_reg(V, Regs), - Move = {move,SrcReg,fetch_stack(V, Stk1)}, - {x,_} = SrcReg, %Assertion - must be X register. - save_carefully_1(Vs, Bef#sr{stk=Stk1}, [Move|Acc]) - end; -save_carefully_1([], Bef, Acc) -> - {reverse(Acc),Bef}. - -%% top_level_block([Instruction], Bef, MaxRegs, St) -> [Instruction]. -%% For the top-level block, allocate a stack frame a necessary, -%% adjust Y register numbering and instructions that return -%% from the function. - -top_level_block(Keis, #sr{stk=[]}, _MaxRegs, #cg{need_frame=false}) -> - Keis; -top_level_block(Keis, Bef, MaxRegs, _St) -> - %% This top block needs an allocate instruction before it, and a - %% deallocate instruction before each return. - FrameSz = length(Bef#sr.stk), - MaxY = FrameSz-1, - Keis1 = flatmap(fun ({call_only,Arity,Func}) -> - [{call_last,Arity,Func,FrameSz}]; - ({call_ext_only,Arity,Func}) -> - [{call_ext_last,Arity,Func,FrameSz}]; - ({apply_only,Arity}) -> - [{apply_last,Arity,FrameSz}]; - (return) -> - [{deallocate,FrameSz},return]; - (Tuple) when is_tuple(Tuple) -> - [turn_yregs(Tuple, MaxY)]; - (Other) -> - [Other] - end, Keis), - [{allocate_zero,FrameSz,MaxRegs}|Keis1]. - -%% turn_yregs(Size, Tuple, MaxY) -> Tuple' -%% Renumber y register so that {y,0} becomes {y,FrameSize-1}, -%% {y,FrameSize-1} becomes {y,0} and so on. This is to make nested -%% catches work. The code generation algorithm gives a lower register -%% number to the outer catch, which is wrong. - -turn_yregs({call,_,_}=I, _MaxY) -> I; -turn_yregs({call_ext,_,_}=I, _MaxY) -> I; -turn_yregs({jump,_}=I, _MaxY) -> I; -turn_yregs({label,_}=I, _MaxY) -> I; -turn_yregs({line,_}=I, _MaxY) -> I; -turn_yregs({test_heap,_,_}=I, _MaxY) -> I; -turn_yregs({bif,Op,F,A,B}, MaxY) -> - {bif,Op,F,turn_yreg(A, MaxY),turn_yreg(B, MaxY)}; -turn_yregs({gc_bif,Op,F,Live,A,B}, MaxY) when is_integer(Live) -> - {gc_bif,Op,F,Live,turn_yreg(A, MaxY),turn_yreg(B, MaxY)}; -turn_yregs({get_tuple_element,S,N,D}, MaxY) -> - {get_tuple_element,turn_yreg(S, MaxY),N,turn_yreg(D, MaxY)}; -turn_yregs({put_tuple,Arity,D}, MaxY) -> - {put_tuple,Arity,turn_yreg(D, MaxY)}; -turn_yregs({select_val,R,F,L}, MaxY) -> - {select_val,turn_yreg(R, MaxY),F,L}; -turn_yregs({test,Op,F,L}, MaxY) -> - {test,Op,F,turn_yreg(L, MaxY)}; -turn_yregs({test,Op,F,Live,A,B}, MaxY) when is_integer(Live) -> - {test,Op,F,Live,turn_yreg(A, MaxY),turn_yreg(B, MaxY)}; -turn_yregs({Op,A}, MaxY) -> - {Op,turn_yreg(A, MaxY)}; -turn_yregs({Op,A,B}, MaxY) -> - {Op,turn_yreg(A, MaxY),turn_yreg(B, MaxY)}; -turn_yregs({Op,A,B,C}, MaxY) -> - {Op,turn_yreg(A, MaxY),turn_yreg(B, MaxY),turn_yreg(C, MaxY)}; -turn_yregs(Tuple, MaxY) -> - turn_yregs(tuple_size(Tuple), Tuple, MaxY). - -turn_yregs(1, Tp, _) -> - Tp; -turn_yregs(N, Tp, MaxY) -> - E = turn_yreg(element(N, Tp), MaxY), - turn_yregs(N-1, setelement(N, Tp, E), MaxY). - -turn_yreg({yy,YY}, MaxY) -> - {y,MaxY-YY}; -turn_yreg({list,Ls},MaxY) -> - {list,turn_yreg(Ls, MaxY)}; -turn_yreg([_|_]=Ts, MaxY) -> - [turn_yreg(T, MaxY) || T <- Ts]; -turn_yreg(Other, _MaxY) -> - Other. - -%% select_cg(Sclause, V, TypeFail, ValueFail, StackReg, State) -> -%% {Is,StackReg,State}. -%% Selecting type and value needs two failure labels, TypeFail is the -%% label to jump to of the next type test when this type fails, and -%% ValueFail is the label when this type is correct but the value is -%% wrong. These are different as in the second case there is no need -%% to try the next type, it will always fail. - -select_cg(#k_type_clause{type=Type,values=Vs}, Var, Tf, Vf, Bef, St) -> - #k_var{name=V} = Var, - select_cg(Type, Vs, V, Tf, Vf, Bef, St). - -select_cg(k_cons, [S], V, Tf, Vf, Bef, St) -> - select_cons(S, V, Tf, Vf, Bef, St); -select_cg(k_nil, [S], V, Tf, Vf, Bef, St) -> - select_nil(S, V, Tf, Vf, Bef, St); -select_cg(k_binary, [S], V, Tf, Vf, Bef, St) -> - select_binary(S, V, Tf, Vf, Bef, St); -select_cg(k_bin_seg, S, V, Tf, _Vf, Bef, St) -> - select_bin_segs(S, V, Tf, Bef, St); -select_cg(k_bin_int, S, V, Tf, _Vf, Bef, St) -> - select_bin_segs(S, V, Tf, Bef, St); -select_cg(k_bin_end, [S], V, Tf, _Vf, Bef, St) -> - select_bin_end(S, V, Tf, Bef, St); -select_cg(k_map, S, V, Tf, Vf, Bef, St) -> - select_map(S, V, Tf, Vf, Bef, St); -select_cg(k_literal, S, V, Tf, Vf, Bef, St) -> - select_literal(S, V, Tf, Vf, Bef, St); -select_cg(Type, Scs, V, Tf, Vf, Bef, St0) -> - {Vis,{Aft,St1}} = - mapfoldl(fun (S, {Int,Sta}) -> - {Val,Is,Inta,Stb} = select_val(S, V, Vf, Bef, Sta), - {{Is,[Val]},{sr_merge(Int, Inta),Stb}} - end, {void,St0}, Scs), - OptVls = combine(lists:sort(combine(Vis))), - {Vls,Sis,St2} = select_labels(OptVls, St1, [], []), - {select_val_cg(Type, fetch_var(V, Bef), Vls, Tf, Vf, Sis), Aft, St2}. - -select_val_cg(k_tuple, R, [Arity,{f,Lbl}], Tf, Vf, [{label,Lbl}|Sis]) -> - [{test,is_tuple,{f,Tf},[R]},{test,test_arity,{f,Vf},[R,Arity]}|Sis]; -select_val_cg(k_tuple, R, Vls, Tf, Vf, Sis) -> - [{test,is_tuple,{f,Tf},[R]},{select_tuple_arity,R,{f,Vf},{list,Vls}}|Sis]; -select_val_cg(Type, R, [Val, {f,Lbl}], Fail, Fail, [{label,Lbl}|Sis]) -> - [{test,is_eq_exact,{f,Fail},[R,{type(Type),Val}]}|Sis]; -select_val_cg(Type, R, [Val, {f,Lbl}], Tf, Vf, [{label,Lbl}|Sis]) -> - [{test,select_type_test(Type),{f,Tf},[R]}, - {test,is_eq_exact,{f,Vf},[R,{type(Type),Val}]}|Sis]; -select_val_cg(Type, R, Vls0, Tf, Vf, Sis) -> - Vls1 = [case Value of - {f,_Lbl} -> Value; - _ -> {type(Type),Value} - end || Value <- Vls0], - [{test,select_type_test(Type),{f,Tf},[R]}, {select_val,R,{f,Vf},{list,Vls1}}|Sis]. - -type(k_atom) -> atom; -type(k_float) -> float; -type(k_int) -> integer. - -select_type_test(k_int) -> is_integer; -select_type_test(k_atom) -> is_atom; -select_type_test(k_float) -> is_float. - -combine([{Is,Vs1}, {Is,Vs2}|Vis]) -> combine([{Is,Vs1 ++ Vs2}|Vis]); -combine([V|Vis]) -> [V|combine(Vis)]; -combine([]) -> []. - -select_labels([{Is,Vs}|Vis], St0, Vls, Sis) -> - {Lbl,St1} = new_label(St0), - select_labels(Vis, St1, add_vls(Vs, Lbl, Vls), [[{label,Lbl}|Is]|Sis]); -select_labels([], St, Vls, Sis) -> - {Vls,append(Sis),St}. - -add_vls([V|Vs], Lbl, Acc) -> - add_vls(Vs, Lbl, [V, {f,Lbl}|Acc]); -add_vls([], _, Acc) -> Acc. - -select_literal(S, V, Tf, Vf, Bef, St) -> - Reg = fetch_var(V, Bef), - F = fun(ValClause, Fail, St0) -> - {Val,Is,Aft,St1} = select_val(ValClause, V, Vf, Bef, St0), - Test = {test,is_eq_exact,{f,Fail},[Reg,{literal,Val}]}, - {[Test|Is],Aft,St1} - end, - match_fmf(F, Tf, St, S). - -select_cons(#k_val_clause{val=#k_cons{hd=Hd,tl=Tl},body=B,anno=#l{i=I,vdb=Vdb}}, - V, Tf, Vf, Bef, St0) -> - Es = [Hd,Tl], - {Eis,Int,St1} = select_extract_cons(V, Es, I, Vdb, Bef, St0), - {Bis,Aft,St2} = match_cg(B, Vf, Int, St1), - {[{test,is_nonempty_list,{f,Tf},[fetch_var(V, Bef)]}] ++ Eis ++ Bis,Aft,St2}. - -select_nil(#k_val_clause{val=#k_nil{},body=B}, V, Tf, Vf, Bef, St0) -> - {Bis,Aft,St1} = match_cg(B, Vf, Bef, St0), - {[{test,is_nil,{f,Tf},[fetch_var(V, Bef)]}] ++ Bis,Aft,St1}. - -select_binary(#k_val_clause{val=#k_binary{segs=#k_var{name=V}},body=B, - anno=#l{i=I,vdb=Vdb}}, V, Tf, Vf, Bef, St0) -> - #cg{ctx=OldCtx} = St0, - Int0 = clear_dead(Bef#sr{reg=Bef#sr.reg}, I, Vdb), - {Bis0,Aft,St1} = match_cg(B, Vf, Int0, St0#cg{ctx=V}), - CtxReg = fetch_var(V, Int0), - Live = max_reg(Bef#sr.reg), - Bis1 = [{test,bs_start_match2,{f,Tf},Live,[CtxReg,{context,V}],CtxReg}, - {bs_save2,CtxReg,{V,V}}|Bis0], - Bis = finish_select_binary(Bis1), - {Bis,Aft,St1#cg{ctx=OldCtx}}; -select_binary(#k_val_clause{val=#k_binary{segs=#k_var{name=Ivar}},body=B, - anno=#l{i=I,vdb=Vdb}}, V, Tf, Vf, Bef, St0) -> - #cg{ctx=OldCtx} = St0, - Regs = put_reg(Ivar, Bef#sr.reg), - Int0 = clear_dead(Bef#sr{reg=Regs}, I, Vdb), - {Bis0,Aft,St1} = match_cg(B, Vf, Int0, St0#cg{ctx=Ivar}), - CtxReg = fetch_var(Ivar, Int0), - Live = max_reg(Bef#sr.reg), - Bis1 = [{test,bs_start_match2,{f,Tf},Live, - [fetch_var(V, Bef),{context,Ivar}],CtxReg}, - {bs_save2,CtxReg,{Ivar,Ivar}}|Bis0], - Bis = finish_select_binary(Bis1), - {Bis,Aft,St1#cg{ctx=OldCtx}}. - -finish_select_binary([{bs_save2,R,Point}=I,{bs_restore2,R,Point}|Is]) -> - [I|finish_select_binary(Is)]; -finish_select_binary([{bs_save2,R,Point}=I,{test,is_eq_exact,_,_}=Test, - {bs_restore2,R,Point}|Is]) -> - [I,Test|finish_select_binary(Is)]; -finish_select_binary([{test,bs_match_string,F,[Ctx,BinList]}|Is]) - when is_list(BinList) -> - I = {test,bs_match_string,F,[Ctx,list_to_bitstring(BinList)]}, - [I|finish_select_binary(Is)]; -finish_select_binary([I|Is]) -> - [I|finish_select_binary(Is)]; -finish_select_binary([]) -> []. - -%% New instructions for selection of binary segments. - -select_bin_segs(Scs, Ivar, Tf, Bef, St) -> - match_fmf(fun(S, Fail, Sta) -> - select_bin_seg(S, Ivar, Fail, Bef, Sta) end, - Tf, St, Scs). - -select_bin_seg(#k_val_clause{val=#k_bin_seg{size=Size,unit=U,type=T, - seg=Seg,flags=Fs0,next=Next}, - body=B, - anno=#l{i=I,vdb=Vdb,a=A}}, Ivar, Fail, Bef, St0) -> - Ctx = St0#cg.ctx, - Fs = [{anno,A}|Fs0], - Es = case Next of - [] -> [Seg]; - _ -> [Seg,Next] - end, - {Mis,Int,St1} = select_extract_bin(Es, Size, U, T, Fs, Fail, - I, Vdb, Bef, Ctx, B, St0), - {Bis,Aft,St2} = match_cg(B, Fail, Int, St1), - CtxReg = fetch_var(Ctx, Bef), - Is = if - Mis =:= [] -> - %% No bs_restore2 instruction needed if no match instructions. - Bis; - true -> - [{bs_restore2,CtxReg,{Ctx,Ivar}}|Mis++Bis] - end, - {Is,Aft,St2}; -select_bin_seg(#k_val_clause{val=#k_bin_int{size=Sz,unit=U,flags=Fs, - val=Val,next=Next}, - body=B, - anno=#l{i=I,vdb=Vdb}}, Ivar, Fail, Bef, St0) -> - Ctx = St0#cg.ctx, - {Mis,Int,St1} = select_extract_int(Next, Val, Sz, U, Fs, Fail, - I, Vdb, Bef, Ctx, St0), - {Bis,Aft,St2} = match_cg(B, Fail, Int, St1), - CtxReg = fetch_var(Ctx, Bef), - Is = case Mis ++ Bis of - [{test,bs_match_string,F,[OtherCtx,Bin1]}, - {bs_save2,OtherCtx,_}, - {bs_restore2,OtherCtx,_}, - {test,bs_match_string,F,[OtherCtx,Bin2]}|Is0] -> - %% We used to do this optimization later, but it - %% turns out that in huge functions with many - %% bs_match_string instructions, it's a big win - %% to do the combination now. To avoid copying the - %% binary data again and again, we'll combine bitstrings - %% in a list and convert all of it to a bitstring later. - [{test,bs_match_string,F,[OtherCtx,[Bin1,Bin2]]}|Is0]; - Is0 -> - Is0 - end, - {[{bs_restore2,CtxReg,{Ctx,Ivar}}|Is],Aft,St2}. - -select_extract_int(#k_var{name=Tl}, Val, #k_int{val=Sz}, U, Fs, Vf, - I, Vdb, Bef, Ctx, St) -> - Bits = U*Sz, - Bin = case member(big, Fs) of - true -> - <<Val:Bits>>; - false -> - true = member(little, Fs), %Assertion. - <<Val:Bits/little>> - end, - Bits = bit_size(Bin), %Assertion. - CtxReg = fetch_var(Ctx, Bef), - Is = if - Bits =:= 0 -> - [{bs_save2,CtxReg,{Ctx,Tl}}]; - true -> - [{test,bs_match_string,{f,Vf},[CtxReg,Bin]}, - {bs_save2,CtxReg,{Ctx,Tl}}] - end, - {Is,clear_dead(Bef, I, Vdb),St}. - -select_extract_bin([#k_var{name=Hd},#k_var{name=Tl}], Size0, Unit, Type, Flags, Vf, - I, Vdb, Bef, Ctx, _Body, St) -> - SizeReg = get_bin_size_reg(Size0, Bef), - {Es,Aft} = - case vdb_find(Hd, Vdb) of - {_,_,Lhd} when Lhd =< I -> - %% The extracted value will not be used. - CtxReg = fetch_var(Ctx, Bef), - Live = max_reg(Bef#sr.reg), - Skip = build_skip_instr(Type, Vf, CtxReg, Live, - SizeReg, Unit, Flags), - {[Skip,{bs_save2,CtxReg,{Ctx,Tl}}],Bef}; - {_,_,_} -> - Reg = put_reg(Hd, Bef#sr.reg), - Int1 = Bef#sr{reg=Reg}, - Rhd = fetch_reg(Hd, Reg), - CtxReg = fetch_reg(Ctx, Reg), - Live = max_reg(Bef#sr.reg), - {[build_bs_instr(Type, Vf, CtxReg, Live, SizeReg, - Unit, Flags, Rhd), - {bs_save2,CtxReg,{Ctx,Tl}}],Int1} - end, - {Es,clear_dead(Aft, I, Vdb),St}; -select_extract_bin([#k_var{name=Hd}], Size, Unit, binary, Flags, Vf, - I, Vdb, Bef, Ctx, Body, St) -> - %% Match the last segment of a binary. We KNOW that the size - %% must be 'all'. - #k_atom{val=all} = Size, %Assertion. - {Es,Aft} = - case vdb_find(Hd, Vdb) of - {_,_,Lhd} when Lhd =< I -> - %% The result will not be used. Furthermore, since we - %% we are at the end of the binary, the position will - %% not be used again; thus, it is safe to do a cheaper - %% test of the unit. - CtxReg = fetch_var(Ctx, Bef), - {case Unit of - 1 -> - []; - _ -> - [{test,bs_test_unit,{f,Vf},[CtxReg,Unit]}] - end,Bef}; - {_,_,_} -> - case is_context_unused(Body) of - false -> - Reg = put_reg(Hd, Bef#sr.reg), - Int1 = Bef#sr{reg=Reg}, - Rhd = fetch_reg(Hd, Reg), - CtxReg = fetch_reg(Ctx, Reg), - Name = bs_get_binary2, - Live = max_reg(Bef#sr.reg), - {[{test,Name,{f,Vf},Live, - [CtxReg,atomic(Size),Unit,{field_flags,Flags}],Rhd}], - Int1}; - true -> - %% Since the matching context will not be used again, - %% we can reuse its register. Reusing the register - %% opens some interesting optimizations in the - %% run-time system. - - Reg0 = Bef#sr.reg, - CtxReg = fetch_reg(Ctx, Reg0), - Reg = replace_reg_contents(Ctx, Hd, Reg0), - Int1 = Bef#sr{reg=Reg}, - Name = bs_get_binary2, - Live = max_reg(Int1#sr.reg), - {[{test,Name,{f,Vf},Live, - [CtxReg,atomic(Size),Unit,{field_flags,Flags}],CtxReg}], - Int1} - end - end, - {Es,clear_dead(Aft, I, Vdb),St}. - -%% is_context_unused(Ke) -> true | false -%% Simple heurististic to determine whether the code that follows -%% will use the current matching context again. (The liveness -%% information is too conservative to be useful for this purpose.) -%% 'true' means that the code that follows will definitely not use -%% the context again (because it is a block, not guard or matching -%% code); 'false' that we are not sure (there could be more -%% matching). - -is_context_unused(#k_alt{then=Then}) -> - %% #k_alt{} can be used for different purposes. If the Then part - %% is a block, it means that matching has finished and is used for a guard - %% to choose between the matched clauses. - is_context_unused(Then); -is_context_unused(#cg_block{}) -> - true; -is_context_unused(_) -> - false. - -select_bin_end(#k_val_clause{val=#k_bin_end{},body=B}, Ivar, Tf, Bef, St0) -> - Ctx = St0#cg.ctx, - {Bis,Aft,St2} = match_cg(B, Tf, Bef, St0), - CtxReg = fetch_var(Ctx, Bef), - {[{bs_restore2,CtxReg,{Ctx,Ivar}}, - {test,bs_test_tail2,{f,Tf},[CtxReg,0]}|Bis],Aft,St2}. - -get_bin_size_reg(#k_var{name=V}, Bef) -> - fetch_var(V, Bef); -get_bin_size_reg(Literal, _Bef) -> - atomic(Literal). - -build_bs_instr(Type, Vf, CtxReg, Live, SizeReg, Unit, Flags, Rhd) -> - {Format,Name} = case Type of - integer -> {plain,bs_get_integer2}; - float -> {plain,bs_get_float2}; - binary -> {plain,bs_get_binary2}; - utf8 -> {utf,bs_get_utf8}; - utf16 -> {utf,bs_get_utf16}; - utf32 -> {utf,bs_get_utf32} - end, - case Format of - plain -> - {test,Name,{f,Vf},Live, - [CtxReg,SizeReg,Unit,{field_flags,Flags}],Rhd}; - utf -> - {test,Name,{f,Vf},Live, - [CtxReg,{field_flags,Flags}],Rhd} - end. - -build_skip_instr(Type, Vf, CtxReg, Live, SizeReg, Unit, Flags) -> - {Format,Name} = case Type of - utf8 -> {utf,bs_skip_utf8}; - utf16 -> {utf,bs_skip_utf16}; - utf32 -> {utf,bs_skip_utf32}; - _ -> {plain,bs_skip_bits2} - end, - case Format of - plain -> - {test,Name,{f,Vf},[CtxReg,SizeReg,Unit,{field_flags,Flags}]}; - utf -> - {test,Name,{f,Vf},[CtxReg,Live,{field_flags,Flags}]} - end. - -select_val(#k_val_clause{val=#k_tuple{es=Es},body=B,anno=#l{i=I,vdb=Vdb}}, - V, Vf, Bef, St0) -> - {Eis,Int,St1} = select_extract_tuple(V, Es, I, Vdb, Bef, St0), - {Bis,Aft,St2} = match_cg(B, Vf, Int, St1), - {length(Es),Eis ++ Bis,Aft,St2}; -select_val(#k_val_clause{val=Val0,body=B}, _V, Vf, Bef, St0) -> - Val = case Val0 of - #k_atom{val=Lit} -> Lit; - #k_float{val=Lit} -> Lit; - #k_int{val=Lit} -> Lit; - #k_literal{val=Lit} -> Lit - end, - {Bis,Aft,St1} = match_cg(B, Vf, Bef, St0), - {Val,Bis,Aft,St1}. - -%% select_extract_tuple(Src, [V], I, Vdb, StackReg, State) -> -%% {[E],StackReg,State}. -%% Extract tuple elements, but only if they do not immediately die. - -select_extract_tuple(Src, Vs, I, Vdb, Bef, St) -> - F = fun (#k_var{name=V}, {Int0,Elem}) -> - case vdb_find(V, Vdb) of - {V,_,L} when L =< I -> {[], {Int0,Elem+1}}; - _Other -> - Reg1 = put_reg(V, Int0#sr.reg), - Int1 = Int0#sr{reg=Reg1}, - Rsrc = fetch_var(Src, Int1), - {[{get_tuple_element,Rsrc,Elem,fetch_reg(V, Reg1)}], - {Int1,Elem+1}} - end - end, - {Es,{Aft,_}} = flatmapfoldl(F, {Bef,0}, Vs), - {Es,Aft,St}. - -select_map(Scs, V, Tf, Vf, Bef, St0) -> - Reg = fetch_var(V, Bef), - {Is,Aft,St1} = - match_fmf(fun(#k_val_clause{val=#k_map{op=exact,es=Es}, - body=B,anno=#l{i=I,vdb=Vdb}}, Fail, St1) -> - select_map_val(V, Es, B, Fail, I, Vdb, Bef, St1) - end, Vf, St0, Scs), - {[{test,is_map,{f,Tf},[Reg]}|Is],Aft,St1}. - -select_map_val(V, Es, B, Fail, I, Vdb, Bef, St0) -> - {Eis,Int,St1} = select_extract_map(V, Es, Fail, I, Vdb, Bef, St0), - {Bis,Aft,St2} = match_cg(B, Fail, Int, St1), - {Eis++Bis,Aft,St2}. - -select_extract_map(_, [], _, _, _, Bef, St) -> {[],Bef,St}; -select_extract_map(Src, Vs, Fail, I, Vdb, Bef, St) -> - %% First split the instruction flow - %% We want one set of each - %% 1) has_map_fields (no target registers) - %% 2) get_map_elements (with target registers) - %% Assume keys are term-sorted - Rsrc = fetch_var(Src, Bef), - - {{HasKs,GetVs,HasVarKs,GetVarVs},Aft} = - foldr(fun(#k_map_pair{key=#k_var{name=K},val=#k_var{name=V}}, - {{HasKsi,GetVsi,HasVarVsi,GetVarVsi},Int0}) -> - case vdb_find(V, Vdb) of - {V,_,L} when L =< I -> - RK = fetch_var(K,Int0), - {{HasKsi,GetVsi,[RK|HasVarVsi],GetVarVsi},Int0}; - _Other -> - Reg1 = put_reg(V, Int0#sr.reg), - Int1 = Int0#sr{reg=Reg1}, - RK = fetch_var(K,Int0), - RV = fetch_reg(V,Reg1), - {{HasKsi,GetVsi,HasVarVsi,[[RK,RV]|GetVarVsi]},Int1} - end; - (#k_map_pair{key=Key,val=#k_var{name=V}}, - {{HasKsi,GetVsi,HasVarVsi,GetVarVsi},Int0}) -> - case vdb_find(V, Vdb) of - {V,_,L} when L =< I -> - {{[atomic(Key)|HasKsi],GetVsi,HasVarVsi,GetVarVsi},Int0}; - _Other -> - Reg1 = put_reg(V, Int0#sr.reg), - Int1 = Int0#sr{reg=Reg1}, - {{HasKsi,[atomic(Key),fetch_reg(V, Reg1)|GetVsi], - HasVarVsi,GetVarVsi},Int1} - end - end, {{[],[],[],[]},Bef}, Vs), - - Code = [{test,has_map_fields,{f,Fail},Rsrc,{list,HasKs}} || HasKs =/= []] ++ - [{test,has_map_fields,{f,Fail},Rsrc,{list,[K]}} || K <- HasVarKs] ++ - [{get_map_elements, {f,Fail},Rsrc,{list,GetVs}} || GetVs =/= []] ++ - [{get_map_elements, {f,Fail},Rsrc,{list,[K,V]}} || [K,V] <- GetVarVs], - {Code, Aft, St}. - - -select_extract_cons(Src, [#k_var{name=Hd},#k_var{name=Tl}], I, Vdb, Bef, St) -> - Rsrc = fetch_var(Src, Bef), - Int = clear_dead(Bef, I, Vdb), - {{_,_,Lhd},{_,_,Ltl}} = {vdb_find(Hd, Vdb),vdb_find(Tl, Vdb)}, - case {Lhd =< I, Ltl =< I} of - {true,true} -> - %% Both dead. - {[],Bef,St}; - {true,false} -> - %% Head dead. - Reg0 = put_reg(Tl, Bef#sr.reg), - Aft = Int#sr{reg=Reg0}, - Rtl = fetch_reg(Tl, Reg0), - {[{get_tl,Rsrc,Rtl}],Aft,St}; - {false,true} -> - %% Tail dead. - Reg0 = put_reg(Hd, Bef#sr.reg), - Aft = Int#sr{reg=Reg0}, - Rhd = fetch_reg(Hd, Reg0), - {[{get_hd,Rsrc,Rhd}],Aft,St}; - {false,false} -> - %% Both used. - Reg0 = put_reg(Tl, put_reg(Hd, Bef#sr.reg)), - Aft = Bef#sr{reg=Reg0}, - Rhd = fetch_reg(Hd, Reg0), - Rtl = fetch_reg(Tl, Reg0), - {[{get_hd,Rsrc,Rhd},{get_tl,Rsrc,Rtl}],Aft,St} - end. - -guard_clause_cg(#k_guard_clause{anno=#l{vdb=Vdb},guard=G,body=B}, Fail, Bef, St0) -> - {Gis,Int,St1} = guard_cg(G, Fail, Vdb, Bef, St0), - {Bis,Aft,St} = match_cg(B, Fail, Int, St1), - {Gis ++ Bis,Aft,St}. - -%% guard_cg(Guard, Fail, Vdb, StackReg, State) -> -%% {[Ainstr],StackReg,State}. -%% A guard is a boolean expression of tests. Tests return true or -%% false. A fault in a test causes the test to return false. Tests -%% never return the boolean, instead we generate jump code to go to -%% the correct exit point. Primops and tests all go to the next -%% instruction on success or jump to a failure label. - -guard_cg(#k_protected{arg=Ts,ret=Rs,anno=#l{vdb=Pdb}}, Fail, _Vdb, Bef, St) -> - protected_cg(Ts, Rs, Fail, Pdb, Bef, St); -guard_cg(#k_test{anno=#l{i=I},op=Test0,args=As,inverted=Inverted}, - Fail, Vdb, Bef, St0) -> - #k_remote{mod=#k_atom{val=erlang},name=#k_atom{val=Test}} = Test0, - case Inverted of - false -> - test_cg(Test, As, Fail, I, Vdb, Bef, St0); - true -> - {Psucc,St1} = new_label(St0), - {Is,Aft,St2} = test_cg(Test, As, Psucc, I, Vdb, Bef, St1), - {Is++[{jump,{f,Fail}},{label,Psucc}],Aft,St2} - end; -guard_cg(G, _Fail, Vdb, Bef, St) -> - %%ok = io:fwrite("cg ~w: ~p~n", [?LINE,{G,Fail,Vdb,Bef}]), - {Gis,Aft,St1} = cg(G, Vdb, Bef, St), - %%ok = io:fwrite("cg ~w: ~p~n", [?LINE,{Aft}]), - {Gis,Aft,St1}. - -%% guard_cg_list([Kexpr], Fail, I, Vdb, StackReg, St) -> -%% {[Ainstr],StackReg,St}. - -guard_cg_list(Kes, Fail, Vdb, Bef, St0) -> - {Keis,{Aft,St1}} = - flatmapfoldl(fun (Ke, {Inta,Sta}) -> - {Keis,Intb,Stb} = - guard_cg(Ke, Fail, Vdb, Inta, Sta), - {Keis,{Intb,Stb}} - end, {Bef,St0}, need_heap(Kes)), - {Keis,Aft,St1}. - -%% protected_cg([Kexpr], [Ret], Fail, I, Vdb, Bef, St) -> {[Ainstr],Aft,St}. -%% Do a protected. Protecteds without return values are just done -%% for effect, the return value is not checked, success passes on to -%% the next instruction and failure jumps to Fail. If there are -%% return values then these must be set to 'false' on failure, -%% control always passes to the next instruction. - -protected_cg(Ts, [], Fail, Vdb, Bef, St0) -> - %% Protect these calls, revert when done. - {Tis,Aft,St1} = guard_cg_list(Ts, Fail, Vdb, Bef, St0#cg{bfail=Fail}), - {Tis,Aft,St1#cg{bfail=St0#cg.bfail}}; -protected_cg(Ts, Rs, _Fail, Vdb, Bef, St0) -> - {Pfail,St1} = new_label(St0), - {Psucc,St2} = new_label(St1), - {Tis,Aft,St3} = guard_cg_list(Ts, Pfail, Vdb, Bef, - St2#cg{bfail=Pfail}), - %%ok = io:fwrite("cg ~w: ~p~n", [?LINE,{Rs,I,Vdb,Aft}]), - %% Set return values to false. - Mis = [{move,{atom,false},fetch_var(V,Aft)}||#k_var{name=V} <- Rs], - {Tis ++ [{jump,{f,Psucc}}, - {label,Pfail}] ++ Mis ++ [{label,Psucc}], - Aft,St3#cg{bfail=St0#cg.bfail}}. - -%% test_cg(TestName, Args, Fail, I, Vdb, Bef, St) -> {[Ainstr],Aft,St}. -%% Generate test instruction. Use explicit fail label here. - -test_cg(is_map, [A], Fail, I, Vdb, Bef, St) -> - %% We must avoid creating code like this: - %% - %% move x(0) y(0) - %% is_map Fail [x(0)] - %% make_fun => x(0) %% Overwrite x(0) - %% put_map_assoc y(0) ... - %% - %% The code is safe, but beam_validator does not understand that. - %% Extending beam_validator to handle such (rare) code as the - %% above would make it slower for all programs. Instead, change - %% the code generator to always prefer the Y register for is_map() - %% and put_map_assoc() instructions, ensuring that they use the - %% same register. - Arg = cg_reg_arg_prefer_y(A, Bef), - Aft = clear_dead(Bef, I, Vdb), - {[{test,is_map,{f,Fail},[Arg]}],Aft,St}; -test_cg(is_boolean, [#k_atom{val=Val}], Fail, I, Vdb, Bef, St) -> - Aft = clear_dead(Bef, I, Vdb), - Is = case is_boolean(Val) of - true -> []; - false -> [{jump,{f,Fail}}] - end, - {Is,Aft,St}; -test_cg(Test, As, Fail, I, Vdb, Bef, St) -> - Args = cg_reg_args(As, Bef), - Aft = clear_dead(Bef, I, Vdb), - {[beam_utils:bif_to_test(Test, Args, {f,Fail})],Aft,St}. - -%% match_fmf(Fun, LastFail, State, [Clause]) -> {Is,Aft,State}. -%% This is a special flatmapfoldl for match code gen where we -%% generate a "failure" label for each clause. The last clause uses -%% an externally generated failure label, LastFail. N.B. We do not -%% know or care how the failure labels are used. - -match_fmf(F, LastFail, St, [H]) -> - F(H, LastFail, St); -match_fmf(F, LastFail, St0, [H|T]) -> - {Fail,St1} = new_label(St0), - {R,Aft1,St2} = F(H, Fail, St1), - {Rs,Aft2,St3} = match_fmf(F, LastFail, St2, T), - {R ++ [{label,Fail}] ++ Rs,sr_merge(Aft1, Aft2),St3}. - -%% call_cg(Func, [Arg], [Ret], Le, Vdb, StackReg, State) -> -%% {[Ainstr],StackReg,State}. -%% enter_cg(Func, [Arg], Le, Vdb, Bef, St) -> {[Ainstr],Aft,St}. -%% Call and enter first put the arguments into registers and save any -%% other registers, then clean up and compress the stack and set the -%% frame size. Finally the actual call is made. Call then needs the -%% return values filled in. - -call_cg(#k_var{}=Var, As, Rs, Le, Vdb, Bef, St0) -> - {Sis,Int} = cg_setup_call(As++[Var], Bef, Le#l.i, Vdb), - %% Put return values in registers. - Reg = load_vars(Rs, clear_regs(Int#sr.reg)), - %% Build complete code and final stack/register state. - Arity = length(As), - {Frees,Aft} = free_dead(clear_dead(Int#sr{reg=Reg}, Le#l.i, Vdb)), - {Sis ++ Frees ++ [line(Le),{call_fun,Arity}],Aft, - need_stack_frame(St0)}; -call_cg(#k_remote{mod=Mod,name=Name}, As, Rs, Le, Vdb, Bef, St0) - when is_record(Mod, k_var); is_record(Name, k_var) -> - {Sis,Int} = cg_setup_call(As++[Mod,Name], Bef, Le#l.i, Vdb), - %% Put return values in registers. - Reg = load_vars(Rs, clear_regs(Int#sr.reg)), - %% Build complete code and final stack/register state. - Arity = length(As), - St = need_stack_frame(St0), - %%{Call,St1} = build_call(Func, Arity, St0), - {Frees,Aft} = free_dead(clear_dead(Int#sr{reg=Reg}, Le#l.i, Vdb)), - {Sis ++ Frees ++ [line(Le),{apply,Arity}],Aft,St}; -call_cg(Func, As, Rs, Le, Vdb, Bef, St0) -> - case St0 of - #cg{bfail=Fail} when Fail =/= 0 -> - %% Inside a guard. The only allowed function call is to - %% erlang:error/1,2. We will generate the following code: - %% - %% move {atom,ok} DestReg - %% jump FailureLabel - #k_remote{mod=#k_atom{val=erlang}, - name=#k_atom{val=error}} = Func, %Assertion. - [#k_var{name=DestVar}] = Rs, - Int0 = clear_dead(Bef, Le#l.i, Vdb), - Reg = put_reg(DestVar, Int0#sr.reg), - Int = Int0#sr{reg=Reg}, - Dst = fetch_reg(DestVar, Reg), - {[{move,{atom,ok},Dst},{jump,{f,Fail}}], - clear_dead(Int, Le#l.i, Vdb),St0}; - #cg{} -> - %% Ordinary function call in a function body. - {Sis,Int} = cg_setup_call(As, Bef, Le#l.i, Vdb), - %% Put return values in registers. - Reg = load_vars(Rs, clear_regs(Int#sr.reg)), - %% Build complete code and final stack/register state. - Arity = length(As), - {Call,St1} = build_call(Func, Arity, St0), - {Frees,Aft} = free_dead(clear_dead(Int#sr{reg=Reg}, Le#l.i, Vdb)), - {Sis ++ Frees ++ [line(Le)|Call],Aft,St1} - end. - -build_call(#k_remote{mod=#k_atom{val=erlang},name=#k_atom{val='!'}}, 2, St0) -> - {[send],need_stack_frame(St0)}; -build_call(#k_remote{mod=#k_atom{val=Mod},name=#k_atom{val=Name}}, Arity, St0) -> - {[{call_ext,Arity,{extfunc,Mod,Name,Arity}}],need_stack_frame(St0)}; -build_call(#k_local{name=Name}, Arity, St0) when is_atom(Name) -> - {Lbl,St1} = local_func_label(Name, Arity, need_stack_frame(St0)), - {[{call,Arity,{f,Lbl}}],St1}. - -free_dead(#sr{stk=Stk0}=Aft) -> - {Instr,Stk} = free_dead(Stk0, 0, [], []), - {Instr,Aft#sr{stk=Stk}}. - -free_dead([dead|Stk], Y, Instr, StkAcc) -> - %% Note: kill/1 is equivalent to init/1 (translated by beam_asm). - %% We use kill/1 to help further optimisation passes. - free_dead(Stk, Y+1, [{kill,{yy,Y}}|Instr], [free|StkAcc]); -free_dead([Any|Stk], Y, Instr, StkAcc) -> - free_dead(Stk, Y+1, Instr, [Any|StkAcc]); -free_dead([], _, Instr, StkAcc) -> {Instr,reverse(StkAcc)}. - -enter_cg(#k_var{} = Var, As, Le, Vdb, Bef, St0) -> - {Sis,Int} = cg_setup_call(As++[Var], Bef, Le#l.i, Vdb), - %% Build complete code and final stack/register state. - Arity = length(As), - {Sis ++ [line(Le),{call_fun,Arity},return], - clear_dead(Int#sr{reg=clear_regs(Int#sr.reg)}, Le#l.i, Vdb), - need_stack_frame(St0)}; -enter_cg(#k_remote{mod=Mod,name=Name}, As, Le, Vdb, Bef, St0) - when is_record(Mod, k_var); is_record(Name, k_var) -> - {Sis,Int} = cg_setup_call(As++[Mod,Name], Bef, Le#l.i, Vdb), - %% Build complete code and final stack/register state. - Arity = length(As), - St = need_stack_frame(St0), - {Sis ++ [line(Le),{apply_only,Arity}], - clear_dead(Int#sr{reg=clear_regs(Int#sr.reg)}, Le#l.i, Vdb), - St}; -enter_cg(Func, As, Le, Vdb, Bef, St0) -> - {Sis,Int} = cg_setup_call(As, Bef, Le#l.i, Vdb), - %% Build complete code and final stack/register state. - Arity = length(As), - {Call,St1} = build_enter(Func, Arity, St0), - Line = enter_line(Func, Arity, Le), - {Sis ++ Line ++ Call, - clear_dead(Int#sr{reg=clear_regs(Int#sr.reg)}, Le#l.i, Vdb), - St1}. - -build_enter(#k_remote{mod=#k_atom{val=erlang},name=#k_atom{val='!'}}, 2, St0) -> - {[send,return],need_stack_frame(St0)}; -build_enter(#k_remote{mod=#k_atom{val=Mod},name=#k_atom{val=Name}}, Arity, St0) -> - St1 = case trap_bif(Mod, Name, Arity) of - true -> need_stack_frame(St0); - false -> St0 - end, - {[{call_ext_only,Arity,{extfunc,Mod,Name,Arity}}],St1}; -build_enter(#k_local{name=Name}, Arity, St0) when is_atom(Name) -> - {Lbl,St1} = local_func_label(Name, Arity, St0), - {[{call_only,Arity,{f,Lbl}}],St1}. - -enter_line(#k_remote{mod=#k_atom{val=Mod},name=#k_atom{val=Name}}, Arity, Le) -> - case erl_bifs:is_safe(Mod, Name, Arity) of - false -> - %% Tail-recursive call, possibly to a BIF. - %% We'll need a line instruction in case the - %% BIF call fails. - [line(Le)]; - true -> - %% Call to a safe BIF. Since it cannot fail, - %% we don't need any line instruction here. - [] - end; -enter_line(_, _, _) -> - %% Tail-recursive call to a local function. A line - %% instruction will not be useful. - []. - -%% local_func_label(Name, Arity, State) -> {Label,State'} -%% local_func_label({Name,Arity}, State) -> {Label,State'} -%% Get the function entry label for a local function. - -local_func_label(Name, Arity, St) -> - local_func_label({Name,Arity}, St). - -local_func_label(Key, #cg{functable=Map}=St0) -> - case Map of - #{Key := Label} -> {Label,St0}; - _ -> - {Label,St} = new_label(St0), - {Label,St#cg{functable=Map#{Key => Label}}} - end. - -%% need_stack_frame(State) -> State' -%% Make a note in the state that this function will need a stack frame. - -need_stack_frame(#cg{need_frame=true}=St) -> St; -need_stack_frame(St) -> St#cg{need_frame=true}. - -%% trap_bif(Mod, Name, Arity) -> true|false -%% Trap bifs that need a stack frame. - -trap_bif(erlang, link, 1) -> true; -trap_bif(erlang, unlink, 1) -> true; -trap_bif(erlang, monitor_node, 2) -> true; -trap_bif(erlang, group_leader, 2) -> true; -trap_bif(erlang, exit, 2) -> true; -trap_bif(_, _, _) -> false. - -%% bif_cg(#k_bif{}, Le, Vdb, StackReg, State) -> -%% {[Ainstr],StackReg,State}. -%% Generate code a BIF. - -bif_cg(#k_bif{op=#k_internal{name=Name},args=As,ret=Rs}, Le, Vdb, Bef, St) -> - internal_cg(Name, As, Rs, Le, Vdb, Bef, St); -bif_cg(#k_bif{op=#k_remote{mod=#k_atom{val=erlang},name=#k_atom{val=Name}}, - args=As,ret=Rs}, Le, Vdb, Bef, St) -> - Ar = length(As), - case is_gc_bif(Name, Ar) of - false -> - bif_cg(Name, As, Rs, Le, Vdb, Bef, St); - true -> - gc_bif_cg(Name, As, Rs, Le, Vdb, Bef, St) - end. - -%% internal_cg(Bif, [Arg], [Ret], Le, Vdb, StackReg, State) -> -%% {[Ainstr],StackReg,State}. - -internal_cg(bs_context_to_binary=Instr, [Src0], [], Le, Vdb, Bef, St0) -> - [Src] = cg_reg_args([Src0], Bef), - {[{Instr,Src}],clear_dead(Bef, Le#l.i, Vdb), St0}; -internal_cg(dsetelement, [Index0,Tuple0,New0], _Rs, Le, Vdb, Bef, St0) -> - [New,Tuple,{integer,Index1}] = cg_reg_args([New0,Tuple0,Index0], Bef), - Index = Index1-1, - {[{set_tuple_element,New,Tuple,Index}], - clear_dead(Bef, Le#l.i, Vdb), St0}; -internal_cg(make_fun, [Func0,Arity0|As], Rs, Le, Vdb, Bef, St0) -> - %% This behaves more like a function call. - #k_atom{val=Func} = Func0, - #k_int{val=Arity} = Arity0, - {Sis,Int} = cg_setup_call(As, Bef, Le#l.i, Vdb), - Reg = load_vars(Rs, clear_regs(Int#sr.reg)), - {FuncLbl,St1} = local_func_label(Func, Arity, St0), - MakeFun = {make_fun2,{f,FuncLbl},0,0,length(As)}, - {Sis ++ [MakeFun], - clear_dead(Int#sr{reg=Reg}, Le#l.i, Vdb), - St1}; -internal_cg(bs_init_writable=I, As, Rs, Le, Vdb, Bef, St) -> - %% This behaves like a function call. - {Sis,Int} = cg_setup_call(As, Bef, Le#l.i, Vdb), - Reg = load_vars(Rs, clear_regs(Int#sr.reg)), - {Sis++[I],clear_dead(Int#sr{reg=Reg}, Le#l.i, Vdb),St}; -internal_cg(build_stacktrace=I, As, Rs, Le, Vdb, Bef, St) -> - %% This behaves like a function call. - {Sis,Int} = cg_setup_call(As, Bef, Le#l.i, Vdb), - Reg = load_vars(Rs, clear_regs(Int#sr.reg)), - {Sis++[I],clear_dead(Int#sr{reg=Reg}, Le#l.i, Vdb),St}; -internal_cg(raise, As, Rs, Le, Vdb, Bef, St) -> - %% raise can be treated like a guard BIF. - bif_cg(raise, As, Rs, Le, Vdb, Bef, St); -internal_cg(guard_error, [ExitCall], _Rs, Le, Vdb, Bef, St) -> - %% A call an exit BIF from inside a #k_guard_match{}. - %% Generate a standard call, but leave the register descriptors - %% alone, effectively pretending that there was no call. - #k_call{op=#k_remote{mod=#k_atom{val=Mod},name=#k_atom{val=Name}}, - args=As} = ExitCall, - Arity = length(As), - {Ms,_} = cg_call_args(As, Bef, Le#l.i, Vdb), - Call = {call_ext,Arity,{extfunc,Mod,Name,Arity}}, - Is = Ms++[line(Le),Call], - {Is,Bef,St}; -internal_cg(raw_raise=I, As, Rs, Le, Vdb, Bef, St) -> - %% This behaves like a function call. - {Sis,Int} = cg_setup_call(As, Bef, Le#l.i, Vdb), - Reg = load_vars(Rs, clear_regs(Int#sr.reg)), - {Sis++[I],clear_dead(Int#sr{reg=Reg}, Le#l.i, Vdb),St}. - -%% bif_cg(Bif, [Arg], [Ret], Le, Vdb, StackReg, State) -> -%% {[Ainstr],StackReg,State}. - -bif_cg(Bif, As, [#k_var{name=V}], Le, Vdb, Bef, St0) -> - Ars = cg_reg_args(As, Bef), - - %% If we are inside a catch and in a body (not in guard) and the - %% BIF may fail, we must save everything that will be alive after - %% the catch (because the code after the code assumes that all - %% variables that are live are stored on the stack). - %% - %% Currently, we are somewhat pessimistic in - %% that we save any variable that will be live after this BIF call. - - MayFail = not erl_bifs:is_safe(erlang, Bif, length(As)), - {Sis,Int0} = - case MayFail of - true -> - maybe_adjust_stack(Bef, Le#l.i, Le#l.i+1, Vdb, St0); - false -> - {[],Bef} - end, - Int1 = clear_dead(Int0, Le#l.i, Vdb), - Reg = put_reg(V, Int1#sr.reg), - Int = Int1#sr{reg=Reg}, - Dst = fetch_reg(V, Reg), - BifFail = {f,St0#cg.bfail}, - %% We need a line instructions for BIFs that may fail in a body. - Line = case BifFail of - {f,0} when MayFail -> - [line(Le)]; - _ -> - [] - end, - {Sis++Line++[{bif,Bif,BifFail,Ars,Dst}], - clear_dead(Int, Le#l.i, Vdb), St0}. - - -%% gc_bif_cg(Bif, [Arg], [Ret], Le, Vdb, StackReg, State) -> -%% {[Ainstr],StackReg,State}. - -gc_bif_cg(Bif, As, [#k_var{name=V}], Le, Vdb, Bef, St0) -> - Ars = cg_reg_args(As, Bef), - - %% If we are inside a catch and in a body (not in guard) and the - %% BIF may fail, we must save everything that will be alive after - %% the catch (because the code after the code assumes that all - %% variables that are live are stored on the stack). - %% - %% Currently, we are somewhat pessimistic in - %% that we save any variable that will be live after this BIF call. - - {Sis,Int0} = maybe_adjust_stack(Bef, Le#l.i, Le#l.i+1, Vdb, St0), - - Int1 = clear_dead(Int0, Le#l.i, Vdb), - Reg = put_reg(V, Int1#sr.reg), - Int = Int1#sr{reg=Reg}, - Dst = fetch_reg(V, Reg), - BifFail = {f,St0#cg.bfail}, - Line = case BifFail of - {f,0} -> [line(Le)]; - {f,_} -> [] - end, - {Sis++Line++[{gc_bif,Bif,BifFail,max_reg(Bef#sr.reg),Ars,Dst}], - clear_dead(Int, Le#l.i, Vdb), St0}. - -%% recv_loop_cg(TimeOut, ReceiveVar, ReceiveMatch, TimeOutExprs, -%% [Ret], Le, Vdb, Bef, St) -> {[Ainstr],Aft,St}. - -recv_loop_cg(Te, Rvar, Rm, Tes, Rs, Le, Vdb, Bef, St0) -> - {Sis,Int0} = adjust_stack(Bef, Le#l.i, Le#l.i, Vdb), - Int1 = Int0#sr{reg=clear_regs(Int0#sr.reg)}, - %% Get labels. - {Rl,St1} = new_label(St0), - {Tl,St2} = new_label(St1), - {Bl,St3} = new_label(St2), - St4 = St3#cg{break=Bl,recv=Rl}, %Set correct receive labels - {Ris,Raft,St5} = cg_recv_mesg(Rvar, Rm, Tl, Int1, St4), - {Wis,Taft,St6} = cg_recv_wait(Te, Tes, Le#l.i, Int1, St5), - Int2 = sr_merge(Raft, Taft), %Merge stack/registers - Reg = load_vars(Rs, Int2#sr.reg), - {Sis ++ [line(Le)] ++ Ris ++ [{label,Tl}] ++ Wis ++ [{label,Bl}], - clear_dead(Int2#sr{reg=Reg}, Le#l.i, Vdb), - St6#cg{break=St0#cg.break,recv=St0#cg.recv}}. - -%% cg_recv_mesg( ) -> {[Ainstr],Aft,St}. - -cg_recv_mesg(#k_var{name=R}, Rm, Tl, Bef, St0) -> - Int0 = Bef#sr{reg=put_reg(R, Bef#sr.reg)}, - Ret = fetch_reg(R, Int0#sr.reg), - %% Int1 = clear_dead(Int0, I, Rm#l.vdb), - Int1 = Int0, - {Mis,Int2,St1} = match_cg(Rm, none, Int1, St0), - {[{label,St1#cg.recv},{loop_rec,{f,Tl},Ret}|Mis],Int2,St1}. - -%% cg_recv_wait(Te, Tes, I, Vdb, Int2, St3) -> {[Ainstr],Aft,St}. - -cg_recv_wait(#k_atom{val=infinity}, #cg_block{anno=Le,es=Tes}, I, Bef, St0) -> - %% We know that the 'after' body will never be executed. - %% But to keep the stack and register information up to date, - %% we will generate the code for the 'after' body, and then discard it. - Int1 = clear_dead(Bef, I, Le#l.vdb), - {_,Int2,St1} = cg_block(Tes, Le#l.vdb, - Int1#sr{reg=clear_regs(Int1#sr.reg)}, St0), - {[{wait,{f,St1#cg.recv}}],Int2,St1}; -cg_recv_wait(#k_int{val=0}, #cg_block{anno=Le,es=Tes}, _I, Bef, St0) -> - {Tis,Int,St1} = cg_block(Tes, Le#l.vdb, Bef, St0), - {[timeout|Tis],Int,St1}; -cg_recv_wait(Te, #cg_block{anno=Le,es=Tes}, I, Bef, St0) -> - Reg = cg_reg_arg(Te, Bef), - %% Must have empty registers here! Bug if anything in registers. - Int0 = clear_dead(Bef, I, Le#l.vdb), - {Tis,Int,St1} = cg_block(Tes, Le#l.vdb, - Int0#sr{reg=clear_regs(Int0#sr.reg)}, St0), - {[{wait_timeout,{f,St1#cg.recv},Reg},timeout] ++ Tis,Int,St1}. - -%% recv_next_cg(Le, Vdb, StackReg, St) -> {[Ainstr],StackReg,St}. -%% Use adjust stack to clear stack, but only need it for Aft. - -recv_next_cg(Le, Vdb, Bef, St) -> - {Sis,Aft} = adjust_stack(Bef, Le#l.i, Le#l.i+1, Vdb), - {[{loop_rec_end,{f,St#cg.recv}}] ++ Sis,Aft,St}. %Joke - -%% try_cg(TryBlock, [BodyVar], TryBody, [ExcpVar], TryHandler, [Ret], -%% Le, Vdb, StackReg, St) -> {[Ainstr],StackReg,St}. - -try_cg(Ta, Vs, Tb, Evs, Th, Rs, Le, Vdb, Bef, St0) -> - {B,St1} = new_label(St0), %Body label - {H,St2} = new_label(St1), %Handler label - {E,St3} = new_label(St2), %End label - #l{i=TryTag} = get_kanno(Ta), - Int1 = Bef#sr{stk=put_catch(TryTag, Bef#sr.stk)}, - TryReg = fetch_stack({catch_tag,TryTag}, Int1#sr.stk), - {Ais,Int2,St4} = cg(Ta, Vdb, Int1, St3#cg{break=B,in_catch=true}), - Int3 = Int2#sr{stk=drop_catch(TryTag, Int2#sr.stk)}, - St5 = St4#cg{break=E,in_catch=St3#cg.in_catch}, - {Bis,Baft,St6} = cg(Tb, Vdb, Int3#sr{reg=load_vars(Vs, Int3#sr.reg)}, St5), - {His,Haft,St7} = cg(Th, Vdb, Int3#sr{reg=load_vars(Evs, Int3#sr.reg)}, St6), - Int4 = sr_merge(Baft, Haft), %Merge stack/registers - Aft = Int4#sr{reg=load_vars(Rs, Int4#sr.reg)}, - {[{'try',TryReg,{f,H}}] ++ Ais ++ - [{label,B},{try_end,TryReg}] ++ Bis ++ - [{label,H},{try_case,TryReg}] ++ His ++ - [{label,E}], - clear_dead(Aft, Le#l.i, Vdb), - St7#cg{break=St0#cg.break}}. - -try_enter_cg(Ta, Vs, Tb, Evs, Th, Le, Vdb, Bef, St0) -> - {B,St1} = new_label(St0), %Body label - {H,St2} = new_label(St1), %Handler label - #l{i=TryTag} = get_kanno(Ta), - Int1 = Bef#sr{stk=put_catch(TryTag, Bef#sr.stk)}, - TryReg = fetch_stack({catch_tag,TryTag}, Int1#sr.stk), - {Ais,Int2,St3} = cg(Ta, Vdb, Int1, St2#cg{break=B,in_catch=true}), - Int3 = Int2#sr{stk=drop_catch(TryTag, Int2#sr.stk)}, - St4 = St3#cg{in_catch=St2#cg.in_catch}, - {Bis,Baft,St5} = cg(Tb, Vdb, Int3#sr{reg=load_vars(Vs, Int3#sr.reg)}, St4), - {His,Haft,St6} = cg(Th, Vdb, Int3#sr{reg=load_vars(Evs, Int3#sr.reg)}, St5), - Int4 = sr_merge(Baft, Haft), %Merge stack/registers - Aft = Int4, - {[{'try',TryReg,{f,H}}] ++ Ais ++ - [{label,B},{try_end,TryReg}] ++ Bis ++ - [{label,H},{try_case,TryReg}] ++ His, - clear_dead(Aft, Le#l.i, Vdb), - St6#cg{break=St0#cg.break}}. - -%% catch_cg(CatchBlock, Ret, Le, Vdb, Bef, St) -> {[Ainstr],Aft,St}. - -catch_cg(#cg_block{es=C}, #k_var{name=R}, Le, Vdb, Bef, St0) -> - {B,St1} = new_label(St0), - CatchTag = Le#l.i, - Int1 = Bef#sr{stk=put_catch(CatchTag, Bef#sr.stk)}, - CatchReg = fetch_stack({catch_tag,CatchTag}, Int1#sr.stk), - {Cis,Int2,St2} = cg_block(C, Le#l.vdb, Int1, - St1#cg{break=B,in_catch=true}), - [] = Int2#sr.reg, %Assertion. - Aft = Int2#sr{reg=[{0,R}],stk=drop_catch(CatchTag, Int2#sr.stk)}, - {[{'catch',CatchReg,{f,B}}] ++ Cis ++ - [{label,B},{catch_end,CatchReg}], - clear_dead(Aft, Le#l.i, Vdb), - St2#cg{break=St1#cg.break,in_catch=St1#cg.in_catch}}. - -%% put_cg([Var], Constr, Le, Vdb, Bef, St) -> {[Ainstr],Aft,St}. -%% We have to be careful how a 'put' works. First the structure is -%% built, then it is filled and finally things can be cleared. The -%% annotation must reflect this and make sure that the return -%% variable is allocated first. -%% -%% put_list and put_map are atomic instructions, both of -%% which can safely resuse one of the source registers as target. - -put_cg([#k_var{name=R}], #k_cons{hd=Hd,tl=Tl}, Le, Vdb, Bef, St) -> - [S1,S2] = cg_reg_args([Hd,Tl], Bef), - Int0 = clear_dead(Bef, Le#l.i, Vdb), - Int1 = Int0#sr{reg=put_reg(R, Int0#sr.reg)}, - Ret = fetch_reg(R, Int1#sr.reg), - {[{put_list,S1,S2,Ret}], Int1, St}; -put_cg([#k_var{name=R}], #k_binary{segs=Segs}, Le, Vdb, Bef, - #cg{bfail=Bfail}=St) -> - %% At run-time, binaries are constructed in three stages: - %% 1) First the size of the binary is calculated. - %% 2) Then the binary is allocated. - %% 3) Then each field in the binary is constructed. - %% For simplicity, we use the target register to also hold the - %% size of the binary. Therefore the target register must *not* - %% be one of the source registers. - - %% First allocate the target register. - Int0 = Bef#sr{reg=put_reg(R, Bef#sr.reg)}, - Target = fetch_reg(R, Int0#sr.reg), - - %% Also allocate a scratch register for size calculations. - Temp = find_scratch_reg(Int0#sr.reg), - - %% First generate the code that constructs each field. - Fail = {f,Bfail}, - PutCode = cg_bin_put(Segs, Fail, Bef), - {Sis,Int1} = maybe_adjust_stack(Int0, Le#l.i, Le#l.i+1, Vdb, St), - MaxRegs = max_reg(Bef#sr.reg), - Aft = clear_dead(Int1, Le#l.i, Vdb), - - %% Now generate the complete code for constructing the binary. - Code = cg_binary(PutCode, Target, Temp, Fail, MaxRegs, Le#l.a), - {Sis++Code,Aft,St}; - -%% Map: single variable key. -put_cg([#k_var{name=R}], #k_map{op=Op,var=Map, - es=[#k_map_pair{key=#k_var{}=K,val=V}]}, - Le, Vdb, Bef, St0) -> - {Sis,Int0} = maybe_adjust_stack(Bef, Le#l.i, Le#l.i+1, Vdb, St0), - - SrcReg = cg_reg_arg_prefer_y(Map, Int0), - Line = line(Le#l.a), - - List = [cg_reg_arg(K,Int0),cg_reg_arg(V,Int0)], - - Live = max_reg(Bef#sr.reg), - - %% The target register can reuse one of the source registers. - Aft0 = clear_dead(Int0, Le#l.i, Vdb), - Aft = Aft0#sr{reg=put_reg(R, Aft0#sr.reg)}, - Target = fetch_reg(R, Aft#sr.reg), - - {Is,St1} = put_cg_map(Line, Op, SrcReg, Target, Live, List, St0), - {Sis++Is,Aft,St1}; - -%% Map: (possibly) multiple literal keys. -put_cg([#k_var{name=R}], #k_map{op=Op,var=Map,es=Es}, Le, Vdb, Bef, St0) -> - - %% assert key literals - [] = [Var || #k_map_pair{key=#k_var{}=Var} <- Es], - - {Sis,Int0} = maybe_adjust_stack(Bef, Le#l.i, Le#l.i+1, Vdb, St0), - SrcReg = cg_reg_arg_prefer_y(Map, Int0), - Line = line(Le#l.a), - - %% fetch registers for values to be put into the map - List = flatmap(fun(#k_map_pair{key=K,val=V}) -> - [atomic(K),cg_reg_arg(V, Int0)] - end, Es), - - Live = max_reg(Bef#sr.reg), - - %% The target register can reuse one of the source registers. - Aft0 = clear_dead(Int0, Le#l.i, Vdb), - Aft = Aft0#sr{reg=put_reg(R, Aft0#sr.reg)}, - Target = fetch_reg(R, Aft#sr.reg), - - {Is,St1} = put_cg_map(Line, Op, SrcReg, Target, Live, List, St0), - {Sis++Is,Aft,St1}; - -%% Everything else. -put_cg([#k_var{name=R}], Con, Le, Vdb, Bef, St) -> - %% Find a place for the return register first. - Int = Bef#sr{reg=put_reg(R, Bef#sr.reg)}, - Ret = fetch_reg(R, Int#sr.reg), - Ais = case Con of - #k_tuple{es=Es} -> - [{put_tuple,length(Es),Ret}] ++ cg_build_args(Es, Bef); - Other -> - [{move,cg_reg_arg(Other, Int),Ret}] - end, - {Ais,clear_dead(Int, Le#l.i, Vdb),St}. - - -put_cg_map(Line, Op0, SrcReg, Target, Live, List, St0) -> - Bfail = St0#cg.bfail, - Fail = {f,St0#cg.bfail}, - Op = case Op0 of - assoc -> put_map_assoc; - exact -> put_map_exact - end, - {OkLbl,St1} = new_label(St0), - {BadLbl,St2} = new_label(St1), - Is = if - Bfail =:= 0 orelse Op =:= put_map_assoc -> - [Line,{Op,{f,0},SrcReg,Target,Live,{list,List}}]; - true -> - %% Ensure that Target is always set, even if - %% the map update operation fails. That is necessary - %% because Target may be included in a test_heap - %% instruction. - [Line, - {Op,{f,BadLbl},SrcReg,Target,Live,{list,List}}, - {jump,{f,OkLbl}}, - {label,BadLbl}, - {move,{atom,ok},Target}, - {jump,Fail}, - {label,OkLbl}] - end, - {Is,St2}. - -%%% -%%% Code generation for constructing binaries. -%%% - -cg_binary([{bs_put_binary,Fail,{atom,all},U,_Flags,Src}|PutCode], - Target, Temp, Fail, MaxRegs, Anno) -> - Line = line(Anno), - Live = cg_live(Target, MaxRegs), - SzCode = cg_bitstr_size(PutCode, Target, Temp, Fail, Live), - BinFlags = {field_flags,[]}, - Code = [Line|SzCode] ++ - [case member(single_use, Anno) of - true -> - {bs_private_append,Fail,Target,U,Src,BinFlags,Target}; - false -> - {bs_append,Fail,Target,0,MaxRegs,U,Src,BinFlags,Target} - end] ++ PutCode, - cg_bin_opt(Code); -cg_binary(PutCode, Target, Temp, Fail, MaxRegs, Anno) -> - Line = line(Anno), - Live = cg_live(Target, MaxRegs), - {InitOp,SzCode} = cg_binary_size(PutCode, Target, Temp, Fail, Live), - - Code = [Line|SzCode] ++ [{InitOp,Fail,Target,0,MaxRegs, - {field_flags,[]},Target}|PutCode], - cg_bin_opt(Code). - -cg_live({x,X}, MaxRegs) when X =:= MaxRegs -> MaxRegs+1; -cg_live({x,X}, MaxRegs) when X < MaxRegs -> MaxRegs. - -%% Generate code that calculate the size of the bitstr to be -%% built in BITS. - -cg_bitstr_size(PutCode, Target, Temp, Fail, Live) -> - {Bits,Es} = cg_bitstr_size_1(PutCode, 0, []), - reverse(cg_gen_binsize(Es, Target, Temp, Fail, Live, - [{move,{integer,Bits},Target}])). - -cg_bitstr_size_1([{bs_put_utf8,_,_,Src}|Next], Bits, Acc) -> - cg_bitstr_size_1(Next, Bits, [{'*',{bs_utf8_size,Src},8}|Acc]); -cg_bitstr_size_1([{bs_put_utf16,_,_,Src}|Next], Bits, Acc) -> - cg_bitstr_size_1(Next, Bits, [{'*',{bs_utf16_size,Src},8}|Acc]); -cg_bitstr_size_1([{bs_put_utf32,_,_,_}|Next], Bits, Acc) -> - cg_bitstr_size_1(Next, Bits+32, Acc); -cg_bitstr_size_1([{_,_,S,U,_,Src}|Next], Bits, Acc) -> - case S of - {integer,N} -> cg_bitstr_size_1(Next, Bits+N*U, Acc); - {atom,all} -> cg_bitstr_size_1(Next, Bits, [{bit_size,Src}|Acc]); - _ when U =:= 1 -> cg_bitstr_size_1(Next, Bits, [S|Acc]); - _ -> cg_bitstr_size_1(Next, Bits, [{'*',S,U}|Acc]) - end; -cg_bitstr_size_1([], Bits, Acc) -> {Bits,Acc}. - -%% Generate code that calculate the size of the bitstr to be -%% built in BYTES or BITS (depending on what is easiest). - -cg_binary_size(PutCode, Target, Temp, Fail, Live) -> - {InitInstruction,Szs} = cg_binary_size_1(PutCode, 0, []), - SizeExpr = reverse(cg_gen_binsize(Szs, Target, Temp, Fail, Live, [{move,{integer,0},Target}])), - {InitInstruction,SizeExpr}. - -cg_binary_size_1([{bs_put_utf8,_Fail,_Flags,Src}|T], Bits, Acc) -> - cg_binary_size_1(T, Bits, [{8,{bs_utf8_size,Src}}|Acc]); -cg_binary_size_1([{bs_put_utf16,_Fail,_Flags,Src}|T], Bits, Acc) -> - cg_binary_size_1(T, Bits, [{8,{bs_utf16_size,Src}}|Acc]); -cg_binary_size_1([{bs_put_utf32,_Fail,_Flags,_Src}|T], Bits, Acc) -> - cg_binary_size_1(T, Bits+32, Acc); -cg_binary_size_1([{_Put,_Fail,S,U,_Flags,Src}|T], Bits, Acc) -> - cg_binary_size_2(S, U, Src, T, Bits, Acc); -cg_binary_size_1([], Bits, Acc) -> - Bytes = Bits div 8, - RemBits = Bits rem 8, - Sizes0 = sort([{1,{integer,RemBits}},{8,{integer,Bytes}}|Acc]), - Sizes = filter(fun({_,{integer,0}}) -> false; - (_) -> true end, Sizes0), - case Sizes of - [{1,_}|_] -> - {bs_init_bits,cg_binary_bytes_to_bits(Sizes, [])}; - [{8,_}|_] -> - {bs_init2,[E || {8,E} <- Sizes]}; - [] -> - {bs_init_bits,[]} - end. - -cg_binary_size_2({integer,N}, U, _, Next, Bits, Acc) -> - cg_binary_size_1(Next, Bits+N*U, Acc); -cg_binary_size_2({atom,all}, U, E, Next, Bits, Acc) -> - if - U rem 8 =:= 0 -> - cg_binary_size_1(Next, Bits, [{8,{byte_size,E}}|Acc]); - true -> - cg_binary_size_1(Next, Bits, [{1,{bit_size,E}}|Acc]) - end; -cg_binary_size_2(Reg, 1, _, Next, Bits, Acc) -> - cg_binary_size_1(Next, Bits, [{1,Reg}|Acc]); -cg_binary_size_2(Reg, 8, _, Next, Bits, Acc) -> - cg_binary_size_1(Next, Bits, [{8,Reg}|Acc]); -cg_binary_size_2(Reg, U, _, Next, Bits, Acc) -> - cg_binary_size_1(Next, Bits, [{1,{'*',Reg,U}}|Acc]). - -cg_binary_bytes_to_bits([{8,{integer,N}}|T], Acc) -> - cg_binary_bytes_to_bits(T, [{integer,8*N}|Acc]); -cg_binary_bytes_to_bits([{8,{byte_size,Reg}}|T], Acc) -> - cg_binary_bytes_to_bits(T, [{bit_size,Reg}|Acc]); -cg_binary_bytes_to_bits([{8,Reg}|T], Acc) -> - cg_binary_bytes_to_bits(T, [{'*',Reg,8}|Acc]); -cg_binary_bytes_to_bits([{1,Sz}|T], Acc) -> - cg_binary_bytes_to_bits(T, [Sz|Acc]); -cg_binary_bytes_to_bits([], Acc) -> - cg_binary_bytes_to_bits_1(sort(Acc)). - -cg_binary_bytes_to_bits_1([{integer,I},{integer,J}|T]) -> - cg_binary_bytes_to_bits_1([{integer,I+J}|T]); -cg_binary_bytes_to_bits_1([H|T]) -> - [H|cg_binary_bytes_to_bits_1(T)]; -cg_binary_bytes_to_bits_1([]) -> []. - -cg_gen_binsize([{'*',{bs_utf8_size,Src},B}|T], Target, Temp, Fail, Live, Acc) -> - Size = {bs_utf8_size,Fail,Src,Temp}, - Add = {bs_add,Fail,[Target,Temp,B],Target}, - cg_gen_binsize(T, Target, Temp, Fail, Live, - [Add,Size|Acc]); -cg_gen_binsize([{'*',{bs_utf16_size,Src},B}|T], Target, Temp, Fail, Live, Acc) -> - Size = {bs_utf16_size,Fail,Src,Temp}, - Add = {bs_add,Fail,[Target,Temp,B],Target}, - cg_gen_binsize(T, Target, Temp, Fail, Live, - [Add,Size|Acc]); -cg_gen_binsize([{'*',A,B}|T], Target, Temp, Fail, Live, Acc) -> - cg_gen_binsize(T, Target, Temp, Fail, Live, - [{bs_add,Fail,[Target,A,B],Target}|Acc]); -cg_gen_binsize([{bit_size,B}|T], Target, Temp, Fail, Live, Acc) -> - cg_gen_binsize([Temp|T], Target, Temp, Fail, Live, - [{gc_bif,bit_size,Fail,Live,[B],Temp}|Acc]); -cg_gen_binsize([{byte_size,B}|T], Target, Temp, Fail, Live, Acc) -> - cg_gen_binsize([Temp|T], Target, Temp, Fail, Live, - [{gc_bif,byte_size,Fail,Live,[B],Temp}|Acc]); -cg_gen_binsize([{bs_utf8_size,B}|T], Target, Temp, Fail, Live, Acc) -> - cg_gen_binsize([Temp|T], Target, Temp, Fail, Live, - [{bs_utf8_size,Fail,B,Temp}|Acc]); -cg_gen_binsize([{bs_utf16_size,B}|T], Target, Temp, Fail, Live, Acc) -> - cg_gen_binsize([Temp|T], Target, Temp, Fail, Live, - [{bs_utf16_size,Fail,B,Temp}|Acc]); -cg_gen_binsize([E0|T], Target, Temp, Fail, Live, Acc) -> - cg_gen_binsize(T, Target, Temp, Fail, Live, - [{bs_add,Fail,[Target,E0,1],Target}|Acc]); -cg_gen_binsize([], _, _, _, _, Acc) -> Acc. - - -%% cg_bin_opt(Code0) -> Code -%% Optimize the size calculations for binary construction. - -cg_bin_opt([{move,S1,{x,X}=D},{gc_bif,Op,Fail,Live0,As,Dst}|Is]) -> - Live = if - X + 1 =:= Live0 -> X; - true -> Live0 - end, - [{gc_bif,Op,Fail,Live,As,D}|cg_bin_opt([{move,S1,Dst}|Is])]; -cg_bin_opt([{move,_,_}=I1,{Op,_,_,_}=I2|Is]) - when Op =:= bs_utf8_size orelse Op =:= bs_utf16_size -> - [I2|cg_bin_opt([I1|Is])]; -cg_bin_opt([{bs_add,_,[{integer,0},Src,1],Dst}|Is]) -> - cg_bin_opt_1([{move,Src,Dst}|Is]); -cg_bin_opt([{bs_add,_,[Src,{integer,0},_],Dst}|Is]) -> - cg_bin_opt_1([{move,Src,Dst}|Is]); -cg_bin_opt(Is) -> - cg_bin_opt_1(Is). - -cg_bin_opt_1([{move,Size,D},{bs_append,Fail,D,Extra,Regs,U,Bin,Flags,D}|Is]) -> - [{bs_append,Fail,Size,Extra,Regs,U,Bin,Flags,D}|cg_bin_opt(Is)]; -cg_bin_opt_1([{move,Size,D},{bs_private_append,Fail,D,U,Bin,Flags,D}|Is]) -> - [{bs_private_append,Fail,Size,U,Bin,Flags,D}|cg_bin_opt(Is)]; -cg_bin_opt_1([{move,Size,D},{Op,Fail,D,Extra,Regs,Flags,D}|Is]) - when Op =:= bs_init2; Op =:= bs_init_bits -> - Bytes = case Size of - {integer,Int} -> Int; - _ -> Size - end, - [{Op,Fail,Bytes,Extra,Regs,Flags,D}|cg_bin_opt(Is)]; -cg_bin_opt_1([{move,S1,D},{bs_add,Fail,[D,S2,U],Dst}|Is]) -> - cg_bin_opt([{bs_add,Fail,[S1,S2,U],Dst}|Is]); -cg_bin_opt_1([{move,S1,D},{bs_add,Fail,[S2,D,U],Dst}|Is]) -> - cg_bin_opt([{bs_add,Fail,[S2,S1,U],Dst}|Is]); -cg_bin_opt_1([I|Is]) -> - [I|cg_bin_opt(Is)]; -cg_bin_opt_1([]) -> - []. - -cg_bin_put(#k_bin_seg{size=S0,unit=U,type=T,flags=Fs,seg=E0,next=Next}, - Fail, Bef) -> - S1 = cg_reg_arg(S0, Bef), - E1 = cg_reg_arg(E0, Bef), - {Format,Op} = case T of - integer -> {plain,bs_put_integer}; - utf8 -> {utf,bs_put_utf8}; - utf16 -> {utf,bs_put_utf16}; - utf32 -> {utf,bs_put_utf32}; - binary -> {plain,bs_put_binary}; - float -> {plain,bs_put_float} - end, - case Format of - plain -> - [{Op,Fail,S1,U,{field_flags,Fs},E1}|cg_bin_put(Next, Fail, Bef)]; - utf -> - [{Op,Fail,{field_flags,Fs},E1}|cg_bin_put(Next, Fail, Bef)] - end; -cg_bin_put(#k_bin_end{}, _, _) -> []. - -cg_build_args(As, Bef) -> - [{put,cg_reg_arg(A, Bef)} || A <- As]. - -%% return_cg([Val], Le, Vdb, Bef, St) -> {[Ainstr],Aft,St}. -%% break_cg([Val], Le, Vdb, Bef, St) -> {[Ainstr],Aft,St}. -%% These are very simple, just put return/break values in registers -%% from 0, then return/break. Use the call setup to clean up stack, -%% but must clear registers to ensure sr_merge works correctly. - -return_cg(Rs, Le, Vdb, Bef, St) -> - {Ms,Int} = cg_setup_call(Rs, Bef, Le#l.i, Vdb), - {Ms ++ [return],Int#sr{reg=clear_regs(Int#sr.reg)},St}. - -break_cg(Bs, Le, Vdb, Bef, St) -> - {Ms,Int} = cg_setup_call(Bs, Bef, Le#l.i, Vdb), - {Ms ++ [{jump,{f,St#cg.break}}], - Int#sr{reg=clear_regs(Int#sr.reg)},St}. - -guard_break_cg(Bs, #l{i=I}, Vdb, #sr{reg=Reg0}=Bef, St) -> - #sr{reg=Reg1} = Int = clear_dead(Bef, I, Vdb), - Reg2 = trim_free(Reg1), - NumLocked = length(Reg2), - Moves0 = gen_moves(Bs, Bef, NumLocked, []), - Moves = order_moves(Moves0, find_scratch_reg(Reg0)), - {BreakVars,_} = mapfoldl(fun(_, RegNum) -> - {{RegNum,gbreakvar},RegNum+1} - end, length(Reg2), Bs), - Reg = Reg2 ++ BreakVars, - Aft = Int#sr{reg=Reg}, - {Moves ++ [{jump,{f,St#cg.break}}],Aft,St}. - -%% cg_reg_arg(Arg0, Info) -> Arg -%% cg_reg_args([Arg0], Info) -> [Arg] -%% Convert argument[s] into registers. Literal values are returned unchanged. - -cg_reg_args(As, Bef) -> [cg_reg_arg(A, Bef) || A <- As]. - -cg_reg_arg(#k_var{name=V}, Bef) -> fetch_var(V, Bef); -cg_reg_arg(Literal, _) -> atomic(Literal). - -cg_reg_arg_prefer_y(#k_var{name=V}, Bef) -> fetch_var_prefer_y(V, Bef); -cg_reg_arg_prefer_y(Literal, _) -> atomic(Literal). - -%% cg_setup_call([Arg], Bef, Cur, Vdb) -> {[Instr],Aft}. -%% Do the complete setup for a call/enter. - -cg_setup_call(As, Bef, I, Vdb) -> - {Ms,Int0} = cg_call_args(As, Bef, I, Vdb), - %% Have set up arguments, can now clean up, compress and save to stack. - Int1 = Int0#sr{stk=clear_dead_stk(Int0#sr.stk, I, Vdb),res=[]}, - {Sis,Int2} = adjust_stack(Int1, I, I+1, Vdb), - {Ms ++ Sis,Int2}. - -%% cg_call_args([Arg], SrState) -> {[Instr],SrState}. -%% Setup the arguments to a call/enter/bif. Put the arguments into -%% consecutive registers starting at {x,0} moving any data which -%% needs to be saved. Return a modified SrState structure with the -%% new register contents. N.B. the resultant register info will -%% contain non-variable values when there are non-variable values. -%% -%% This routine is complicated by unsaved values in x registers. -%% We'll move away any unsaved values that are in the registers -%% to be overwritten by the arguments. - -cg_call_args(As, Bef, I, Vdb) -> - Regs0 = load_arg_regs(Bef#sr.reg, As), - Unsaved = unsaved_registers(Regs0, Bef#sr.stk, I, I+1, Vdb), - {UnsavedMoves,Regs} = move_unsaved(Unsaved, Bef#sr.reg, Regs0), - Moves0 = gen_moves(As, Bef), - Moves = order_moves(Moves0, find_scratch_reg(Regs)), - {UnsavedMoves ++ Moves,Bef#sr{reg=Regs}}. - -%% load_arg_regs([Reg], Arguments) -> [Reg] -%% Update the register descriptor to include the arguments (from {x,0} -%% and upwards). Values in argument register are overwritten. -%% Values in x registers above the arguments are preserved. - -load_arg_regs(Regs, As) -> load_arg_regs(Regs, As, 0). - -load_arg_regs([_|Rs], [#k_var{name=V}|As], I) -> [{I,V}|load_arg_regs(Rs, As, I+1)]; -load_arg_regs([_|Rs], [A|As], I) -> [{I,A}|load_arg_regs(Rs, As, I+1)]; -load_arg_regs([], [#k_var{name=V}|As], I) -> [{I,V}|load_arg_regs([], As, I+1)]; -load_arg_regs([], [A|As], I) -> [{I,A}|load_arg_regs([], As, I+1)]; -load_arg_regs(Rs, [], _) -> Rs. - -%% Returns the variables must be saved and are currently in the -%% x registers that are about to be overwritten by the arguments. - -unsaved_registers(Regs, Stk, Fb, Lf, Vdb) -> - [V || {V,F,L} <- Vdb, - F < Fb, - L >= Lf, - not on_stack(V, Stk), - not in_reg(V, Regs)]. - -in_reg(V, Regs) -> keymember(V, 2, Regs). - -%% Move away unsaved variables from the registers that are to be -%% overwritten by the arguments. -move_unsaved(Vs, OrigRegs, NewRegs) -> - move_unsaved(Vs, OrigRegs, NewRegs, []). - -move_unsaved([V|Vs], OrigRegs, NewRegs0, Acc) -> - NewRegs = put_reg(V, NewRegs0), - Src = fetch_reg(V, OrigRegs), - Dst = fetch_reg(V, NewRegs), - move_unsaved(Vs, OrigRegs, NewRegs, [{move,Src,Dst}|Acc]); -move_unsaved([], _, Regs, Acc) -> {Acc,Regs}. - -%% gen_moves(As, Sr) -%% Generate the basic move instruction to move the arguments -%% to their proper registers. The list will be sorted on -%% destinations. (I.e. the move to {x,0} will be first -- -%% see the comment to order_moves/2.) - -gen_moves(As, Sr) -> gen_moves(As, Sr, 0, []). - -gen_moves([#k_var{name=V}|As], Sr, I, Acc) -> - case fetch_var(V, Sr) of - {x,I} -> gen_moves(As, Sr, I+1, Acc); - Reg -> gen_moves(As, Sr, I+1, [{move,Reg,{x,I}}|Acc]) - end; -gen_moves([A0|As], Sr, I, Acc) -> - A = atomic(A0), - gen_moves(As, Sr, I+1, [{move,A,{x,I}}|Acc]); -gen_moves([], _, _, Acc) -> lists:keysort(3, Acc). - -%% order_moves([Move], ScratchReg) -> [Move] -%% Orders move instruction so that source registers are not -%% destroyed before they are used. If there are cycles -%% (such as {move,{x,0},{x,1}}, {move,{x,1},{x,1}}), -%% the scratch register is used to break up the cycle. -%% If possible, the first move of the input list is placed -%% last in the result list (to make the move to {x,0} occur -%% just before the call to allow the Beam loader to coalesce -%% the instructions). - -order_moves(Ms, Scr) -> order_moves(Ms, Scr, []). - -order_moves([{move,_,_}=M|Ms0], ScrReg, Acc0) -> - {Chain,Ms} = collect_chain(Ms0, [M], ScrReg), - Acc = reverse(Chain, Acc0), - order_moves(Ms, ScrReg, Acc); -order_moves([], _, Acc) -> Acc. - -collect_chain(Ms, Path, ScrReg) -> - collect_chain(Ms, Path, [], ScrReg). - -collect_chain([{move,Src,Same}=M|Ms0], [{move,Same,_}|_]=Path, Others, ScrReg) -> - case lists:keyfind(Src, 3, Path) of - false -> - collect_chain(reverse(Others, Ms0), [M|Path], [], ScrReg); - _ -> % We have a cycle. - {break_up_cycle(M, Path, ScrReg),reverse(Others, Ms0)} - end; -collect_chain([M|Ms], Path, Others, ScrReg) -> - collect_chain(Ms, Path, [M|Others], ScrReg); -collect_chain([], Path, Others, _) -> - {Path,Others}. - -break_up_cycle({move,Src,_}=M, Path, ScrReg) -> - [{move,ScrReg,Src},M|break_up_cycle1(Src, Path, ScrReg)]. - -break_up_cycle1(Dst, [{move,Src,Dst}|Path], ScrReg) -> - [{move,Src,ScrReg}|Path]; -break_up_cycle1(Dst, [M|Path], LastMove) -> - [M|break_up_cycle1(Dst, Path, LastMove)]. - -%% clear_dead(Sr, Until, Vdb) -> Aft. -%% Remove all variables in Sr which have died AT ALL so far. - -clear_dead(#sr{stk=Stk}=Sr0, Until, Vdb) -> - Sr = Sr0#sr{reg=clear_dead_reg(Sr0, Until, Vdb), - stk=clear_dead_stk(Stk, Until, Vdb)}, - reserve(Sr). - -clear_dead_reg(Sr, Until, Vdb) -> - [case R of - {_I,V} = IV -> - case vdb_find(V, Vdb) of - {V,_,L} when L > Until -> IV; - _ -> free %Remove anything else - end; - {reserved,_I,_V}=Reserved -> Reserved; - free -> free - end || R <- Sr#sr.reg]. - -clear_dead_stk(Stk, Until, Vdb) -> - [case S of - {V} = T -> - case vdb_find(V, Vdb) of - {V,_,L} when L > Until -> T; - _ -> dead %Remove anything else - end; - free -> free; - dead -> dead - end || S <- Stk]. - - -%% sr_merge(Sr1, Sr2) -> Sr. -%% Merge two stack/register states keeping the longest of both stack -%% and register. Perform consistency check on both, elements must be -%% the same. Allow frame size 'void' to make easy creation of -%% "empty" frame. - -sr_merge(#sr{reg=R1,stk=S1,res=[]}, #sr{reg=R2,stk=S2,res=[]}) -> - #sr{reg=longest(R1, R2),stk=longest(S1, S2),res=[]}; -sr_merge(void, S2) -> S2#sr{res=[]}. - -longest([H|T1], [H|T2]) -> [H|longest(T1, T2)]; -longest([dead|T1], [free|T2]) -> [dead|longest(T1, T2)]; -longest([free|T1], [dead|T2]) -> [dead|longest(T1, T2)]; -longest([dead|_] = L, []) -> L; -longest([], [dead|_] = L) -> L; -longest([free|_] = L, []) -> L; -longest([], [free|_] = L) -> L; -longest([], []) -> []. - -trim_free([R|Rs0]) -> - case {trim_free(Rs0),R} of - {[],free} -> []; - {Rs,R} -> [R|Rs] - end; -trim_free([]) -> []. - -%% maybe_adjust_stack(Bef, FirstBefore, LastFrom, Vdb, St) -> {[Ainstr],Aft}. -%% Adjust the stack, but only if the code is inside a catch and not -%% inside a guard. Use this funtion before instructions that may -%% cause an exception. - -maybe_adjust_stack(Bef, Fb, Lf, Vdb, St) -> - case St of - #cg{in_catch=true,bfail=0} -> - adjust_stack(Bef, Fb, Lf, Vdb); - #cg{} -> - {[],Bef} - end. - -%% adjust_stack(Bef, FirstBefore, LastFrom, Vdb) -> {[Ainstr],Aft}. -%% Do complete stack adjustment by compressing stack and adding -%% variables to be saved. Try to optimise ordering on stack by -%% having reverse order to their lifetimes. -%% -%% In Beam, there is a fixed stack frame and no need to do stack compression. - -adjust_stack(Bef, Fb, Lf, Vdb) -> - Stk0 = Bef#sr.stk, - {Stk1,Saves} = save_stack(Stk0, Fb, Lf, Vdb), - {saves(Saves, Bef#sr.reg, Stk1), - Bef#sr{stk=Stk1}}. - -%% save_stack(Stack, FirstBefore, LastFrom, Vdb) -> {[SaveVar],NewStack}. -%% Save variables which are used past current point and which are not -%% already on the stack. - -save_stack(Stk0, Fb, Lf, Vdb) -> - %% New variables that are in use but not on stack. - New = new_not_on_stack(Stk0, Fb, Lf, Vdb), - - %% Add new variables that are not just dropped immediately. - %% N.B. foldr works backwards from the end!! - Saves = [V || {V,_,_} <- keysort(3, New)], - Stk1 = foldr(fun (V, Stk) -> put_stack(V, Stk) end, Stk0, Saves), - {Stk1,Saves}. - -%% new_not_on_stack(Stack, FirstBefore, LastFrom, Vdb) -> -%% [{Variable,First,Last}] -%% Return information about all variables that are used past current -%% point and that are not already on the stack. - -new_not_on_stack(Stk, Fb, Lf, Vdb) -> - [VFL || {V,F,L} = VFL <- Vdb, - F < Fb, - L >= Lf, - not on_stack(V, Stk)]. - -%% saves([SaveVar], Reg, Stk) -> [{move,Reg,Stk}]. -%% Generate move instructions to save variables onto stack. The -%% stack/reg info used is that after the new stack has been made. - -saves(Ss, Reg, Stk) -> - [{move,fetch_reg(V, Reg),fetch_stack(V, Stk)} || V <- Ss]. - -%% fetch_var(VarName, StkReg) -> r{R} | sp{Sp}. -%% find_var(VarName, StkReg) -> ok{r{R} | sp{Sp}} | error. -%% Fetch/find a variable in either the registers or on the -%% stack. Fetch KNOWS it's there. - -fetch_var(V, Sr) -> - case find_reg(V, Sr#sr.reg) of - {ok,R} -> R; - error -> fetch_stack(V, Sr#sr.stk) - end. - -fetch_var_prefer_y(V, #sr{reg=Reg,stk=Stk}) -> - case find_stack(V, Stk) of - {ok,R} -> R; - error -> fetch_reg(V, Reg) - end. - -load_vars(Vs, Regs) -> - foldl(fun (#k_var{name=V}, Rs) -> put_reg(V, Rs) end, Regs, Vs). - -%% put_reg(Val, Regs) -> Regs. -%% find_reg(Val, Regs) -> {ok,r{R}} | error. -%% fetch_reg(Val, Regs) -> r{R}. -%% Functions to interface the registers. - -% put_regs(Vs, Rs) -> foldl(fun put_reg/2, Rs, Vs). - -put_reg(V, Rs) -> put_reg_1(V, Rs, 0). - -put_reg_1(V, [free|Rs], I) -> [{I,V}|Rs]; -put_reg_1(V, [{reserved,I,V}|Rs], I) -> [{I,V}|Rs]; -put_reg_1(V, [R|Rs], I) -> [R|put_reg_1(V, Rs, I+1)]; -put_reg_1(V, [], I) -> [{I,V}]. - -fetch_reg(V, [{I,V}|_]) -> {x,I}; -fetch_reg(V, [_|SRs]) -> fetch_reg(V, SRs). - -find_reg(V, [{I,V}|_]) -> {ok,{x,I}}; -find_reg(V, [_|SRs]) -> find_reg(V, SRs); -find_reg(_, []) -> error. - -%% For the bit syntax, we need a scratch register if we are constructing -%% a binary that will not be used. - -find_scratch_reg(Rs) -> find_scratch_reg(Rs, 0). - -find_scratch_reg([free|_], I) -> {x,I}; -find_scratch_reg([_|Rs], I) -> find_scratch_reg(Rs, I+1); -find_scratch_reg([], I) -> {x,I}. - -replace_reg_contents(Old, New, [{I,Old}|Rs]) -> [{I,New}|Rs]; -replace_reg_contents(Old, New, [R|Rs]) -> [R|replace_reg_contents(Old, New, Rs)]. - -%%clear_regs(Regs) -> map(fun (R) -> free end, Regs). -clear_regs(_) -> []. - -max_reg(Regs) -> - foldl(fun ({I,_}, _) -> I; - (_, Max) -> Max end, - -1, Regs) + 1. - -%% put_stack(Val, [{Val}]) -> [{Val}]. -%% fetch_stack(Var, Stk) -> sp{S}. -%% find_stack(Var, Stk) -> ok{sp{S}} | error. -%% Functions to interface the stack. - -put_stack(Val, []) -> [{Val}]; -put_stack(Val, [dead|Stk]) -> [{Val}|Stk]; -put_stack(Val, [free|Stk]) -> [{Val}|Stk]; -put_stack(Val, [NotFree|Stk]) -> [NotFree|put_stack(Val, Stk)]. - -put_stack_carefully(Val, Stk0) -> - try - put_stack_carefully1(Val, Stk0) - catch - throw:error -> - error - end. - -put_stack_carefully1(_, []) -> throw(error); -put_stack_carefully1(Val, [dead|Stk]) -> [{Val}|Stk]; -put_stack_carefully1(Val, [free|Stk]) -> [{Val}|Stk]; -put_stack_carefully1(Val, [NotFree|Stk]) -> - [NotFree|put_stack_carefully1(Val, Stk)]. - -fetch_stack(Var, Stk) -> fetch_stack(Var, Stk, 0). - -fetch_stack(V, [{V}|_], I) -> {yy,I}; -fetch_stack(V, [_|Stk], I) -> fetch_stack(V, Stk, I+1). - -find_stack(Var, Stk) -> find_stack(Var, Stk, 0). - -find_stack(V, [{V}|_], I) -> {ok,{yy,I}}; -find_stack(V, [_|Stk], I) -> find_stack(V, Stk, I+1); -find_stack(_, [], _) -> error. - -on_stack(V, Stk) -> keymember(V, 1, Stk). - -%% put_catch(CatchTag, Stack) -> Stack' -%% drop_catch(CatchTag, Stack) -> Stack' -%% Special interface for putting and removing catch tags, to ensure that -%% catches nest properly. Also used for try tags. - -put_catch(Tag, Stk0) -> put_catch(Tag, reverse(Stk0), []). - -put_catch(Tag, [], Stk) -> - put_stack({catch_tag,Tag}, Stk); -put_catch(Tag, [{{catch_tag,_}}|_]=RevStk, Stk) -> - reverse(RevStk, put_stack({catch_tag,Tag}, Stk)); -put_catch(Tag, [Other|Stk], Acc) -> - put_catch(Tag, Stk, [Other|Acc]). - -drop_catch(Tag, [{{catch_tag,Tag}}|Stk]) -> [free|Stk]; -drop_catch(Tag, [Other|Stk]) -> [Other|drop_catch(Tag, Stk)]. - -%% atomic(Klit) -> Lit. -%% atomic_list([Klit]) -> [Lit]. - -atomic(#k_literal{val=V}) -> {literal,V}; -atomic(#k_int{val=I}) -> {integer,I}; -atomic(#k_float{val=F}) -> {float,F}; -atomic(#k_atom{val=A}) -> {atom,A}; -%%atomic(#k_char{val=C}) -> {char,C}; -atomic(#k_nil{}) -> nil. - -%% new_label(St) -> {L,St}. - -new_label(#cg{lcount=Next}=St) -> - {Next,St#cg{lcount=Next+1}}. - -%% line(Le) -> {line,[] | {location,File,Line}} -%% Create a line instruction, containing information about -%% the current filename and line number. A line information -%% instruction should be placed before any operation that could -%% cause an exception. - -line(#l{a=Anno}) -> - line(Anno); -line([Line,{file,Name}]) when is_integer(Line) -> - line_1(Name, Line); -line([_|_]=A) -> - {Name,Line} = find_loc(A, no_file, 0), - line_1(Name, Line); -line([]) -> - {line,[]}. - -line_1(no_file, _) -> - {line,[]}; -line_1(_, 0) -> - %% Missing line number or line number 0. - {line,[]}; -line_1(Name, Line) -> - {line,[{location,Name,Line}]}. - -find_loc([Line|T], File, _) when is_integer(Line) -> - find_loc(T, File, Line); -find_loc([{file,File}|T], _, Line) -> - find_loc(T, File, Line); -find_loc([_|T], File, Line) -> - find_loc(T, File, Line); -find_loc([], File, Line) -> {File,Line}. - -flatmapfoldl(F, Accu0, [Hd|Tail]) -> - {R,Accu1} = F(Hd, Accu0), - {Rs,Accu2} = flatmapfoldl(F, Accu1, Tail), - {R++Rs,Accu2}; -flatmapfoldl(_, Accu, []) -> {[],Accu}. - -%% Keep track of life time for variables. -%% -%% init_vars([{var,VarName}]) -> Vdb. -%% new_vars([VarName], I, Vdb) -> Vdb. -%% use_vars([VarName], I, Vdb) -> Vdb. -%% add_var(VarName, F, L, Vdb) -> Vdb. -%% -%% The list of variable names for new_vars/3 and use_vars/3 -%% must be sorted. - -init_vars(Vs) -> - vdb_new(Vs). - -new_vars([], _, Vdb) -> Vdb; -new_vars([V], I, Vdb) -> vdb_store_new(V, {V,I,I}, Vdb); -new_vars(Vs, I, Vdb) -> vdb_update_vars(Vs, Vdb, I). - -use_vars([], _, Vdb) -> - Vdb; -use_vars([V], I, Vdb) -> - case vdb_find(V, Vdb) of - {V,F,L} when I > L -> vdb_update(V, {V,F,I}, Vdb); - {V,_,_} -> Vdb; - error -> vdb_store_new(V, {V,I,I}, Vdb) - end; -use_vars(Vs, I, Vdb) -> vdb_update_vars(Vs, Vdb, I). - -add_var(V, F, L, Vdb) -> - vdb_store_new(V, {V,F,L}, Vdb). - -%% vdb - -vdb_new(Vs) -> - ordsets:from_list([{V,0,0} || #k_var{name=V} <- Vs]). - --type var() :: atom(). - --spec vdb_find(var(), [vdb_entry()]) -> 'error' | vdb_entry(). - -vdb_find(V, Vdb) -> - case lists:keyfind(V, 1, Vdb) of - false -> error; - Vd -> Vd - end. - -vdb_update(V, Update, [{V,_,_}|Vdb]) -> - [Update|Vdb]; -vdb_update(V, Update, [Vd|Vdb]) -> - [Vd|vdb_update(V, Update, Vdb)]. - -vdb_store_new(V, New, [{V1,_,_}=Vd|Vdb]) when V > V1 -> - [Vd|vdb_store_new(V, New, Vdb)]; -vdb_store_new(V, New, [{V1,_,_}|_]=Vdb) when V < V1 -> - [New|Vdb]; -vdb_store_new(_, New, []) -> [New]. - -vdb_update_vars([V|_]=Vs, [{V1,_,_}=Vd|Vdb], I) when V > V1 -> - [Vd|vdb_update_vars(Vs, Vdb, I)]; -vdb_update_vars([V|Vs], [{V1,_,_}|_]=Vdb, I) when V < V1 -> - %% New variable. - [{V,I,I}|vdb_update_vars(Vs, Vdb, I)]; -vdb_update_vars([V|Vs], [{_,F,L}=Vd|Vdb], I) -> - %% Existing variable. - if - I > L -> [{V,F,I}|vdb_update_vars(Vs, Vdb, I)]; - true -> [Vd|vdb_update_vars(Vs, Vdb, I)] - end; -vdb_update_vars([V|Vs], [], I) -> - %% New variable. - [{V,I,I}|vdb_update_vars(Vs, [], I)]; -vdb_update_vars([], Vdb, _) -> Vdb. - -%% vdb_sub(Min, Max, Vdb) -> Vdb. -%% Extract variables which are used before and after Min. Lock -%% variables alive after Max. - -vdb_sub(Min, Max, Vdb) -> - [ if L >= Max -> {V,F,locked}; - true -> Vd - end || {V,F,L}=Vd <- Vdb, - F < Min, - L >= Min ]. diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl index 66e578b776..3699c9d22e 100644 --- a/lib/compiler/src/v3_core.erl +++ b/lib/compiler/src/v3_core.erl @@ -228,7 +228,8 @@ function({function,_,Name,Arity,Cs0}, Ws0, File, Opts) -> body(Cs0, Name, Arity, St0) -> Anno = lineno_anno(element(2, hd(Cs0)), St0), - {Args,St1} = new_vars(Anno, Arity, St0), + {Args0,St1} = new_vars(Anno, Arity, St0), + Args = reverse(Args0), %Nicer order case clauses(Cs0, St1) of {Cs1,[],St2} -> {Ps,St3} = new_vars(Arity, St2), %Need new variables here @@ -329,7 +330,7 @@ gexpr({protect,Line,Arg}, Bools0, St0) -> {#iprotect{anno=#a{anno=Anno},body=Eps++[E]},[],Bools0,St} end; gexpr({op,_,'andalso',_,_}=E0, Bools, St0) -> - {op,L,'andalso',E1,E2} = right_assoc(E0, 'andalso', St0), + {op,L,'andalso',E1,E2} = right_assoc(E0, 'andalso'), Anno = lineno_anno(L, St0), {#c_var{name=V0},St} = new_var(Anno, St0), V = {var,L,V0}, @@ -337,7 +338,7 @@ gexpr({op,_,'andalso',_,_}=E0, Bools, St0) -> E = make_bool_switch_guard(L, E1, V, E2, False), gexpr(E, Bools, St); gexpr({op,_,'orelse',_,_}=E0, Bools, St0) -> - {op,L,'orelse',E1,E2} = right_assoc(E0, 'orelse', St0), + {op,L,'orelse',E1,E2} = right_assoc(E0, 'orelse'), Anno = lineno_anno(L, St0), {#c_var{name=V0},St} = new_var(Anno, St0), V = {var,L,V0}, @@ -767,7 +768,7 @@ expr({op,_,'++',{lc,Llc,E,Qs0},More}, St0) -> {Y,Yps,St} = lc_tq(Llc, E, Qs, Mc, St2), {Y,Mps++Yps,St}; expr({op,_,'andalso',_,_}=E0, St0) -> - {op,L,'andalso',E1,E2} = right_assoc(E0, 'andalso', St0), + {op,L,'andalso',E1,E2} = right_assoc(E0, 'andalso'), Anno = lineno_anno(L, St0), {#c_var{name=V0},St} = new_var(Anno, St0), V = {var,L,V0}, @@ -775,7 +776,7 @@ expr({op,_,'andalso',_,_}=E0, St0) -> E = make_bool_switch(L, E1, V, E2, False, St0), expr(E, St); expr({op,_,'orelse',_,_}=E0, St0) -> - {op,L,'orelse',E1,E2} = right_assoc(E0, 'orelse', St0), + {op,L,'orelse',E1,E2} = right_assoc(E0, 'orelse'), Anno = lineno_anno(L, St0), {#c_var{name=V0},St} = new_var(Anno, St0), V = {var,L,V0}, @@ -2059,17 +2060,9 @@ fail_clause(Pats, Anno, Arg) -> args=[Arg]}]}. %% Optimization for Dialyzer. -right_assoc(E, Op, St) -> - case member(dialyzer, St#core.opts) of - true -> - right_assoc2(E, Op); - false -> - E - end. - -right_assoc2({op,L1,Op,{op,L2,Op,E1,E2},E3}, Op) -> - right_assoc2({op,L2,Op,E1,{op,L1,Op,E2,E3}}, Op); -right_assoc2(E, _Op) -> E. +right_assoc({op,L1,Op,{op,L2,Op,E1,E2},E3}, Op) -> + right_assoc({op,L2,Op,E1,{op,L1,Op,E2,E3}}, Op); +right_assoc(E, _Op) -> E. annotate_tuple(A, Es, St) -> case member(dialyzer, St#core.opts) of diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl index aef0b6cc9f..e2b8787224 100644 --- a/lib/compiler/src/v3_kernel.erl +++ b/lib/compiler/src/v3_kernel.erl @@ -82,8 +82,7 @@ -export([module/2,format_error/1]). -import(lists, [map/2,foldl/3,foldr/3,mapfoldl/3,splitwith/2,member/2, - keymember/3,keyfind/3,partition/2,droplast/1,last/1,sort/1, - reverse/1]). + keyfind/3,partition/2,droplast/1,last/1,sort/1,reverse/1]). -import(ordsets, [add_element/2,del_element/2,union/2,union/1,subtract/2]). -import(cerl, [c_tuple/1]). @@ -1415,8 +1414,6 @@ is_remote_bif(_, _, _) -> false. %% return multiple values. Only used in bodies where a BIF may be %% called for effect only. -bif_vals(dsetelement, 3) -> 0; -bif_vals(bs_context_to_binary, 1) -> 0; bif_vals(_, _) -> 1. bif_vals(_, _, _) -> 1. @@ -1593,19 +1590,12 @@ match_var([U|Us], Cs0, Def, St) -> %% constructor/constant as first argument. Group the constructors %% according to type, the order is really irrelevant but tries to be %% smart. - -match_con(Us, Cs0, Def, St) -> - %% Expand literals at the top level. - Cs = [expand_pat_lit_clause(C) || C <- Cs0], - match_con_1(Us, Cs, Def, St). - -match_con_1([U|_Us] = L, Cs, Def, St0) -> +match_con([U|_Us] = L, Cs, Def, St0) -> %% Extract clauses for different constructors (types). %%ok = io:format("match_con ~p~n", [Cs]), - Ttcs0 = select_types([k_binary], Cs) ++ select_bin_con(Cs) ++ - select_types([k_cons,k_tuple,k_map,k_atom,k_float, - k_int,k_nil], Cs), - Ttcs = opt_single_valued(Ttcs0), + Ttcs0 = select_types(Cs, [], [], [], [], [], [], [], [], []), + Ttcs1 = [{T, Types} || {T, [_ | _] = Types} <- Ttcs0], + Ttcs = opt_single_valued(Ttcs1), %%ok = io:format("ttcs = ~p~n", [Ttcs]), {Scs,St1} = mapfoldl(fun ({T,Tcs}, St) -> @@ -1616,8 +1606,41 @@ match_con_1([U|_Us] = L, Cs, Def, St0) -> St0, Ttcs), {build_alt_1st_no_fail(build_select(U, Scs), Def),St1}. -select_types(Types, Cs) -> - [{T,Tcs} || T <- Types, begin Tcs = select(T, Cs), Tcs =/= [] end]. +select_types([NoExpC | Cs], Bin, BinCon, Cons, Tuple, Map, Atom, Float, Int, Nil) -> + C = expand_pat_lit_clause(NoExpC), + case clause_con(C) of + k_binary -> + select_types(Cs, [C |Bin], BinCon, Cons, Tuple, Map, Atom, Float, Int, Nil); + k_bin_seg -> + select_types(Cs, Bin, [C | BinCon], Cons, Tuple, Map, Atom, Float, Int, Nil); + k_bin_end -> + select_types(Cs, Bin, [C | BinCon], Cons, Tuple, Map, Atom, Float, Int, Nil); + k_cons -> + select_types(Cs, Bin, BinCon, [C | Cons], Tuple, Map, Atom, Float, Int, Nil); + k_tuple -> + select_types(Cs, Bin, BinCon, Cons, [C | Tuple], Map, Atom, Float, Int, Nil); + k_map -> + select_types(Cs, Bin, BinCon, Cons, Tuple, [C | Map], Atom, Float, Int, Nil); + k_atom -> + select_types(Cs, Bin, BinCon, Cons, Tuple, Map, [C | Atom], Float, Int, Nil); + k_float -> + select_types(Cs, Bin, BinCon, Cons, Tuple, Map, Atom, [C | Float], Int, Nil); + k_int -> + select_types(Cs, Bin, BinCon, Cons, Tuple, Map, Atom, Float, [C | Int], Nil); + k_nil -> + select_types(Cs, Bin, BinCon, Cons, Tuple, Map, Atom, Float, Int, [C | Nil]) + end; +select_types([], Bin, BinCon, Cons, Tuple, Map, Atom, Float, Int, Nil) -> + [{k_binary, reverse(Bin)}] ++ handle_bin_con(reverse(BinCon)) ++ + [ + {k_cons, reverse(Cons)}, + {k_tuple, reverse(Tuple)}, + {k_map, reverse(Map)}, + {k_atom, reverse(Atom)}, + {k_float, reverse(Float)}, + {k_int, reverse(Int)}, + {k_nil, reverse(Nil)} + ]. expand_pat_lit_clause(#iclause{pats=[#ialias{pat=#k_literal{anno=A,val=Val}}=Alias|Ps]}=C) -> P = expand_pat_lit(Val, A), @@ -1746,20 +1769,12 @@ combine_bin_segs(#k_bin_end{}) -> combine_bin_segs(_) -> throw(not_possible). -%% select_bin_con([Clause]) -> [{Type,[Clause]}]. -%% Extract clauses for the k_bin_seg constructor. As k_bin_seg +%% handle_bin_con([Clause]) -> [{Type,[Clause]}]. +%% Handle clauses for the k_bin_seg constructor. As k_bin_seg %% matching can overlap, the k_bin_seg constructors cannot be %% reordered, only grouped. -select_bin_con(Cs0) -> - Cs1 = lists:filter(fun (C) -> - Con = clause_con(C), - (Con =:= k_bin_seg) or (Con =:= k_bin_end) - end, Cs0), - select_bin_con_1(Cs1). - - -select_bin_con_1(Cs) -> +handle_bin_con(Cs) -> try %% The usual way to match literals is to first extract the %% value to a register, and then compare the register to the @@ -1798,14 +1813,14 @@ select_bin_con_1(Cs) -> end catch throw:not_possible -> - select_bin_con_2(Cs) + handle_bin_con_not_possible(Cs) end. -select_bin_con_2([C1|Cs]) -> +handle_bin_con_not_possible([C1|Cs]) -> Con = clause_con(C1), {More,Rest} = splitwith(fun (C) -> clause_con(C) =:= Con end, Cs), - [{Con,[C1|More]}|select_bin_con_2(Rest)]; -select_bin_con_2([]) -> []. + [{Con,[C1|More]}|handle_bin_con_not_possible(Rest)]; +handle_bin_con_not_possible([]) -> []. %% select_bin_int([Clause]) -> {k_bin_int,[Clause]} %% If the first pattern in each clause selects the same integer, @@ -1905,10 +1920,6 @@ select_utf8(Val0) -> throw(not_possible) end. -%% select(Con, [Clause]) -> [Clause]. - -select(T, Cs) -> [ C || C <- Cs, clause_con(C) =:= T ]. - %% match_value([Var], Con, [Clause], Default, State) -> {SelectExpr,State}. %% At this point all the clauses have the same constructor, we must %% now separate them according to value. @@ -2044,8 +2055,9 @@ get_match(#k_binary{}, St0) -> {[V]=Mes,St1} = new_vars(1, St0), {#k_binary{segs=V},Mes,St1}; get_match(#k_bin_seg{size=#k_atom{val=all},next={k_bin_end,[]}}=Seg, St0) -> - {[S]=Vars,St1} = new_vars(1, St0), - {Seg#k_bin_seg{seg=S,next=[]},Vars,St1}; + {[S,N0],St1} = new_vars(2, St0), + N = set_kanno(N0, [no_usage]), + {Seg#k_bin_seg{seg=S,next=N},[S],St1}; get_match(#k_bin_seg{}=Seg, St0) -> {[S,N0],St1} = new_vars(2, St0), N = set_kanno(N0, [no_usage]), @@ -2343,8 +2355,7 @@ uexpr(#k_bif{anno=A,op=Op,args=As}=Bif, {break,Rs}, St0) -> {Brs,St1} = bif_returns(Op, Rs, St0), {Bif#k_bif{anno=#k{us=Used,ns=lit_list_vars(Brs),a=A},ret=Brs}, Used,St1}; -uexpr(#k_match{anno=A,vars=Vs0,body=B0}, Br, St0) -> - Vs = handle_reuse_annos(Vs0, St0), +uexpr(#k_match{anno=A,vars=Vs,body=B0}, Br, St0) -> Rs = break_rets(Br), {B1,Bu,St1} = umatch(B0, Br, St0), case is_in_guard(St1) of @@ -2374,9 +2385,10 @@ uexpr(#k_try{anno=A,arg=A0,vars=Vs,body=B0,evars=Evs,handler=H0}, true -> {[#k_var{name=X}],#k_var{name=X}} = {Vs,B0}, %Assertion. #k_atom{val=false} = H0, %Assertion. - {A1,Bu,St1} = uexpr(A0, Br, St0), + {Avs,St1} = new_vars(length(Rs0), St0), + {A1,Bu,St} = uexpr(A0, {break,Avs}, St1), {#k_protected{anno=#k{us=Bu,ns=lit_list_vars(Rs0),a=A}, - arg=A1,ret=Rs0},Bu,St1}; + arg=A1,ret=Rs0,inner=Avs},Bu,St}; false -> {Avs,St1} = new_vars(length(Vs), St0), {A1,Au,St2} = ubody(A0, {break,Avs}, St1), @@ -2446,33 +2458,6 @@ make_fdef(Anno, Name, Arity, Vs, Body) -> vars=Vs,body=Body,ret=[]}, #k_fdef{anno=Anno,func=Name,arity=Arity,vars=Vs,body=Match}. - -%% handle_reuse_annos([#k_var{}], State) -> State. -%% In general, it is only safe to reuse a variable for a match context -%% if the original value of the variable will no longer be needed. -%% -%% If a variable has been bound in an outer letrec and is therefore -%% free in the current function, the variable may still be used. -%% We don't bother to check whether the variable is actually used, -%% but simply clears the 'reuse_for_context' annotation for any variable -%% that is free. -handle_reuse_annos(Vs, St) -> - [handle_reuse_anno(V, St) || V <- Vs]. - -handle_reuse_anno(#k_var{anno=A}=V, St) -> - case member(reuse_for_context, A) of - false -> V; - true -> handle_reuse_anno_1(V, St) - end. - -handle_reuse_anno_1(#k_var{anno=Anno,name=Vname}=V, #kern{ff={F,A}}=St) -> - FreeVs = get_free(F, A, St), - case keymember(Vname, #k_var.name, FreeVs) of - true -> V#k_var{anno=Anno--[reuse_for_context]}; - false -> V - end; -handle_reuse_anno_1(V, _St) -> V. - %% get_free(Name, Arity, State) -> [Free]. %% store_free(Name, Arity, [Free], State) -> State. @@ -2516,8 +2501,7 @@ umatch(#k_alt{anno=A,first=F0,then=T0}, Br, St0) -> Used = union(Fu, Tu), {#k_alt{anno=#k{us=Used,ns=[],a=A},first=F1,then=T1}, Used,St2}; -umatch(#k_select{anno=A,var=V0,types=Ts0}, Br, St0) -> - V = handle_reuse_anno(V0, St0), +umatch(#k_select{anno=A,var=V,types=Ts0}, Br, St0) -> {Ts1,Tus,St1} = umatch_list(Ts0, Br, St0), Used = case member(no_usage, get_kanno(V)) of true -> Tus; diff --git a/lib/compiler/src/v3_kernel.hrl b/lib/compiler/src/v3_kernel.hrl index e6f0d3c1f7..e26360a6da 100644 --- a/lib/compiler/src/v3_kernel.hrl +++ b/lib/compiler/src/v3_kernel.hrl @@ -66,7 +66,7 @@ -record(k_receive_next, {anno=[]}). -record(k_try, {anno=[],arg,vars,body,evars,handler,ret=[]}). -record(k_try_enter, {anno=[],arg,vars,body,evars,handler}). --record(k_protected, {anno=[],arg,ret=[]}). +-record(k_protected, {anno=[],arg,ret=[],inner}). -record(k_catch, {anno=[],body,ret=[]}). -record(k_guard_match, {anno=[],vars,body,ret=[]}). diff --git a/lib/compiler/test/Makefile b/lib/compiler/test/Makefile index da5d207db9..db8eb7e2e1 100644 --- a/lib/compiler/test/Makefile +++ b/lib/compiler/test/Makefile @@ -14,6 +14,7 @@ MODULES= \ beam_except_SUITE \ beam_jump_SUITE \ beam_reorder_SUITE \ + beam_ssa_SUITE \ beam_type_SUITE \ beam_utils_SUITE \ bif_SUITE \ @@ -52,6 +53,7 @@ NO_OPT= \ beam_except \ beam_jump \ beam_reorder \ + beam_ssa \ beam_type \ beam_utils \ bif \ @@ -75,6 +77,7 @@ INLINE= \ andor \ apply \ beam_block \ + beam_ssa \ beam_utils \ bif \ bs_bincomp \ @@ -94,16 +97,30 @@ INLINE= \ receive \ record +R21= \ + bs_construct \ + bs_match + CORE_MODULES = \ lfe_andor_SUITE \ lfe_guard_SUITE +NO_MOD_OPT = $(NO_OPT) + +NO_SSA_OPT = $(NO_OPT) + NO_OPT_MODULES= $(NO_OPT:%=%_no_opt_SUITE) NO_OPT_ERL_FILES= $(NO_OPT_MODULES:%=%.erl) POST_OPT_MODULES= $(NO_OPT:%=%_post_opt_SUITE) POST_OPT_ERL_FILES= $(POST_OPT_MODULES:%=%.erl) INLINE_MODULES= $(INLINE:%=%_inline_SUITE) INLINE_ERL_FILES= $(INLINE_MODULES:%=%.erl) +R21_MODULES= $(R21:%=%_r21_SUITE) +R21_ERL_FILES= $(R21_MODULES:%=%.erl) +NO_MOD_OPT_MODULES= $(NO_MOD_OPT:%=%_no_module_opt_SUITE) +NO_MOD_OPT_ERL_FILES= $(NO_MOD_OPT_MODULES:%=%.erl) +NO_SSA_OPT_MODULES= $(NO_SSA_OPT:%=%_no_ssa_opt_SUITE) +NO_SSA_OPT_ERL_FILES= $(NO_SSA_OPT_MODULES:%=%.erl) ERL_FILES= $(MODULES:%=%.erl) CORE_FILES= $(CORE_MODULES:%=%.core) @@ -124,7 +141,7 @@ RELSYSDIR = $(RELEASE_PATH)/compiler_test # ---------------------------------------------------- ERL_MAKE_FLAGS += -ERL_COMPILE_FLAGS += +clint +clint0 +ERL_COMPILE_FLAGS += +clint +clint0 +ssalint EBIN = . @@ -132,15 +149,24 @@ EBIN = . # Targets # ---------------------------------------------------- -make_emakefile: $(NO_OPT_ERL_FILES) $(POST_OPT_ERL_FILES) $(INLINE_ERL_FILES) +make_emakefile: $(NO_OPT_ERL_FILES) $(POST_OPT_ERL_FILES) $(NO_SSA_OPT_ERL_FILES) \ + $(INLINE_ERL_FILES) $(R21_ERL_FILES) $(NO_MOD_OPT_ERL_FILES) $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) \ > $(EMAKEFILE) - $(ERL_TOP)/make/make_emakefile +no_copt +no_postopt $(ERL_COMPILE_FLAGS) \ + $(ERL_TOP)/make/make_emakefile +no_copt +no_postopt \ + +no_ssa_opt +no_recv_opt $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(NO_OPT_MODULES) >> $(EMAKEFILE) + $(ERL_TOP)/make/make_emakefile +no_share_opt +no_bsm_opt +no_fun_opt \ + +no_ssa_opt +no_recv_opt $(ERL_COMPILE_FLAGS) \ + -o$(EBIN) $(NO_SSA_OPT_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +no_copt $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(POST_OPT_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +inline $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(INLINE_MODULES) >> $(EMAKEFILE) + $(ERL_TOP)/make/make_emakefile +r21 $(ERL_COMPILE_FLAGS) \ + -o$(EBIN) $(R21_MODULES) >> $(EMAKEFILE) + $(ERL_TOP)/make/make_emakefile +no_module_opt $(ERL_COMPILE_FLAGS) \ + -o$(EBIN) $(NO_MOD_OPT_MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +from_core $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(CORE_MODULES) >> $(EMAKEFILE) @@ -161,12 +187,21 @@ docs: %_no_opt_SUITE.erl: %_SUITE.erl sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ +%_no_ssa_opt_SUITE.erl: %_SUITE.erl + sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ + %_post_opt_SUITE.erl: %_SUITE.erl sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ %_inline_SUITE.erl: %_SUITE.erl sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ +%_r21_SUITE.erl: %_SUITE.erl + sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ + +%_no_module_opt_SUITE.erl: %_SUITE.erl + sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ + # ---------------------------------------------------- # Release Target # ---------------------------------------------------- @@ -179,7 +214,9 @@ release_tests_spec: make_emakefile $(INSTALL_DATA) compiler.spec compiler.cover \ $(EMAKEFILE) $(ERL_FILES) "$(RELSYSDIR)" $(INSTALL_DATA) $(NO_OPT_ERL_FILES) $(POST_OPT_ERL_FILES) \ - $(INLINE_ERL_FILES) "$(RELSYSDIR)" + $(INLINE_ERL_FILES) $(R21_ERL_FILES) \ + $(NO_MOD_OPT_ERL_FILES) \ + $(NO_SSA_OPT_ERL_FILES) "$(RELSYSDIR)" $(INSTALL_DATA) $(CORE_FILES) "$(RELSYSDIR)" for file in $(ERL_DUMMY_FILES); do \ module=`basename $$file .erl`; \ diff --git a/lib/compiler/test/apply_SUITE.erl b/lib/compiler/test/apply_SUITE.erl index 0f82a56fb7..2ee518b1a0 100644 --- a/lib/compiler/test/apply_SUITE.erl +++ b/lib/compiler/test/apply_SUITE.erl @@ -73,6 +73,7 @@ mfa(Config) when is_list(Config) -> {'EXIT',_} = (catch ?APPLY2(Mod, (id(bazzzzzz)), a, b)), {'EXIT',_} = (catch ?APPLY2({}, baz, a, b)), {'EXIT',_} = (catch ?APPLY2(?MODULE, [], a, b)), + {'EXIT',_} = (catch bad_literal_call(1)), ok = apply(Mod, foo, id([])), {[a,b|c]} = apply(Mod, bar, id([[a,b|c]])), @@ -92,6 +93,13 @@ mfa(Config) when is_list(Config) -> apply(Mod, foo, []). +%% The single call to this function with a literal argument caused type +%% optimization to swap out the 'mod' field of a #b_remote{}, which was +%% mishandled during code generation as it assumed that the module would always +%% be an atom. +bad_literal_call(I) -> + I:foo(). + foo() -> ok. diff --git a/lib/compiler/test/beam_except_SUITE.erl b/lib/compiler/test/beam_except_SUITE.erl index 2b4a780899..8e3b373d29 100644 --- a/lib/compiler/test/beam_except_SUITE.erl +++ b/lib/compiler/test/beam_except_SUITE.erl @@ -21,7 +21,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, - multiple_allocs/1,coverage/1]). + multiple_allocs/1,bs_get_tail/1,coverage/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -31,6 +31,7 @@ all() -> groups() -> [{p,[parallel], [multiple_allocs, + bs_get_tail, coverage]}]. init_per_suite(Config) -> @@ -63,6 +64,17 @@ place(lee) -> conditions() -> (talking = going) = storage + [large = wanted]. +bs_get_tail(Config) -> + {<<"abc">>,0,0,Config} = bs_get_tail_1(id(<<0:32, "abc">>), 0, 0, Config), + {'EXIT', + {function_clause, + [{?MODULE,bs_get_tail_1,[<<>>,0,0,Config],_}|_]}} = + (catch bs_get_tail_1(id(<<>>), 0, 0, Config)), + ok. + +bs_get_tail_1(<<_:32, Rest/binary>>, Z1, Z2, F1) -> + {Rest,Z1,Z2,F1}. + coverage(_) -> File = {file,"fake.erl"}, ok = fc(a), @@ -72,9 +84,16 @@ coverage(_) -> {'EXIT',{function_clause, [{?MODULE,fc,[y],[File,{line,2}]}|_]}} = (catch fc(y)), - {'EXIT',{function_clause, - [{?MODULE,fc,[[a,b,c]],[File,{line,6}]}|_]}} = - (catch fc([a,b,c])), + case ?MODULE of + beam_except_no_opt_SUITE -> + %% There will be a different stack fram in + %% unoptimized code. + ok; + _ -> + {'EXIT',{function_clause, + [{?MODULE,fc,[[a,b,c]],[File,{line,6}]}|_]}} = + (catch fc([a,b,c])) + end, {'EXIT',{undef,[{erlang,error,[a,b,c],_}|_]}} = (catch erlang:error(a, b, c)), @@ -83,8 +102,24 @@ coverage(_) -> (catch bar(x)), {'EXIT',{{case_clause,{1}},[{?MODULE,bar,1,[File,{line,9}]}|_]}} = (catch bar(0)), + + Self = self(), + {'EXIT',{{strange,Self},[{?MODULE,foo,[any],[File,{line,14}]}|_]}} = + (catch foo(any)), + + {ok,succeed,1,2} = foobar(succeed, 1, 2), + {'EXIT',{function_clause,[{?MODULE,foobar,[[fail],1,2], + [{file,"fake.erl"},{line,16}]}|_]}} = + (catch foobar([fail], 1, 2)), + {'EXIT',{function_clause,[{?MODULE,fake_function_clause,[{a,b},42.0],_}|_]}} = + (catch fake_function_clause({a,b})), + ok. +fake_function_clause(A) -> error(function_clause, [A,42.0]). + +id(I) -> I. + -file("fake.erl", 1). fc(a) -> %Line 2 ok; %Line 3 @@ -96,3 +131,9 @@ bar(X) -> %Line 8 case {X+1} of %Line 9 1 -> ok %Line 10 end. %Line 11 +%% Cover collection code for function_clause exceptions. +foo(A) -> %Line 13 + error({strange,self()}, [A]). %Line 14 +%% Cover beam_except:tag_literal/1. +foobar(A, B, C) when is_atom(A) -> %Line 16 + {ok,A,B,C}. %Line 17 diff --git a/lib/compiler/test/beam_jump_SUITE.erl b/lib/compiler/test/beam_jump_SUITE.erl index faedc0c1f1..a456f31d79 100644 --- a/lib/compiler/test/beam_jump_SUITE.erl +++ b/lib/compiler/test/beam_jump_SUITE.erl @@ -22,7 +22,8 @@ -export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, init_per_group/2,end_per_group/2, undefined_label/1,ambiguous_catch_try_state/1, - build_tuple/1]). + unsafe_move_elimination/1,build_tuple/1, + coverage/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -34,7 +35,9 @@ groups() -> [{p,[parallel], [undefined_label, ambiguous_catch_try_state, - build_tuple + unsafe_move_elimination, + build_tuple, + coverage ]}]. init_per_suite(Config) -> @@ -75,6 +78,82 @@ checks(Wanted) -> %% Must be one line to cause the unsafe optimization. {catch case river() of sheet -> begin +Wanted, if "da" -> Wanted end end end, catch case river() of sheet -> begin + Wanted, if "da" -> Wanted end end end}. +unsafe_move_elimination(_Config) -> + {{left,right,false},false} = unsafe_move_elimination_1(left, right, false), + {{false,right,false},false} = unsafe_move_elimination_1(false, right, true), + {{true,right,right},right} = unsafe_move_elimination_1(true, right, true), + [ok = unsafe_move_elimination_2(I) || I <- lists:seq(0,16)], + ok. + +unsafe_move_elimination_1(Left, Right, Simple0) -> + id(1), + + %% The move at label 29 would be removed by beam_jump, which is unsafe because + %% the two select_val instructions have different source registers. + %% + %% {select_val,{y,0},{f,25},{list,[{atom,true},{f,27},{atom,false},{f,29}]}}. + %% ^^^^^ ^^^^^^^^^^^^^^^^^^^ + %% {label,27}. + %% {kill,{y,0}}. + %% {move,{y,2},{x,0}}. + %% {line,...}. + %% {call,1,{f,31}}. + %% {select_val,{x,0},{f,33},{list,[{atom,true},{f,35},{atom,false},{f,29}]}}. + %% ^^^^^ ^^^^^^^^^^^^^^^^^^^ + %% {label,29}. + %% {move,{atom,false},{y,0}}. <=== REMOVED (unsafely). + %% {jump,{f,37}}. + + Simple = case case Simple0 of + false -> false; + true -> id(Left) + end + of + false -> + false; + true -> + id(Right) + end, + {id({Left,Right,Simple}),Simple}. + +unsafe_move_elimination_2(Int) -> + %% The type optimization pass would recognize that TagInt can only be + %% [0 .. 7], so the first 'case' would select_val over [0 .. 6] and swap + %% out the fail label with the block for 7. + %% + %% A later optimization would merge this block with 'expects_h' in the + %% second case, as the latter is only reachable from the former. + %% + %% ... but this broke down when the move elimination optimization didn't + %% take the fail label of the first select_val into account. This caused it + %% to believe that the only way to reach 'expects_h' was through the second + %% case when 'Tag' =:= 'h', which made it remove the move instruction + %% added in the first case, passing garbage to expects_h/2. + TagInt = Int band 2#111, + Tag = case TagInt of + 0 -> a; + 1 -> b; + 2 -> c; + 3 -> d; + 4 -> e; + 5 -> f; + 6 -> g; + 7 -> h + end, + case Tag of + g -> expects_g(TagInt, Tag); + h -> expects_h(TagInt, Tag); + _ -> Tag = id(Tag), ok + end. + +expects_g(6, Atom) -> + Atom = id(g), + ok. + +expects_h(7, Atom) -> + Atom = id(h), + ok. + -record(message2, {id, p1}). -record(message3, {id, p1, p2}). @@ -87,3 +166,45 @@ do_build_tuple(Message) -> Res = {res, rand:uniform(100)}, {Message#message3.id, Res} end. + +coverage(_Config) -> + ok = coverage_1(ok), + {error,badarg} = coverage_1({error,badarg}), + + gt = coverage_2(100, 42), + le = coverage_2(100, 999), + le = coverage_2([], []), + gt = coverage_2([], xxx), + + ok. + +coverage_1(Var) -> + case id(Var) of + ok -> ok; + Error -> Error + end. + +%% Cover beam_jump:invert_test(is_ne_exact). +coverage_2(Pre1, Pre2) -> + case + case Pre1 == [] of + false -> + false; + true -> + Pre2 /= [] + end + of + true -> + gt; + false -> + case Pre1 > Pre2 of + true -> + gt; + false -> + le + end + end. + + +id(I) -> + I. diff --git a/lib/compiler/test/beam_ssa_SUITE.erl b/lib/compiler/test/beam_ssa_SUITE.erl new file mode 100644 index 0000000000..15cf9bcbf3 --- /dev/null +++ b/lib/compiler/test/beam_ssa_SUITE.erl @@ -0,0 +1,497 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(beam_ssa_SUITE). + +-export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, + init_per_group/2,end_per_group/2, + calls/1,tuple_matching/1,recv/1,maps/1, + cover_ssa_dead/1,combine_sw/1,share_opt/1]). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [{group,p}]. + +groups() -> + [{p,test_lib:parallel(), + [tuple_matching, + calls, + recv, + maps, + cover_ssa_dead, + combine_sw, + share_opt + ]}]. + +init_per_suite(Config) -> + test_lib:recompile(?MODULE), + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +calls(Config) -> + Ret = {return,value,Config}, + Ret = fun_call(fun(42) -> ok end, Ret), + Ret = apply_fun(fun(a, b) -> ok end, [a,b], Ret), + Ret = apply_mfa(test_lib, id, [anything], Ret), + {'EXIT',{badarg,_}} = (catch call_error()), + {'EXIT',{badarg,_}} = (catch call_error(42)), + 5 = start_it([erlang,length,1,2,3,4,5]), + ok. + +fun_call(Fun, X0) -> + X = id(X0), + Fun(42), + X. + +apply_fun(Fun, Args, X0) -> + X = id(X0), + apply(Fun, Args), + X. + +apply_mfa(Mod, Name, Args, X0) -> + X = id(X0), + apply(Mod, Name, Args), + X. + +call_error() -> + error(badarg), + ok. + +call_error(I) -> + <<I:(-8)>>, + ok. + +start_it([_|_]=MFA) -> + case MFA of + [M,F|Args] -> M:F(Args) + end. + +tuple_matching(_Config) -> + do_tuple_matching({tag,42}), + + true = is_two_tuple({a,b}), + false = is_two_tuple({a,b,c}), + false = is_two_tuple(atom), + + ok. + +do_tuple_matching(Arg) -> + Res = do_tuple_matching_1(Arg), + Res = do_tuple_matching_2(Arg), + Res = do_tuple_matching_3(Arg), + Res. + +do_tuple_matching_1({tag,V}) -> + {ok,V}. + +do_tuple_matching_2(Tuple) when is_tuple(Tuple) -> + Size = tuple_size(Tuple), + if + Size =:= 2 -> + {ok,element(2, Tuple)} + end. + +do_tuple_matching_3(Tuple) when is_tuple(Tuple) -> + Size = tuple_size(Tuple), + if + Size =:= 2 -> + 2 = id(Size), + {ok,element(2, Tuple)} + end. + +is_two_tuple(Arg) -> + case is_tuple(Arg) of + false -> false; + true -> tuple_size(Arg) == 2 + end. + +-record(reporter_state, {res,run_config}). +-record(run_config, {report_interval=0}). + +recv(_Config) -> + Parent = self(), + + %% Test sync_wait_mon/2. + Succ = fun() -> Parent ! {ack,self(),{result,42}} end, + {result,42} = sync_wait_mon(spawn_monitor(Succ), infinity), + + Down = fun() -> exit(down) end, + {error,down} = sync_wait_mon(spawn_monitor(Down), infinity), + + Exit = fun() -> + Self = self(), + spawn(fun() -> exit(Self, kill_me) end), + receive _ -> ok end + end, + {error,kill_me} = sync_wait_mon(spawn_monitor(Exit), infinity), + + Timeout = fun() -> receive _ -> ok end end, + {error,timeout} = sync_wait_mon(spawn_monitor(Timeout), 0), + + %% Test reporter_loop/1. + {a,Parent} = reporter_loop(#reporter_state{res={a,Parent}, + run_config=#run_config{}}), + + %% Test bad_sink/0. + bad_sink(), + + %% Test tricky_recv_1/0. + self() ! 1, + a = tricky_recv_1(), + self() ! 2, + b = tricky_recv_1(), + + %% Test tricky_recv_2/0. + self() ! 1, + {1,yes} = tricky_recv_2(), + self() ! 2, + {2,maybe} = tricky_recv_2(), + + %% Test 'receive after infinity' in try/catch. + Pid = spawn(fun recv_after_inf_in_try/0), + exit(Pid, done), + + %% Test tricky_recv_3(). + self() ! {{self(),r0},{1,42,"name"}}, + {Parent,r0,[<<1:32,1:8,42:8>>,"name",0]} = tricky_recv_3(), + self() ! {{self(),r1},{2,99,<<"data">>}}, + {Parent,r1,<<1:32,2:8,99:8,"data">>} = tricky_recv_3(), + + %% Test tricky_recv_4(). + self() ! {[self(),r0],{1,42,"name"}}, + {Parent,r0,[<<1:32,1:8,42:8>>,"name",0]} = tricky_recv_4(), + self() ! {[self(),r1],{2,99,<<"data">>}}, + {Parent,r1,<<1:32,2:8,99:8,"data">>} = tricky_recv_4(), + + ok. + +sync_wait_mon({Pid, Ref}, Timeout) -> + receive + {ack,Pid,Return} -> + erlang:demonitor(Ref, [flush]), + Return; + {'DOWN',Ref,_Type,Pid,Reason} -> + {error,Reason}; + {'EXIT',Pid,Reason} -> + erlang:demonitor(Ref, [flush]), + {error,Reason} + after Timeout -> + erlang:demonitor(Ref, [flush]), + exit(Pid, kill), + {error,timeout} + end. + +reporter_loop(State) -> + RC = State#reporter_state.run_config, + receive after RC#run_config.report_interval -> + State#reporter_state.res + end. + +bad_sink() -> + {ok,Pid} = my_spawn(self()), + %% The get_tuple_element instruction for the matching + %% above was sinked into the receive loop. That will + %% not work (and would be bad for performance if it + %% would work). + receive + {ok,Pid} -> + ok; + error -> + exit(failed) + end, + exit(Pid, kill). + +my_spawn(Parent) -> + Pid = spawn(fun() -> + Parent ! {ok,self()}, + receive _ -> ok end + end), + {ok,Pid}. + +tricky_recv_1() -> + receive + X=1 -> + id(42), + a; + X=2 -> + b + end, + case X of + 1 -> a; + 2 -> b + end. + +tricky_recv_2() -> + receive + X=1 -> + Y = case id(X) of + 1 -> yes; + _ -> no + end, + a; + X=2 -> + Y = maybe, + b + end, + {X,Y}. + +recv_after_inf_in_try() -> + try + %% Used to crash beam_kernel_to_ssa. + receive after infinity -> ok end + catch + _A:_B -> + receive after infinity -> ok end + end. + +tricky_recv_3() -> + {Pid, R, Request} = + receive + {{Pid0,R0}, {1, Proto0, Name0}} -> + {Pid0, R0, + [<<1:32, 1:8, Proto0:8>>,Name0,0]}; + {{Pid1,R1}, {2, Proto1, Data1}} -> + {Pid1, R1, + <<1:32, 2:8, Proto1:8, Data1/binary>>} + end, + id({Pid,R,Request}). + +tricky_recv_4() -> + {Pid, R, Request} = + receive + {[Pid0,R0], {1, Proto0, Name0}} -> + {Pid0, R0, + [<<1:32, 1:8, Proto0:8>>,Name0,0]}; + {[Pid1,R1], {2, Proto1, Data1}} -> + {Pid1, R1, + <<1:32, 2:8, Proto1:8, Data1/binary>>} + end, + id({Pid,R,Request}). + +maps(_Config) -> + {'EXIT',{{badmatch,#{}},_}} = (catch maps_1(any)), + ok. + +maps_1(K) -> + _ = id(42), + #{K:=V} = #{}, + V. + +-record(wx_ref, {type=any_type,ref=any_ref}). + +cover_ssa_dead(_Config) -> + str = format_str(str, escapable, [], true), + [iolist,str] = format_str(str, escapable, iolist, true), + bad = format_str(str, not_escapable, [], true), + bad = format_str(str, not_escapable, iolist, true), + bad = format_str(str, escapable, [], false), + bad = format_str(str, escapable, [], bad), + + DefWxRef = #wx_ref{}, + {DefWxRef,77,9999,[]} = contains(#wx_ref{}, 77, 9999), + {DefWxRef,77.0,9999,[]} = contains(#wx_ref{}, 77.0, 9999), + {DefWxRef,77,9999.0,[]} = contains(#wx_ref{}, 77, 9999.0), + {DefWxRef,77.0,9999.0,[]} = contains(#wx_ref{}, 77.0, 9999.0), + {any_type,any_ref,42,43,[option]} = contains(#wx_ref{}, {42,43}, [option]), + {any_type,any_ref,42,43,[]} = contains(#wx_ref{}, {42,43}, []), + {any_type,any_ref,42.0,43,[]} = contains(#wx_ref{}, {42.0,43}, []), + {any_type,any_ref,42,43.0,[]} = contains(#wx_ref{}, {42,43.0}, []), + {any_type,any_ref,42.0,43.0,[]} = contains(#wx_ref{}, {42.0,43.0}, []), + + nope = conv_alub(false, '=:='), + ok = conv_alub(true, '=:='), + ok = conv_alub(true, none), + error = conv_alub(false, none), + + {false,false} = eval_alu(false, false, false), + {true,false} = eval_alu(false, false, true), + {false,true} = eval_alu(false, true, false), + {false,false} = eval_alu(false, true, true), + {false,true} = eval_alu(true, false, false), + {false,false} = eval_alu(true, false, true), + {true,true} = eval_alu(true, true, false), + {false,true} = eval_alu(true, true, true), + + 100.0 = percentage(1.0, 0.0), + 100.0 = percentage(1, 0), + 0.0 = percentage(0, 0), + 0.0 = percentage(0.0, 0.0), + 40.0 = percentage(4.0, 10.0), + 60.0 = percentage(6, 10), + + %% Cover '=:=', followed by '=/='. + false = 'cover__=:=__=/='(41), + true = 'cover__=:=__=/='(42), + false = 'cover__=:=__=/='(43), + + %% Cover '<', followed by '=/='. + true = 'cover__<__=/='(41), + false = 'cover__<__=/='(42), + false = 'cover__<__=/='(43), + + %% Cover '=<', followed by '=/='. + true = 'cover__=<__=/='(41), + true = 'cover__=<__=/='(42), + false = 'cover__=<__=/='(43), + + %% Cover '>=', followed by '=/='. + false = 'cover__>=__=/='(41), + true = 'cover__>=__=/='(42), + true = 'cover__>=__=/='(43), + + %% Cover '>', followed by '=/='. + false = 'cover__>__=/='(41), + false = 'cover__>__=/='(42), + true = 'cover__>__=/='(43), + + ok. + +'cover__=:=__=/='(X) when X =:= 42 -> X =/= 43; +'cover__=:=__=/='(_) -> false. + +'cover__<__=/='(X) when X < 42 -> X =/= 42; +'cover__<__=/='(_) -> false. + +'cover__=<__=/='(X) when X =< 42 -> X =/= 43; +'cover__=<__=/='(_) -> false. + +'cover__>=__=/='(X) when X >= 42 -> X =/= 41; +'cover__>=__=/='(_) -> false. + +'cover__>__=/='(X) when X > 42 -> X =/= 42; +'cover__>__=/='(_) -> false. + +format_str(Str, FormatData, IoList, EscChars) -> + Escapable = FormatData =:= escapable, + case id(Str) of + IoStr when Escapable, EscChars, IoList == [] -> + id(IoStr); + IoStr when Escapable, EscChars -> + [IoList,id(IoStr)]; + _ -> + bad + end. + +contains(This, X, Y) when is_record(This, wx_ref), is_number(X), is_number(Y) -> + {This,X,Y,[]}; +contains(#wx_ref{type=ThisT,ref=ThisRef}, {CX,CY}, Options) + when is_number(CX), is_number(CY), is_list(Options) -> + {ThisT,ThisRef,CX,CY,Options}. + +conv_alub(HasDst, CmpOp) -> + case (not HasDst) andalso CmpOp =/= none of + true -> nope; + false -> + case HasDst of + false -> error; + true -> ok + end + end. + +eval_alu(Sign1, Sign2, N) -> + V = (Sign1 andalso Sign2 andalso (not N)) + or ((not Sign1) andalso (not Sign2) andalso N), + C = (Sign1 andalso Sign2) + or ((not N) andalso (Sign1 orelse Sign2)), + {V,C}. + +percentage(Divident, Divisor) -> + if Divisor == 0 andalso Divident /= 0 -> + 100.0; + Divisor == 0 -> + 0.0; + true -> + Divident / Divisor * 100 + end. + +combine_sw(_Config) -> + [a] = do_comb_sw_1(a), + [b,b] = do_comb_sw_1(b), + [c] = do_comb_sw_1(c), + [c] = do_comb_sw_1(c), + [] = do_comb_sw_1(z), + + [a] = do_comb_sw_2(a), + [b2,b1] = do_comb_sw_2(b), + [c] = do_comb_sw_2(c), + [c] = do_comb_sw_2(c), + [] = do_comb_sw_2(z), + + ok. + +do_comb_sw_1(X) -> + put(?MODULE, []), + if + X == a; X == b -> + put(?MODULE, [X|get(?MODULE)]); + true -> + ok + end, + if + X == b; X == c -> + put(?MODULE, [X|get(?MODULE)]); + true -> + ok + end, + erase(?MODULE). + +do_comb_sw_2(X) -> + put(?MODULE, []), + case X of + a -> + put(?MODULE, [a|get(?MODULE)]); + b -> + put(?MODULE, [b1|get(?MODULE)]); + _ -> + ok + end, + case X of + b -> + put(?MODULE, [b2|get(?MODULE)]); + c -> + put(?MODULE, [c|get(?MODULE)]); + _ -> + ok + end, + erase(?MODULE). + +share_opt(_Config) -> + ok = do_share_opt(0). + +do_share_opt(A) -> + %% The compiler would be stuck in an infinite loop in beam_ssa_share. + case A of + 0 -> a; + 1 -> b; + 2 -> c + end, + receive after 1 -> ok end. + + +%% The identity function. +id(I) -> I. diff --git a/lib/compiler/test/beam_type_SUITE.erl b/lib/compiler/test/beam_type_SUITE.erl index 9f691716e3..882e281a44 100644 --- a/lib/compiler/test/beam_type_SUITE.erl +++ b/lib/compiler/test/beam_type_SUITE.erl @@ -21,9 +21,10 @@ -export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, init_per_group/2,end_per_group/2, - integers/1,coverage/1,booleans/1,setelement/1,cons/1, - tuple/1,record_float/1,binary_float/1,float_compare/1, - arity_checks/1,elixir_binaries/1,find_best/1]). + integers/1,numbers/1,coverage/1,booleans/1,setelement/1, + cons/1,tuple/1,record_float/1,binary_float/1,float_compare/1, + arity_checks/1,elixir_binaries/1,find_best/1, + test_size/1,cover_lists_functions/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -33,6 +34,7 @@ all() -> groups() -> [{p,[parallel], [integers, + numbers, coverage, booleans, setelement, @@ -43,7 +45,9 @@ groups() -> float_compare, arity_checks, elixir_binaries, - find_best + find_best, + test_size, + cover_lists_functions ]}]. init_per_suite(Config) -> @@ -113,8 +117,8 @@ do_integers_4(_, _, Res) -> Res. do_integers_5(X0, Y0) -> - %% X and Y will use the same register. - X = X0 band 1, + %% _X and Y will use the same register. + _X = X0 band 1, Y = Y0 band 3, case Y of 0 -> zero; @@ -123,6 +127,59 @@ do_integers_5(X0, Y0) -> 3 -> three end. +numbers(_Config) -> + Int = id(42), + true = is_integer(Int), + true = is_number(Int), + false = is_float(Int), + + Float = id(42.0), + true = is_float(Float), + true = is_number(Float), + false = is_integer(Float), + + Number = id(1) + id(2), + true = is_number(Number), + true = is_integer(Number), + false = is_float(Number), + + AnotherNumber = id(99.0) + id(1), + true = is_float(AnotherNumber), + true = is_number(AnotherNumber), + false = is_integer(AnotherNumber), + + NotNumber = id(atom), + true = is_atom(NotNumber), + false = is_number(NotNumber), + false = is_integer(NotNumber), + false = is_float(NotNumber), + + true = is_number(Int), + true = is_number(Float), + true = is_number(Number), + true = is_number(AnotherNumber), + + %% Cover beam_ssa_type:join/2. + + Join1 = case id(a) of + a -> 3 + id(7); %Number. + b -> id(5) / id(2) %Float. + end, + true = is_integer(Join1), + + Join2 = case id(a) of + a -> id(5) / 2; %Float. + b -> 3 + id(7) %Number. + end, + true = is_float(Join2), + + %% Cover beam_ssa_type:meet/2. + + Meet1 = id(0) + -10.0, %Float. + 10.0 = abs(Meet1), %Number. + + ok. + coverage(Config) -> {'EXIT',{badarith,_}} = (catch id(1) bsl 0.5), {'EXIT',{badarith,_}} = (catch id(2.0) bsl 2), @@ -164,15 +221,52 @@ coverage(Config) -> ok. booleans(_Config) -> - {'EXIT',{{case_clause,_},_}} = (catch do_booleans(42)), + {'EXIT',{{case_clause,_},_}} = (catch do_booleans_1(42)), + + ok = do_booleans_2(42, 41), + error = do_booleans_2(42, 42), + + AnyAtom = id(atom), + true = is_atom(AnyAtom), + false = is_boolean(AnyAtom), + + MaybeBool = id(maybe), + case MaybeBool of + true -> ok; + maybe -> ok; + false -> ok + end, + false = is_boolean(MaybeBool), + + NotBool = id(a), + case NotBool of + a -> ok; + b -> ok; + c -> ok + end, + false = is_boolean(NotBool), + ok. -do_booleans(B) -> +do_booleans_1(B) -> case is_integer(B) of yes -> yes; no -> no end. +do_booleans_2(A, B) -> + Not = not do_booleans_cmp(A, B), + case Not of + true -> + case Not of + true -> error; + false -> ok + end; + false -> ok + end. + +do_booleans_cmp(A, B) -> A > B. + setelement(_Config) -> T0 = id({a,42}), {a,_} = T0, @@ -181,13 +275,59 @@ setelement(_Config) -> cons(_Config) -> [did] = cons(assigned, did), + + true = cons_is_empty_list([]), + false = cons_is_empty_list([a]), + + false = cons_not(true), + true = cons_not(false), + + {$a,"bc"} = cons_hdtl(true), + {$d,"ef"} = cons_hdtl(false), ok. cons(assigned, Instrument) -> [Instrument] = [did]. +cons_is_empty_list(L) -> + Cons = case L of + [] -> "true"; + _ -> "false" + end, + id(1), + case Cons of + "true" -> true; + "false" -> false + end. + +cons_not(B) -> + Cons = case B of + true -> "true"; + false -> "false" + end, + id(1), + case Cons of + "true" -> false; + "false" -> true + end. + +cons_hdtl(B) -> + Cons = case B of + true -> "abc"; + false -> "def" + end, + id(1), + {id(hd(Cons)),id(tl(Cons))}. + +-record(bird, {a=a,b=id(42)}). + tuple(_Config) -> {'EXIT',{{badmatch,{necessary}},_}} = (catch do_tuple()), + + [] = [X || X <- [], #bird{a = a} == {r,X,foo}], + [] = [X || X <- [], #bird{b = b} == {bird,X}], + [] = [X || X <- [], 3 == X#bird.a], + ok. do_tuple() -> @@ -324,6 +464,28 @@ find_best([], <<"a">>) -> find_best([], nil) -> {error,<<"should not get here">>}. +test_size(_Config) -> + 2 = do_test_size({a,b}), + 4 = do_test_size(<<42:32>>), + ok. + +do_test_size(Term) when is_tuple(Term) -> + size(Term); +do_test_size(Term) when is_binary(Term) -> + size(Term). + +cover_lists_functions(Config) -> + case lists:suffix([no|Config], Config) of + true -> + ct:fail(should_be_false); + false -> + ok + end, + Zipped = lists:zipwith(fun(A, B) -> {A,B} end, + lists:duplicate(length(Config), zip), + Config), + true = is_list(Zipped), + ok. id(I) -> I. diff --git a/lib/compiler/test/beam_utils_SUITE.erl b/lib/compiler/test/beam_utils_SUITE.erl index ff0f72d519..eb0af59f9d 100644 --- a/lib/compiler/test/beam_utils_SUITE.erl +++ b/lib/compiler/test/beam_utils_SUITE.erl @@ -197,7 +197,7 @@ do_bs_init_4(Arg1, Arg2) -> id(Rewrite) end/binary, "/shared">>); - Other -> + _Other -> error end. @@ -553,7 +553,7 @@ not_used_p(_C, S, K, L) when is_record(K, k) -> id(K) end. -is_used_fr(Config) -> +is_used_fr(_Config) -> 1 = is_used_fr(self(), self()), 1 = is_used_fr(self(), other), receive 1 -> ok end, @@ -572,7 +572,7 @@ is_used_fr(X, Y) -> X ! 1. %% ERL-778. -unsafe_is_function(Config) -> +unsafe_is_function(_Config) -> {undefined,any} = unsafe_is_function(undefined, any), {ok,any} = unsafe_is_function(fun() -> ok end, any), {'EXIT',{{case_clause,_},_}} = (catch unsafe_is_function(fun(_) -> ok end, any)), diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl index d3e544a9cc..6b1438abdd 100644 --- a/lib/compiler/test/beam_validator_SUITE.erl +++ b/lib/compiler/test/beam_validator_SUITE.erl @@ -34,7 +34,8 @@ undef_label/1,illegal_instruction/1,failing_gc_guard_bif/1, map_field_lists/1,cover_bin_opt/1, val_dsetel/1,bad_tuples/1,bad_try_catch_nesting/1, - receive_stacked/1]). + receive_stacked/1,aliased_types/1,type_conflict/1, + infer_on_eq/1,infer_dead_value/1]). -include_lib("common_test/include/ct.hrl"). @@ -63,7 +64,8 @@ groups() -> undef_label,illegal_instruction,failing_gc_guard_bif, map_field_lists,cover_bin_opt,val_dsetel, bad_tuples,bad_try_catch_nesting, - receive_stacked]}]. + receive_stacked,aliased_types,type_conflict, + infer_on_eq,infer_dead_value]}]. init_per_suite(Config) -> test_lib:recompile(?MODULE), @@ -107,13 +109,12 @@ xrange(Config) when is_list(Config) -> Errors = do_val(xrange, Config), [{{t,sum_1,2}, {{bif,'+',{f,0},[{x,-1},{x,1}],{x,0}},4, - {uninitialized_reg,{x,-1}}}}, + {bad_register,{x,-1}}}}, {{t,sum_2,2}, - {{bif,'+',{f,0},[{x,0},{x,1023}],{x,0}},4, - {uninitialized_reg,{x,1023}}}}, + {{bif,'+',{f,0},[{x,0},{x,1023}],{x,0}},4,limit}}, {{t,sum_3,2}, {{bif,'+',{f,0},[{x,0},{x,1}],{x,-1}},4, - {invalid_store,{x,-1},number}}}, + {bad_register,{x,-1}}}}, {{t,sum_4,2}, {{bif,'+',{f,0},[{x,0},{x,1}],{x,1023}},4,limit}}] = Errors, ok. @@ -122,15 +123,15 @@ yrange(Config) when is_list(Config) -> Errors = do_val(yrange, Config), [{{t,sum_1,2}, {{move,{x,1},{y,-1}},5, - {invalid_store,{y,-1},term}}}, + {bad_register,{y,-1}}}}, {{t,sum_2,2}, {{bif,'+',{f,0},[{x,0},{y,1024}],{x,0}},7, - {uninitialized_reg,{y,1024}}}}, + limit}}, {{t,sum_3,2}, {{move,{x,1},{y,1024}},5,limit}}, {{t,sum_4,2}, {{move,{x,1},{y,-1}},5, - {invalid_store,{y,-1},term}}}] = Errors, + {bad_register,{y,-1}}}}] = Errors, ok. stack(Config) when is_list(Config) -> @@ -157,9 +158,9 @@ call_last(Config) when is_list(Config) -> merge_undefined(Config) when is_list(Config) -> Errors = do_val(merge_undefined, Config), [{{t,handle_call,2}, - {{call_ext,1,{extfunc,erlang,exit,1}}, - 10, - {uninitialized_reg,{y,0}}}}] = Errors, + {{call_ext,2,{extfunc,debug,filter,2}}, + 22, + {uninitialized_reg,{y,_}}}}] = Errors, ok. uninit(Config) when is_list(Config) -> @@ -178,7 +179,7 @@ unsafe_catch(Config) when is_list(Config) -> Errors = do_val(unsafe_catch, Config), [{{t,small,2}, {{bs_put_integer,{f,0},{integer,16},1, - {field_flags,[unsigned,big]},{y,0}}, + {field_flags,[unsigned,big]},{y,0}}, 20, {unassigned,{y,0}}}}] = Errors, ok. @@ -211,19 +212,19 @@ bad_catch_try(Config) when is_list(Config) -> Errors = do_val(bad_catch_try, Config), [{{bad_catch_try,bad_1,1}, {{'catch',{x,0},{f,3}}, - 5,{invalid_store,{x,0},{catchtag,[3]}}}}, + 5,{invalid_tag_register,{x,0}}}}, {{bad_catch_try,bad_2,1}, {{catch_end,{x,9}}, - 8,{source_not_y_reg,{x,9}}}}, + 8,{invalid_tag_register,{x,9}}}}, {{bad_catch_try,bad_3,1}, - {{catch_end,{y,1}},9,{bad_type,{atom,kalle}}}}, + {{catch_end,{y,1}},9,{invalid_tag,{y,1},{atom,kalle}}}}, {{bad_catch_try,bad_4,1}, - {{'try',{x,0},{f,15}},5,{invalid_store,{x,0},{trytag,[15]}}}}, + {{'try',{x,0},{f,15}},5,{invalid_tag_register,{x,0}}}}, {{bad_catch_try,bad_5,1}, - {{try_case,{y,1}},12,{bad_type,term}}}, + {{try_case,{y,1}},12,{invalid_tag,{y,1},term}}}, {{bad_catch_try,bad_6,1}, {{move,{integer,1},{y,1}},7, - {invalid_store,{y,1},{integer,1}}}}] = Errors, + {invalid_store,{y,1}}}}] = Errors, ok. cons_guard(Config) when is_list(Config) -> @@ -247,7 +248,7 @@ freg_range(Config) when is_list(Config) -> {{t,sum_3,2}, {{bif,fadd,{f,0},[{fr,0},{fr,1}],{fr,-1}}, 7, - {bad_target,{fr,-1}}}}, + {bad_register,{fr,-1}}}}, {{t,sum_4,2}, {{bif,fadd,{f,0},[{fr,0},{fr,1}],{fr,1024}}, 7, @@ -539,37 +540,37 @@ receive_stacked(Config) -> [{{receive_stacked,f1,0}, {{loop_rec_end,{f,3}}, 17, - {fragile_message_reference,{y,0}}}}, + {fragile_message_reference,{y,_}}}}, {{receive_stacked,f2,0}, - {{test_heap,3,0},10,{fragile_message_reference,{y,1}}}}, + {{test_heap,3,0},10,{fragile_message_reference,{y,_}}}}, {{receive_stacked,f3,0}, - {{test_heap,3,0},10,{fragile_message_reference,{y,1}}}}, + {{test_heap,3,0},10,{fragile_message_reference,{y,_}}}}, {{receive_stacked,f4,0}, - {{test_heap,3,0},10,{fragile_message_reference,{y,1}}}}, + {{test_heap,3,0},10,{fragile_message_reference,{y,_}}}}, {{receive_stacked,f5,0}, {{loop_rec_end,{f,23}}, 23, - {fragile_message_reference,{y,1}}}}, + {fragile_message_reference,{y,_}}}}, {{receive_stacked,f6,0}, - {{gc_bif,byte_size,{f,29},0,[{y,0}],{x,0}}, + {{gc_bif,byte_size,{f,29},0,[{y,_}],{x,0}}, 12, - {fragile_message_reference,{y,0}}}}, + {fragile_message_reference,{y,_}}}}, {{receive_stacked,f7,0}, {{loop_rec_end,{f,33}}, 20, - {fragile_message_reference,{y,0}}}}, + {fragile_message_reference,{y,_}}}}, {{receive_stacked,f8,0}, {{loop_rec_end,{f,38}}, 20, - {fragile_message_reference,{y,0}}}}, + {fragile_message_reference,{y,_}}}}, {{receive_stacked,m1,0}, {{loop_rec_end,{f,43}}, 19, - {fragile_message_reference,{y,0}}}}, + {fragile_message_reference,{y,_}}}}, {{receive_stacked,m2,0}, {{loop_rec_end,{f,48}}, 33, - {fragile_message_reference,{y,0}}}}] = Errors, + {fragile_message_reference,{y,_}}}}] = Errors, %% Compile the original source code as a smoke test. Data = proplists:get_value(data_dir, Config), @@ -579,6 +580,126 @@ receive_stacked(Config) -> ok. +aliased_types(Config) -> + Seq = lists:seq(1, 5), + 1 = aliased_types_1(Seq, Config), + + {1,1} = aliased_types_2(Seq), + {42,none} = aliased_types_2([]), + + gurka = aliased_types_3([gurka]), + gaffel = aliased_types_3([gaffel]), + + ok. + +%% ERL-735: validator failed to track types on aliased registers, rejecting +%% legitimate optimizations. +%% +%% move x0 y0 +%% bif hd L1 x0 +%% get_hd y0 %% The validator failed to see that y0 was a list +%% +aliased_types_1(Bug, Config) -> + if + Config =/= [gurka, gaffel] -> %% Pointless branch. + _ = hd(Bug), + lists:seq(1, 5), + hd(Bug) + end. + +%% ERL-832: validator failed to realize that a Y register was a cons. +aliased_types_2(Bug) -> + Res = case Bug of + [] -> id(42); + _ -> hd(Bug) + end, + {Res,case Bug of + [] -> none; + _ -> hd(Bug) + end}. + +%% ERL-832 part deux; validator failed to realize that an aliased register was +%% a cons. +aliased_types_3(Bug) -> + List = [Y || Y <- Bug], + case List of + [] -> Bug; + _ -> + if + hd(List) -> a:a(); + true -> ok + end, + hd(List) + end. + + +%% ERL-867; validation proceeded after a type conflict, causing incorrect types +%% to be joined. + +-record(r, { e1 = e1, e2 = e2 }). + +type_conflict(Config) when is_list(Config) -> + {e1, e2} = type_conflict_1(#r{}), + ok. + +type_conflict_1(C) -> + Src = id(C#r.e2), + TRes = try id(Src) of + R -> R + catch + %% C:R can never match, yet it assumed that the type of 'C' was + %% an atom from here on. + C:R -> R + end, + {C#r.e1, TRes}. + +%% ERL-886; validation failed to infer types on both sides of '=:=' + +infer_on_eq(Config) when is_list(Config) -> + {ok, gurka} = infer_on_eq_1(id({gurka})), + {ok, gaffel} = infer_on_eq_2(id({gaffel})), + {ok, elefant} = infer_on_eq_3(id({elefant})), + {ok, myra} = infer_on_eq_4(id({myra})), + ok. + +infer_on_eq_1(T) -> + 1 = erlang:tuple_size(T), + {ok, erlang:element(1, T)}. + +infer_on_eq_2(T) -> + Size = erlang:tuple_size(T), + Size = 1, + {ok, erlang:element(1, T)}. + +infer_on_eq_3(T) -> + true = 1 =:= erlang:tuple_size(T), + {ok, erlang:element(1, T)}. + +infer_on_eq_4(T) -> + true = erlang:tuple_size(T) =:= 1, + {ok, erlang:element(1, T)}. + +%% ERIERL-348; types were inferred for dead values, causing validation to fail. + +infer_dead_value(Config) when is_list(Config) -> + a = idv_1({a, b, c, d, e, f, g}, {0, 0, 0, 0, 0, 0, 0}), + b = idv_1({a, b, c, d, 0, 0, 0}, {a, b, c, d, 0, 0, 0}), + c = idv_1({0, 0, 0, 0, 0, f, g}, {0, 0, 0, 0, 0, f, g}), + error = idv_1(gurka, gaffel), + ok. + +idv_1({_A, _B, _C, _D, _E, _F, _G}, + {0, 0, 0, 0, 0, 0, 0}) -> + a; +idv_1({A, B, C, D,_E, _F, _G}=_Tuple1, + {A, B, C, D, 0, 0, 0}=_Tuple2) -> + b; +idv_1({_A, _B, _C, _D, _E, F, G}, + {0, 0, 0, 0, 0, F, G}) -> + c; +idv_1(_A, _B) -> + error. + %%%------------------------------------------------------------------------- transform_remove(Remove, Module) -> @@ -637,3 +758,6 @@ night(Turned) -> ok. participating(_, _, _, _) -> ok. + +id(I) -> + I. diff --git a/lib/compiler/test/beam_validator_SUITE_data/bad_bin_match.S b/lib/compiler/test/beam_validator_SUITE_data/bad_bin_match.S index a60ca1e89a..c7610971f1 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/bad_bin_match.S +++ b/lib/compiler/test/beam_validator_SUITE_data/bad_bin_match.S @@ -11,5 +11,5 @@ {label,1}. {func_info,{atom,t},{atom,t},1}. {label,2}. - {test,bs_start_match2,{f,1},1,[{x,0},0],{x,0}}. + {test,bs_start_match3,{f,1},1,[{x,0}],{x,0}}. return. diff --git a/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S b/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S index 481d55045d..aa344807e4 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S +++ b/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S @@ -15,8 +15,9 @@ {select_val,{x,0},{f,1},{list,[{atom,gurka},{f,3},{atom,delete},{f,4}]}}. {label,3}. {allocate_heap,2,6,2}. - %% The Y registers are not initialized here. {test,is_eq_exact,{f,5},[{x,0},{atom,ok}]}. + %% This is unreachable since {x,0} is known not to be 'ok'. We should not + %% fail with "uninitialized y registers" on erlang:exit/1 {move,{atom,nisse},{x,0}}. {call_ext,1,{extfunc,erlang,exit,1}}. {label,4}. @@ -29,6 +30,7 @@ {call_ext,2,{extfunc,io,format,2}}. {test,is_ne_exact,{f,6},[{x,0},{atom,ok}]}. {label,5}. + %% The Y registers are not initialized here. {move,{atom,logReader},{x,1}}. {move,{atom,console},{x,0}}. {call_ext,2,{extfunc,debug,filter,2}}. diff --git a/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S index cca052a9c4..a878204d16 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S +++ b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.S @@ -172,7 +172,7 @@ {allocate_zero,1,0}. {label,28}. {loop_rec,{f,30},{x,0}}. - {test,bs_start_match2,{f,29},1,[{x,0},0],{x,0}}. + {test,bs_start_match3,{f,29},1,[{x,0}],{x,0}}. {test,bs_get_integer2, {f,29}, 1, @@ -219,7 +219,7 @@ {allocate_zero,1,0}. {label,33}. {loop_rec,{f,35},{x,0}}. - {test,bs_start_match2,{f,34},1,[{x,0},0],{x,0}}. + {test,bs_start_match3,{f,34},1,[{x,0}],{x,0}}. {test,bs_get_integer2, {f,34}, 1, @@ -240,7 +240,7 @@ {y,0}}. {'%',{no_bin_opt,{binary_used_in,{test,is_binary,{f,34},[{y,0}]}}, [63,{file,"receive_stacked.erl"}]}}. - {test,is_binary,{f,34},[{y,0}]}. + {test,is_eq_exact,{f,34},[{y,0},{literal,<<0,1,2,3>>}]}. remove_message. {move,{integer,42},{x,0}}. {line,[{location,"receive_stacked.erl",64}]}. @@ -262,7 +262,7 @@ {allocate_zero,1,0}. {label,38}. {loop_rec,{f,40},{x,0}}. - {test,bs_start_match2,{f,39},1,[{x,0},0],{x,1}}. + {test,bs_start_match3,{f,39},1,[{x,0}],{x,1}}. {test,bs_get_integer2, {f,39}, 2, @@ -283,7 +283,7 @@ {y,0}}. {'%',{no_bin_opt,{[{x,1},{y,0}],{loop_rec_end,{f,38}},not_handled}, [70,{file,"receive_stacked.erl"}]}}. - {test,is_binary,{f,39},[{x,0}]}. + {test,is_eq_exact,{f,39},[{x,0},{literal,<<0,1,2,3>>}]}. remove_message. {move,{integer,42},{x,0}}. {line,[{location,"receive_stacked.erl",71}]}. diff --git a/lib/compiler/test/bif_SUITE.erl b/lib/compiler/test/bif_SUITE.erl index 42ba5d5365..423a7666af 100644 --- a/lib/compiler/test/bif_SUITE.erl +++ b/lib/compiler/test/bif_SUITE.erl @@ -23,7 +23,7 @@ -export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, init_per_group/2,end_per_group/2, - beam_validator/1,trunc_and_friends/1,cover_safe_bifs/1]). + beam_validator/1,trunc_and_friends/1,cover_safe_and_pure_bifs/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -35,7 +35,7 @@ groups() -> [{p,[parallel], [beam_validator, trunc_and_friends, - cover_safe_bifs + cover_safe_and_pure_bifs ]}]. init_per_suite(Config) -> @@ -106,7 +106,7 @@ trunc_template(Func, Bif) -> catch error:badarg -> ok end, ok."). -cover_safe_bifs(Config) -> +cover_safe_and_pure_bifs(Config) -> _ = get(), _ = get_keys(a), _ = group_leader(), @@ -118,5 +118,6 @@ cover_safe_bifs(Config) -> _ = processes(), _ = registered(), _ = term_to_binary(Config), + 42 = list_to_integer("2A", 16), ok. diff --git a/lib/compiler/test/bs_construct_SUITE.erl b/lib/compiler/test/bs_construct_SUITE.erl index ccc49df005..bccd70d6cb 100644 --- a/lib/compiler/test/bs_construct_SUITE.erl +++ b/lib/compiler/test/bs_construct_SUITE.erl @@ -27,6 +27,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, + verify_highest_opcode/1, two/1,test1/1,fail/1,float_bin/1,in_guard/1,in_catch/1, nasty_literals/1,coerce_to_float/1,side_effect/1, opt/1,otp_7556/1,float_arith/1,otp_8054/1, @@ -43,7 +44,8 @@ all() -> groups() -> [{p,[parallel], - [two,test1,fail,float_bin,in_guard,in_catch, + [verify_highest_opcode, + two,test1,fail,float_bin,in_guard,in_catch, nasty_literals,side_effect,opt,otp_7556,float_arith, otp_8054,cover]}]. @@ -68,6 +70,20 @@ init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> end_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> ok. +verify_highest_opcode(_Config) -> + case ?MODULE of + bs_construct_r21_SUITE -> + {ok,Beam} = file:read_file(code:which(?MODULE)), + case test_lib:highest_opcode(Beam) of + Highest when Highest =< 163 -> + ok; + TooHigh -> + ct:fail({too_high_opcode_for_21,TooHigh}) + end; + _ -> + ok + end. + two(Config) when is_list(Config) -> <<0,1,2,3,4,6,7,8,9>> = two_1([0], [<<1,2,3,4>>,<<6,7,8,9>>]), ok. @@ -153,6 +169,8 @@ l(I_13, I_big1, I_16, Bin) -> [0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 16#77,16#FF,16#FF,16#FF,16#FF,16#FF,16#FF,16#FF,16#FF,16#FF,16#FF, 16#FF,16#FF,16#FF,16#FF,16#FF,16#FF]), + ?T(<< (<<"abc",7:3>>):3/binary >>, + [$a,$b,$c]), %% Mix different units. ?T(<<37558955:(I_16-12)/unit:8,1:1>>, @@ -311,6 +329,9 @@ fail(Config) when is_list(Config) -> {'EXIT',{badarg,_}} = (catch <<0:(-(1 bsl 100))>>), {'EXIT',{badarg,_}} = (catch <<Bin/binary,0:(-(1 bsl 100))>>), + %% Unaligned sizes with literal binaries. + {'EXIT',{badarg,_}} = (catch <<0,(<<7777:17>>)/binary>>), + ok. float_bin(Config) when is_list(Config) -> diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index a751f6fda5..d97f49c56e 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -24,6 +24,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, + verify_highest_opcode/1, size_shadow/1,int_float/1,otp_5269/1,null_fields/1,wiger/1, bin_tail/1,save_restore/1, partitioned_bs_match/1,function_clause/1, @@ -40,8 +41,10 @@ map_and_binary/1,unsafe_branch_caching/1, bad_literals/1,good_literals/1,constant_propagation/1, parse_xml/1,get_payload/1,escape/1,num_slots_different/1, - beam_bsm/1,guard/1,is_ascii/1,non_opt_eq/1,erl_689/1, - bs_start_match2_defs/1]). + beam_bsm/1,guard/1,is_ascii/1,non_opt_eq/1, + expression_before_match/1,erl_689/1,restore_on_call/1, + restore_after_catch/1,matches_on_parameter/1,big_positions/1, + matching_meets_apply/1,bs_start_match2_defs/1]). -export([coverage_id/1,coverage_external_ignore/2]). @@ -57,8 +60,9 @@ all() -> [{group,p}]. groups() -> - [{p,[parallel], - [size_shadow,int_float,otp_5269,null_fields,wiger, + [{p,[], + [verify_highest_opcode, + size_shadow,int_float,otp_5269,null_fields,wiger, bin_tail,save_restore, partitioned_bs_match,function_clause,unit, shared_sub_bins,bin_and_float,dec_subidentifiers, @@ -73,8 +77,10 @@ groups() -> map_and_binary,unsafe_branch_caching, bad_literals,good_literals,constant_propagation,parse_xml, get_payload,escape,num_slots_different, - beam_bsm,guard,is_ascii,non_opt_eq,erl_689, - bs_start_match2_defs]}]. + beam_bsm,guard,is_ascii,non_opt_eq, + expression_before_match,erl_689,restore_on_call, + matches_on_parameter,big_positions, + matching_meets_apply,bs_start_match2_defs]}]. init_per_suite(Config) -> @@ -97,6 +103,20 @@ init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> end_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> ok. +verify_highest_opcode(_Config) -> + case ?MODULE of + bs_match_r21_SUITE -> + {ok,Beam} = file:read_file(code:which(?MODULE)), + case test_lib:highest_opcode(Beam) of + Highest when Highest =< 163 -> + ok; + TooHigh -> + ct:fail({too_high_opcode_for_21,TooHigh}) + end; + _ -> + ok + end. + size_shadow(Config) when is_list(Config) -> %% Originally OTP-5270. 7 = size_shadow_1(), @@ -250,6 +270,12 @@ bin_tail(Config) when is_list(Config) -> ok = bin_tail_e(<<2:2,1:1,1:5,42:64>>), error = bin_tail_e(<<3:2,1:1,1:5,42:64>>), error = bin_tail_e(<<>>), + + MD5 = erlang:md5(<<42>>), + <<"abc">> = bin_tail_f(<<MD5/binary,"abc">>, MD5, 3), + error = bin_tail_f(<<MD5/binary,"abc">>, MD5, 999), + {'EXIT',{_,_}} = (catch bin_tail_f(<<0:16/unit:8>>, MD5, 0)), + ok. bin_tail_c(Bin, Offset) -> @@ -306,6 +332,14 @@ bin_tail_e_var(Bin) -> <<2:2,_:1,1:5,Tail/binary>> -> Tail; _ -> error end. + +bin_tail_f(Bin, MD5, Size) -> + case Bin of + <<MD5:16/binary, Tail:Size/binary>> -> + Tail; + <<MD5:16/binary, _/binary>> -> + error + end. save_restore(Config) when is_list(Config) -> 0 = save_restore_1(<<0:2,42:6>>), @@ -457,6 +491,15 @@ unit(Config) when is_list(Config) -> 127 = peek7(<<127:7>>), 100 = peek7(<<100:7,19:7>>), fc(peek7, [<<1,2>>], catch peek7(<<1,2>>)), + + 1 = unit_opt(1, -1), + 8 = unit_opt(8, -1), + + <<1:32,"abc">> = unit_opt_2(<<1:32,"abc">>), + <<"def">> = unit_opt_2(<<2:32,"def">>), + {'EXIT',_} = (catch unit_opt_2(<<1:32,33:7>>)), + {'EXIT',_} = (catch unit_opt_2(<<2:32,55:7>>)), + ok. peek1(<<B:8,_/bitstring>>) -> B. @@ -467,6 +510,27 @@ peek8(<<B:8,_/binary>>) -> B. peek16(<<B:16,_/binary-unit:16>>) -> B. +unit_opt(U, X) -> + %% Cover type analysis in beam_ssa_type. + Bin = case U of + 1 -> <<X:7>>; + 8 -> <<X>> + end, + %% The type of Bin will be set to {binary,gcd(1, 8)}. + case Bin of + <<_/binary-unit:8>> -> 8; + <<_/binary-unit:1>> -> 1 + end. + +unit_opt_2(<<St:32,KO/binary>> = Bin0) -> + Bin = if + St =:= 1 -> + Bin0; + St =:= 2 -> + <<KO/binary>> + end, + id(Bin). + shared_sub_bins(Config) when is_list(Config) -> {15,[<<>>,<<5>>,<<4,5>>,<<3,4,5>>,<<2,3,4,5>>]} = sum(<<1,2,3,4,5>>, [], 0), ok. @@ -694,6 +758,20 @@ coverage(Config) when is_list(Config) -> binary = coverage_bitstring(<<7>>), bitstring = coverage_bitstring(<<7:4>>), other = coverage_bitstring([a]), + + %% Cover code in beam_trim. + + {done,<<17,53>>,[253,155,200]} = + coverage_trim(<<253,155,200,17,53>>, e0, e1, e2, e3, []), + + <<"(right|linux)">> = coverage_trim_1(<<"">>, <<"right">>, <<"linux">>), + <<"/(right|linux)">> = coverage_trim_1(<<"/">>, <<"right">>, <<"linux">>), + <<"(left|linux)/(right|linux)">> = + coverage_trim_1(<<"left">>, <<"right">>, <<"linux">>), + + {10,<<"-">>,""} = coverage_trim_2(<<"-">>, 10, []), + {8,<<"-">>,"aa"} = coverage_trim_2(<<"aa-">>, 10, []), + ok. coverage_fold(Fun, Acc, <<H,T/binary>>) -> @@ -788,6 +866,37 @@ coverage_bitstring(Bin) when is_binary(Bin) -> binary; coverage_bitstring(<<_/bitstring>>) -> bitstring; coverage_bitstring(_) -> other. +coverage_trim(<<C:8,T/binary>> = Bin, E0, E1, E2, E3, Acc) -> + case id(C > 128) of + true -> + coverage_trim(T, E0, E1, E2, E3, [C|Acc]); + false -> + {done,Bin,lists:reverse(Acc)} + end. + +coverage_trim_1(<<>>, Right, OsType) -> + do_coverage_trim_1(Right, OsType); +coverage_trim_1(<<"/">>, Right, OsType) -> + <<"/",(do_coverage_trim_1(Right, OsType))/binary>>; +coverage_trim_1(Left, Right, OsType) -> + <<(do_coverage_trim_1(Left, OsType))/binary, + "/", + (do_coverage_trim_1(Right, OsType))/binary>>. + +do_coverage_trim_1(A, OsType) -> + <<"(",A/binary,"|",OsType/binary,")">>. + +coverage_trim_2(<<C/utf8,R/binary>> = Bin, I, L) -> + case printable_char(C) of + true -> + coverage_trim_2(R, I - 1, [C | L]); + false -> + {I,Bin,lists:reverse(L)} + end. + +printable_char($a) -> true; +printable_char(_) -> false. + multiple_uses(Config) when is_list(Config) -> {344,62879,345,<<245,159,1,89>>} = multiple_uses_1(<<1,88,245,159,1,89>>), true = multiple_uses_2(<<0,0,197,18>>), @@ -1754,11 +1863,10 @@ do_erl_689_2b(_, <<Length, Data/binary>>) -> %% ERL-753 bs_start_match2_defs(_Config) -> - {<<"http://127.0.0.1:1234/vsaas/hello">>} = api_url(<<"hello">>, dummy), - {"https://127.0.0.1:4321/vsaas/hello"} = api_url({https, "hello"}, dummy). + {<<"http://127.0.0.1:1234/vsaas/hello">>} = api_url(<<"hello">>), + {"https://127.0.0.1:4321/vsaas/hello"} = api_url({https, "hello"}). -api_url(URL, Auth) -> - Header = [], +api_url(URL) -> case URL of <<_/binary>> -> {<<"http://127.0.0.1:1234/vsaas/",URL/binary>>}; {https, [_|_] = URL1} -> {"https://127.0.0.1:4321/vsaas/"++URL1} @@ -1767,4 +1875,134 @@ api_url(URL, Auth) -> check(F, R) -> R = F(). +%% Make sure that an expression that comes between function start and a match +%% expression passes validation. +expression_before_match(Config) when is_list(Config) -> + <<_,R/binary>> = id(<<0,1,2,3>>), + {1, <<2,3>>} = expression_before_match_1(R), + ok. + +expression_before_match_1(R) -> + A = id(1), + case R of + <<1,Bar/binary>> -> {A, Bar}; + <<>> -> {A, baz} + end. + +%% Make sure that context positions are updated on calls. +restore_on_call(Config) when is_list(Config) -> + ok = restore_on_call_plain(<<0, 1, 2>>), + <<"x">> = restore_on_call_match(<<0, "x">>), + ok. + +restore_on_call_plain(<<0, Rest/binary>>) -> + <<2>> = restore_on_call_plain_1(Rest), + %% {badmatch, <<>>} on missing restore. + <<2>> = restore_on_call_plain_1(Rest), + ok. + +restore_on_call_plain_1(<<1, Rest/binary>>) -> Rest; +restore_on_call_plain_1(Other) -> Other. + +%% Calls a function that moves the match context passed to it, and then matches +%% on its result to confuse the reposition algorithm's success/fail logic. +restore_on_call_match(<<0, Bin/binary>>) -> + case skip_until_zero(Bin) of + {skipped, Rest} -> + Rest; + not_found -> + %% The match context did not get repositioned before the + %% bs_get_tail instruction here. + Bin + end. + +skip_until_zero(<<0,Rest/binary>>) -> + {skipped, Rest}; +skip_until_zero(<<_C,Rest/binary>>) -> + skip_until_zero(Rest); +skip_until_zero(_) -> + not_found. + +%% 'catch' must invalidate positions. +restore_after_catch(Config) when is_list(Config) -> + <<0, 1>> = restore_after_catch_1(<<0, 1>>), + ok. + +restore_after_catch_1(<<A/binary>>) -> + try throw_after_byte(A) of + _ -> impossible + catch + throw:_Any -> + %% Will equal <<1>> if the bug is present. + A + end. + +throw_after_byte(<<_,_/binary>>) -> + throw(away). + +matches_on_parameter(Config) when is_list(Config) -> + %% This improves coverage for matching on "naked" parameters. + {<<"urka">>, <<"a">>} = matches_on_parameter_1(<<"gurka">>), + ok = (catch matches_on_parameter_2(<<"10001110101">>, 0)). + +matches_on_parameter_1(Bin) -> + <<"g", A/binary>> = Bin, + <<_,_,"rk", B/binary>> = Bin, + {A, B}. + +matches_on_parameter_2(Bin, Offset) -> + <<_:Offset, Bit:1, Rest/bits>> = Bin, + case bit_size(Rest) of + 0 -> throw(ok); + _ -> [Bit | matches_on_parameter_2(Bin, Offset + 1)] + end. + +big_positions(Config) when is_list(Config) -> + %% This provides coverage for when match context positions no longer fit + %% into an immediate on 32-bit platforms. + + A = <<0:((1 bsl 27) - 8), $A, 1:1, "gurka", $A>>, + B = <<0:((1 bsl 27) - 8), $B, "hello", $B>>, + + {a,$A} = bp_start_match(A), + {b,$B} = bp_start_match(B), + {a,$A} = bp_getpos(A), + {b,$B} = bp_getpos(B), + + ok. + +%% After the first iteration the context's position will no longer fit into an +%% immediate. To improve performance the bs_start_match3 instruction will +%% return a new context with an updated base position so that we won't have to +%% resort to using bigints. +bp_start_match(<<_:(1 bsl 27),T/bits>>) -> bp_start_match(T); +bp_start_match(<<1:1,"gurka",A>>) -> {a,A}; +bp_start_match(<<"hello",B>>) -> {b,B}. + +%% This is a corner case where the above didn't work perfectly; if the position +%% was _just_ small enough to fit into an immediate when bs_start_match3 was +%% hit, but too large at bs_get_position, then it must be saved as a bigint. +bp_getpos(<<_:((1 bsl 27) - 8),T/bits>>) -> bp_getpos(T); +bp_getpos(<<A,1:1,"gurka",A>>) -> {a,A}; +bp_getpos(<<B,"hello",B>>) -> {b,B}. + +matching_meets_apply(_Config) -> + <<"abc">> = do_matching_meets_apply(<<"/abc">>, []), + 42 = do_matching_meets_apply(<<"">>, {erlang,-42}), + 100 = do_matching_meets_apply(no_binary, {erlang,-100}), + ok. + +do_matching_meets_apply(<<$/, Rest/binary>>, _Handler) -> + id(Rest); +do_matching_meets_apply(<<_/binary>>=Name, never_matches_a) -> + %% Used to crash the compiler because variables in a remote + %% were not handled properly by beam_ssa_bsm. + Name:foo(gurka); +do_matching_meets_apply(<<_/binary>>=Name, never_matches_b) -> + %% Another case of the above. + foo:Name(gurka); +do_matching_meets_apply(_Bin, {Handler, State}) -> + %% Another case of the above. + Handler:abs(State). + id(I) -> I. diff --git a/lib/compiler/test/compilation_SUITE.erl b/lib/compiler/test/compilation_SUITE.erl index 139f7af0d4..74f9dbd9b4 100644 --- a/lib/compiler/test/compilation_SUITE.erl +++ b/lib/compiler/test/compilation_SUITE.erl @@ -170,7 +170,7 @@ try_it(Module, Conf) -> atom_to_list(Module)), Out = proplists:get_value(priv_dir,Conf), io:format("Compiling: ~s\n", [Src]), - CompRc0 = compile:file(Src, [clint0,clint,{outdir,Out},report, + CompRc0 = compile:file(Src, [clint0,clint,ssalint,{outdir,Out},report, bin_opt_info|OtherOpts]), io:format("Result: ~p\n",[CompRc0]), {ok,_Mod} = CompRc0, @@ -189,7 +189,7 @@ try_it(Module, Conf) -> ct:timetrap(Timetrap), io:format("Compiling (with old inliner): ~s\n", [Src]), - CompRc2 = compile:file(Src, [clint, + CompRc2 = compile:file(Src, [clint,ssalint, {outdir,Out},report,bin_opt_info, {inline,1000}|OtherOpts]), io:format("Result: ~p\n",[CompRc2]), @@ -355,7 +355,7 @@ compile_compiler(Files, OutDir, Version, InlineOpts) -> io:format("~ts", [code:which(compile)]), io:format("Compiling ~s into ~ts", [Version,OutDir]), Opts = [report, - clint0,clint, + clint0,clint,ssalint, bin_opt_info, {outdir,OutDir}, {d,'COMPILER_VSN',"\""++Version++"\""}, diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index 6b230710b3..53627b9d81 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -28,12 +28,12 @@ init_per_group/2,end_per_group/2, app_test/1,appup_test/1, debug_info/4, custom_debug_info/1, custom_compile_info/1, - file_1/1, forms_2/1, module_mismatch/1, big_file/1, outdir/1, + file_1/1, forms_2/1, module_mismatch/1, outdir/1, binary/1, makedep/1, cond_and_ifdef/1, listings/1, listings_big/1, other_output/1, kernel_listing/1, encrypted_abstr/1, strict_record/1, utf8_atoms/1, utf8_functions/1, extra_chunks/1, cover/1, env/1, core_pp/1, tuple_calls/1, - core_roundtrip/1, asm/1, optimized_guards/1, + core_roundtrip/1, asm/1, sys_pre_attributes/1, dialyzer/1, warnings/1, pre_load_check/1, env_compiler_options/1, bc_options/1, deterministic_include/1, deterministic_paths/1 @@ -46,11 +46,11 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. -spec all() -> all_return_type(). all() -> - [app_test, appup_test, file_1, forms_2, module_mismatch, big_file, outdir, + [app_test, appup_test, file_1, forms_2, module_mismatch, outdir, binary, makedep, cond_and_ifdef, listings, listings_big, other_output, kernel_listing, encrypted_abstr, tuple_calls, strict_record, utf8_atoms, utf8_functions, extra_chunks, - cover, env, core_pp, core_roundtrip, asm, optimized_guards, + cover, env, core_pp, core_roundtrip, asm, sys_pre_attributes, dialyzer, warnings, pre_load_check, env_compiler_options, custom_debug_info, bc_options, custom_compile_info, deterministic_include, deterministic_paths]. @@ -104,6 +104,7 @@ file_1(Config) when is_list(Config) -> compile_and_verify(Simple, Target, []), compile_and_verify(Simple, Target, [native]), compile_and_verify(Simple, Target, [debug_info]), + compile_and_verify(Simple, Target, [no_postopt]), {ok,simple} = compile:file(Simple, [no_line_info]), %Coverage {ok,simple} = compile:file(Simple, [{eprof,beam_z}]), %Coverage @@ -231,17 +232,6 @@ module_mismatch(Config) when is_list(Config) -> ok. -big_file(Config) when is_list(Config) -> - {Big,Target} = get_files(Config, big, "big_file"), - ok = file:set_cwd(filename:dirname(Target)), - compile_and_verify(Big, Target, []), - compile_and_verify(Big, Target, [debug_info]), - compile_and_verify(Big, Target, [no_postopt]), - - %% Cleanup. - ok = file:delete(Target), - ok. - %% Tests that the {outdir, Dir} option works. outdir(Config) when is_list(Config) -> @@ -370,41 +360,37 @@ do_file_listings(DataDir, PrivDir, [File|Files]) -> TargetDir = filename:join(PrivDir, listings), ok = file:make_dir(TargetDir), - %% Test all dedicated listing options. - do_listing(Simple, TargetDir, 'S'), - do_listing(Simple, TargetDir, 'E'), - do_listing(Simple, TargetDir, 'P'), - do_listing(Simple, TargetDir, dpp, ".pp"), - do_listing(Simple, TargetDir, dabstr, ".abstr"), - do_listing(Simple, TargetDir, dexp, ".expand"), - do_listing(Simple, TargetDir, dcore, ".core"), - do_listing(Simple, TargetDir, doldinline, ".oldinline"), - do_listing(Simple, TargetDir, dinline, ".inline"), - do_listing(Simple, TargetDir, dcore, ".core"), - do_listing(Simple, TargetDir, dcopt, ".copt"), - do_listing(Simple, TargetDir, dcbsm, ".core_bsm"), - do_listing(Simple, TargetDir, dsetel, ".dsetel"), - do_listing(Simple, TargetDir, dkern, ".kernel"), - do_listing(Simple, TargetDir, dcg, ".codegen"), - do_listing(Simple, TargetDir, dblk, ".block"), - do_listing(Simple, TargetDir, dexcept, ".except"), - do_listing(Simple, TargetDir, dbs, ".bs"), - do_listing(Simple, TargetDir, dtype, ".type"), - do_listing(Simple, TargetDir, ddead, ".dead"), - do_listing(Simple, TargetDir, djmp, ".jump"), - do_listing(Simple, TargetDir, dclean, ".clean"), - do_listing(Simple, TargetDir, dpeep, ".peep"), - do_listing(Simple, TargetDir, dopt, ".optimize"), - - %% First clean up. - Listings = filename:join(PrivDir, listings), - lists:foreach(fun(F) -> ok = file:delete(F) end, - filelib:wildcard(filename:join(Listings, "*"))), + List = [{'S',".S"}, + {'E',".E"}, + {'P',".P"}, + {dpp, ".pp"}, + {dabstr, ".abstr"}, + {dexp, ".expand"}, + {dcore, ".core"}, + {doldinline, ".oldinline"}, + {dinline, ".inline"}, + {dcore, ".core"}, + {dcopt, ".copt"}, + {dcbsm, ".core_bsm"}, + {dkern, ".kernel"}, + {dssa, ".ssa"}, + {dssaopt, ".ssaopt"}, + {dprecg, ".precodegen"}, + {dcg, ".codegen"}, + {dblk, ".block"}, + {dexcept, ".except"}, + {djmp, ".jump"}, + {dclean, ".clean"}, + {dpeep, ".peep"}, + {dopt, ".optimize"}, + {diffable, ".S"}], + p_listings(List, Simple, TargetDir), %% Test options that produce a listing file if 'binary' is not given. do_listing(Simple, TargetDir, to_pp, ".P"), do_listing(Simple, TargetDir, to_exp, ".E"), do_listing(Simple, TargetDir, to_core0, ".core"), + Listings = filename:join(PrivDir, listings), ok = file:delete(filename:join(Listings, File ++ ".core")), do_listing(Simple, TargetDir, to_core, ".core"), do_listing(Simple, TargetDir, to_kernel, ".kernel"), @@ -420,21 +406,35 @@ do_file_listings(DataDir, PrivDir, [File|Files]) -> listings_big(Config) when is_list(Config) -> {Big,Target} = get_files(Config, big, listings_big), TargetDir = filename:dirname(Target), - do_listing(Big, TargetDir, 'S'), - do_listing(Big, TargetDir, 'E'), - do_listing(Big, TargetDir, 'P'), - do_listing(Big, TargetDir, dkern, ".kernel"), - do_listing(Big, TargetDir, to_dis, ".dis"), - - TargetNoext = filename:rootname(Target, code:objfile_extension()), - {ok,big} = compile:file(TargetNoext, [from_asm,{outdir,TargetDir}]), - - %% Cleanup. - ok = file:delete(Target), - lists:foreach(fun(F) -> ok = file:delete(F) end, - filelib:wildcard(filename:join(TargetDir, "*"))), - ok = file:del_dir(TargetDir), - ok. + List = [{'S',".S"}, + {'E',".E"}, + {'P',".P"}, + {dkern, ".kernel"}, + {dssa, ".ssa"}, + {dssaopt, ".ssaopt"}, + {dprecg, ".precodegen"}, + {to_dis, ".dis"}], + p_listings(List, Big, TargetDir). + +p_listings(List, File, BaseDir) -> + Run = fun({Option,Extension}) -> + Uniq = erlang:unique_integer([positive]), + Dir = filename:join(BaseDir, integer_to_list(Uniq)), + ok = file:make_dir(Dir), + try + do_listing(File, Dir, Option, Extension), + ok + catch + Class:Error:Stk -> + io:format("~p:~p\n~p\n", [Class,Error,Stk]), + error + after + _ = [ok = file:delete(F) || + F <- filelib:wildcard(filename:join(Dir, "*"))], + ok = file:del_dir(Dir) + end + end, + test_lib:p_run(Run, List). other_output(Config) when is_list(Config) -> {Simple,_Target} = get_files(Config, simple, "other_output"), @@ -681,9 +681,6 @@ cover(Config) when is_list(Config) -> io:format("~p\n", [compile:options()]), ok. -do_listing(Source, TargetDir, Type) -> - do_listing(Source, TargetDir, Type, "." ++ atom_to_list(Type)). - do_listing(Source, TargetDir, Type, Ext) -> io:format("Source: ~p TargetDir: ~p\n Type: ~p Ext: ~p\n", [Source, TargetDir, Type, Ext]), @@ -920,7 +917,7 @@ do_core_pp_1(M, A, Outdir) -> ok = file:delete(CoreFile), %% Compile as usual (including optimizations). - compile_forms(M, Core, [clint,from_core,binary]), + compile_forms(M, Core, [clint,ssalint,from_core,binary]), %% Don't optimize to test that we are not dependent %% on the Core Erlang optmimization passes. @@ -929,7 +926,7 @@ do_core_pp_1(M, A, Outdir) -> %% records; if sys_core_fold was run it would fix %% that; if sys_core_fold was not run v3_kernel would %% crash.) - compile_forms(M, Core, [clint,from_core,no_copt,binary]), + compile_forms(M, Core, [clint,ssalint,from_core,no_copt,binary]), ok. @@ -1170,95 +1167,6 @@ do_asm(Beam, Outdir) -> error end. -%% Make sure that guards are fully optimized. Guards should -%% should use 'test' instructions, not 'bif' instructions. - -optimized_guards(_Config) -> - TestBeams = get_unique_beam_files(), - test_lib:p_run(fun(F) -> do_opt_guards(F) end, TestBeams). - -do_opt_guards(Beam) -> - {ok,{M,[{abstract_code,{raw_abstract_v1,A}}]}} = - beam_lib:chunks(Beam, [abstract_code]), - try - {ok,M,Asm} = compile:forms(A, ['S']), - do_opt_guards_mod(Asm) - catch Class:Error:Stk -> - io:format("~p: ~p ~p\n~p\n", [M,Class,Error,Stk]), - error - end. - -do_opt_guards_mod({Mod,_Exp,_Attr,Asm,_NumLabels}) -> - case do_opt_guards_fs(Mod, Asm) of - [] -> - ok; - [_|_]=Bifs -> - io:format("ERRORS FOR ~p:\n~p\n", [Mod,Bifs]), - error - end. - -do_opt_guards_fs(Mod, [{function,Name,Arity,_,Is}|Fs]) -> - Bifs0 = do_opt_guards_fun(Is), - - %% The compiler does not attempt to optimize 'xor'. - %% Therefore, ignore all functions that use 'xor' in - %% a guard. - Bifs = case lists:any(fun({bif,'xor',_,_,_}) -> true; - (_) -> false - end, Bifs0) of - true -> []; - false -> Bifs0 - end, - - %% Filter out the allowed exceptions. - FA = {Name,Arity}, - case {Bifs,is_exception(Mod, FA)} of - {[_|_],true} -> - io:format("~p:~p/~p IGNORED:\n~p\n", - [Mod,Name,Arity,Bifs]), - do_opt_guards_fs(Mod, Fs); - {[_|_],false} -> - [{FA,Bifs}|do_opt_guards_fs(Mod, Fs)]; - {[],false} -> - do_opt_guards_fs(Mod, Fs); - {[],true} -> - io:format("Redundant exception for ~p:~p/~p\n", - [Mod,Name,Arity]), - error(redundant) - end; -do_opt_guards_fs(_, []) -> []. - -do_opt_guards_fun([{bif,Name,{f,F},As,_}=I|Is]) when F =/= 0 -> - Arity = length(As), - case erl_internal:comp_op(Name, Arity) orelse - erl_internal:bool_op(Name, Arity) orelse - erl_internal:new_type_test(Name, Arity) of - true -> - [I|do_opt_guards_fun(Is)]; - false -> - do_opt_guards_fun(Is) - end; -do_opt_guards_fun([_|Is]) -> - do_opt_guards_fun(Is); -do_opt_guards_fun([]) -> []. - -is_exception(bs_match_SUITE, {matching_and_andalso_2,2}) -> true; -is_exception(bs_match_SUITE, {matching_and_andalso_3,2}) -> true; -is_exception(guard_SUITE, {'-complex_not/1-fun-4-',1}) -> true; -is_exception(guard_SUITE, {'-complex_not/1-fun-5-',1}) -> true; -is_exception(guard_SUITE, {basic_andalso_orelse,1}) -> true; -is_exception(guard_SUITE, {bad_guards,1}) -> true; -is_exception(guard_SUITE, {bad_guards_2,2}) -> true; -is_exception(guard_SUITE, {bad_guards_3,2}) -> true; -is_exception(guard_SUITE, {cqlc,4}) -> true; -is_exception(guard_SUITE, {csemi7,3}) -> true; -is_exception(guard_SUITE, {misc,1}) -> true; -is_exception(guard_SUITE, {nested_not_2b,4}) -> true; -is_exception(guard_SUITE, {tricky_1,2}) -> true; -is_exception(map_SUITE, {map_guard_update,2}) -> true; -is_exception(map_SUITE, {map_guard_update_variables,3}) -> true; -is_exception(_, _) -> false. - sys_pre_attributes(Config) -> DataDir = proplists:get_value(data_dir, Config), File = filename:join(DataDir, "attributes.erl"), @@ -1475,44 +1383,55 @@ env_compiler_options(_Config) -> bc_options(Config) -> DataDir = proplists:get_value(data_dir, Config), - 101 = highest_opcode(DataDir, small_float, [no_get_hd_tl,no_line_info]), - - 103 = highest_opcode(DataDir, big, - [no_get_hd_tl,no_record_opt, - no_line_info,no_stack_trimming]), - - 125 = highest_opcode(DataDir, small_float, - [no_get_hd_tl,no_line_info,no_float_opt]), - - 132 = highest_opcode(DataDir, small, - [no_get_hd_tl,no_record_opt,no_float_opt,no_line_info]), - - 136 = highest_opcode(DataDir, big, [no_get_hd_tl,no_record_opt,no_line_info]), - - 153 = highest_opcode(DataDir, big, [no_get_hd_tl,no_record_opt]), - 153 = highest_opcode(DataDir, big, [r16]), - 153 = highest_opcode(DataDir, big, [r17]), - 153 = highest_opcode(DataDir, big, [r18]), - 153 = highest_opcode(DataDir, big, [r19]), - 153 = highest_opcode(DataDir, small_float, [r16]), - 153 = highest_opcode(DataDir, small_float, []), - - 158 = highest_opcode(DataDir, small_maps, [r17]), - 158 = highest_opcode(DataDir, small_maps, [r18]), - 158 = highest_opcode(DataDir, small_maps, [r19]), - 158 = highest_opcode(DataDir, small_maps, [r20]), - 158 = highest_opcode(DataDir, small_maps, []), - - 163 = highest_opcode(DataDir, big, []), - + L = [{101, small_float, [no_get_hd_tl,no_line_info]}, + {103, big, [no_put_tuple2,no_get_hd_tl,no_ssa_opt_record, + no_line_info,no_stack_trimming]}, + {125, small_float, [no_get_hd_tl,no_line_info,no_ssa_opt_float]}, + + {132, small, [no_put_tuple2,no_get_hd_tl,no_ssa_opt_record, + no_ssa_opt_float,no_line_info,no_bsm3]}, + + {153, small, [r20]}, + {153, small, [r21]}, + + {136, big, [no_put_tuple2,no_get_hd_tl, + no_ssa_opt_record,no_line_info]}, + + {153, big, [no_put_tuple2,no_get_hd_tl, no_ssa_opt_record]}, + {153, big, [r16]}, + {153, big, [r17]}, + {153, big, [r18]}, + {153, big, [r19]}, + {153, small_float, [r16]}, + {153, small_float, []}, + + {158, small_maps, [r17]}, + {158, small_maps, [r18]}, + {158, small_maps, [r19]}, + {158, small_maps, [r20]}, + {158, small_maps, [r21]}, + + {164, small_maps, []}, + {164, big, []} + ], + + Test = fun({Expected,Mod,Options}) -> + case highest_opcode(DataDir, Mod, Options) of + Expected -> + ok; + Got -> + io:format("*** module ~p, options ~p => got ~p; expected ~p\n", + [Mod,Options,Got,Expected]), + error + end + end, + test_lib:p_run(Test, L), ok. highest_opcode(DataDir, Mod, Opt) -> Src = filename:join(DataDir, atom_to_list(Mod)++".erl"), {ok,Mod,Beam} = compile:file(Src, [binary|Opt]), - {ok,{Mod,[{"Code",Code}]}} = beam_lib:chunks(Beam, ["Code"]), - <<16:32,0:32,HighestOpcode:32,_/binary>> = Code, - HighestOpcode. + test_lib:highest_opcode(Beam). deterministic_include(Config) when is_list(Config) -> DataDir = proplists:get_value(data_dir, Config), diff --git a/lib/compiler/test/compiler.cover b/lib/compiler/test/compiler.cover index 3fd7fc1937..fac0f9947c 100644 --- a/lib/compiler/test/compiler.cover +++ b/lib/compiler/test/compiler.cover @@ -1,5 +1,4 @@ -{incl_app,compiler,details}. - %% -*- erlang -*- +{local_only,compiler,true}. +{incl_app,compiler,details}. {excl_mods,compiler,[core_scan,core_parse]}. - diff --git a/lib/compiler/test/core_fold_SUITE.erl b/lib/compiler/test/core_fold_SUITE.erl index 47606014c3..adfebd5158 100644 --- a/lib/compiler/test/core_fold_SUITE.erl +++ b/lib/compiler/test/core_fold_SUITE.erl @@ -212,9 +212,14 @@ bifs(Config) when is_list(Config) -> {ok,#{K:=V}} = id(list_to_tuple([ok,#{K=>V}])), ok. --define(CMP_SAME(A0, B), (fun(A) -> true = A == B, false = A /= B end)(id(A0))). --define(CMP_DIFF(A0, B), (fun(A) -> false = A == B, true = A /= B end)(id(A0))). - +-define(CMP_SAME0(A0, B), (fun(A) -> true = A == B, false = A /= B end)(id(A0))). +-define(CMP_SAME1(A0, B), (fun(A) -> false = A /= B, true = A == B end)(id(A0))). +-define(CMP_SAME(A0, B), (true = ?CMP_SAME0(A0, B) =:= not ?CMP_SAME1(A0, B))). + +-define(CMP_DIFF0(A0, B), (fun(A) -> false = A == B, true = A /= B end)(id(A0))). +-define(CMP_DIFF1(A0, B), (fun(A) -> true = A /= B, false = A == B end)(id(A0))). +-define(CMP_DIFF(A0, B), (true = ?CMP_DIFF0(A0, B) =:= not ?CMP_DIFF1(A0, B))). + eq(Config) when is_list(Config) -> ?CMP_SAME([a,b,c], [a,b,c]), ?CMP_SAME([42.0], [42.0]), @@ -278,6 +283,8 @@ coverage(Config) when is_list(Config) -> a = cover_remove_non_vars_alias({a,b,c}), error = cover_will_match_lit_list(), {ok,[a]} = cover_is_safe_bool_expr(a), + false = cover_is_safe_bool_expr2(a), + ok = cover_eval_is_function(fun id/1), ok = cover_opt_guard_try(#cover_opt_guard_try{list=[a]}), error = cover_opt_guard_try(#cover_opt_guard_try{list=[]}), @@ -341,6 +348,15 @@ cover_is_safe_bool_expr(X) -> false end. +cover_is_safe_bool_expr2(X) -> + try + V = [X], + is_function(V, 1) + catch + _:_ -> + false + end. + cover_opt_guard_try(Msg) -> if length(Msg#cover_opt_guard_try.list) =/= 1 -> @@ -349,6 +365,12 @@ cover_opt_guard_try(Msg) -> ok end. +cover_eval_is_function(X) -> + case X of + {a,_} -> is_function(X); + _ -> ok + end. + bsm_an_inlined(<<_:8>>, _) -> ok; bsm_an_inlined(_, _) -> error. @@ -356,7 +378,7 @@ unused_multiple_values_error(Config) when is_list(Config) -> PrivDir = proplists:get_value(priv_dir, Config), Dir = test_lib:get_data_dir(Config), Core = filename:join(Dir, "unused_multiple_values_error"), - Opts = [no_copt,clint,return,from_core,{outdir,PrivDir} + Opts = [no_copt,clint,ssalint,return,from_core,{outdir,PrivDir} |test_lib:opt_opts(?MODULE)], {error,[{unused_multiple_values_error, [{none,core_lint,{return_mismatch,{hello,1}}}]}], @@ -480,7 +502,7 @@ source(true, Activities) -> Activities end. -tim(#{reduction := Emergency}) -> +tim(#{reduction := _Emergency}) -> try fun() -> surgery end catch diff --git a/lib/compiler/test/float_SUITE.erl b/lib/compiler/test/float_SUITE.erl index 012810aba2..0fa8070dc8 100644 --- a/lib/compiler/test/float_SUITE.erl +++ b/lib/compiler/test/float_SUITE.erl @@ -20,15 +20,17 @@ -module(float_SUITE). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, - pending/1,bif_calls/1,math_functions/1,mixed_float_and_int/1]). + pending/1,bif_calls/1,math_functions/1,mixed_float_and_int/1, + subtract_number_type/1,float_followed_by_guard/1]). -include_lib("common_test/include/ct.hrl"). suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> +all() -> [pending, bif_calls, math_functions, - mixed_float_and_int]. + mixed_float_and_int, subtract_number_type, + float_followed_by_guard]. groups() -> []. @@ -176,5 +178,31 @@ mixed_float_and_int(Config) when is_list(Config) -> pc(Cov, NotCov, X) -> round(Cov/(Cov+NotCov)*100) + 42 + 2.0*X. +subtract_number_type(Config) when is_list(Config) -> + 120 = fact(5). + +fact(N) -> + fact(N, 1). + +fact(0, P) -> P; +fact(1, P) -> P; +fact(N, P) -> fact(N-1, P*N). + +float_followed_by_guard(Config) when is_list(Config) -> + true = ffbg_1(5, 1), + false = ffbg_1(1, 5), + ok. + +ffbg_1(A, B0) -> + %% This is a non-guard block followed by a *guard block* that starts with a + %% floating point operation, and the compiler erroneously assumed that it + %% was safe to skip fcheckerror because the next block started with a float + %% op. + B = id(B0) / 1.0, + if + A - B > 0.0 -> true; + A - B =< 0.0 -> false + end. + id(I) -> I. diff --git a/lib/compiler/test/fun_SUITE.erl b/lib/compiler/test/fun_SUITE.erl index e00885fcd6..1df0a05275 100644 --- a/lib/compiler/test/fun_SUITE.erl +++ b/lib/compiler/test/fun_SUITE.erl @@ -249,6 +249,13 @@ badfun(_Config) -> expect_badfun(X, catch X(put(?FUNCTION_NAME, of_course))), of_course = erase(?FUNCTION_NAME), + %% A literal as a Fun used to crash the code generator. This only happened + %% when type optimization had reduced `Fun` to a literal, hence the match. + Literal = fun(literal = Fun) -> + Fun() + end, + expect_badfun(literal, catch Literal(literal)), + ok. expect_badfun(Term, Exit) -> diff --git a/lib/compiler/test/guard_SUITE.erl b/lib/compiler/test/guard_SUITE.erl index 6ad73b46f7..ed0a56f064 100644 --- a/lib/compiler/test/guard_SUITE.erl +++ b/lib/compiler/test/guard_SUITE.erl @@ -35,8 +35,7 @@ basic_andalso_orelse/1,traverse_dcd/1, check_qlc_hrl/1,andalso_semi/1,t_tuple_size/1,binary_part/1, bad_constants/1,bad_guards/1, - guard_in_catch/1,beam_bool_SUITE/1, - cover_beam_dead/1]). + guard_in_catch/1,beam_bool_SUITE/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -54,8 +53,7 @@ groups() -> rel_ops,rel_op_combinations, literal_type_tests,basic_andalso_orelse,traverse_dcd, check_qlc_hrl,andalso_semi,t_tuple_size,binary_part, - bad_constants,bad_guards,guard_in_catch,beam_bool_SUITE, - cover_beam_dead]}]. + bad_constants,bad_guards,guard_in_catch,beam_bool_SUITE]}]. init_per_suite(Config) -> test_lib:recompile(?MODULE), @@ -1297,6 +1295,32 @@ rel_ops(Config) when is_list(Config) -> Empty = id([]), ?T(==, [], Empty), + %% Cover beam_ssa_dead:turn_op('/='). + ok = (fun(A, B) when is_atom(A) -> + X = id(A /= B), + if + X -> ok; + true -> error + end + end)(a, b), + ok = (fun(A, B) when is_atom(A) -> + X = id(B /= A), + if + X -> ok; + true -> error + end + end)(a, b), + + %% Cover beam_ssa_dead. + Arrow = fun([T1,T2]) when T1 == $>, T2 == $>; + T1 == $<, T2 == $| -> true; + (_) -> false + end, + true = Arrow(">>"), + true = Arrow("<|"), + false = Arrow("><"), + false = Arrow(""), + ok. -undef(TestOp). @@ -1330,6 +1354,9 @@ rel_op_combinations_1(N, Digits) -> Bool = is_digit_6(N), Bool = is_digit_7(N), Bool = is_digit_8(N), + Bool = is_digit_9(42, N), + Bool = is_digit_10(N, 0), + Bool = is_digit_11(N, 0), rel_op_combinations_1(N-1, Digits). is_digit_1(X) when 16#0660 =< X, X =< 16#0669 -> true; @@ -1373,6 +1400,24 @@ is_digit_8(X) when X =< 16#0669, X > (16#0660-1) -> true; is_digit_8(16#0670) -> false; is_digit_8(_) -> false. +is_digit_9(A, 0) when A =:= 42 -> false; +is_digit_9(_, X) when X > 16#065F, X < 16#066A -> true; +is_digit_9(_, X) when 16#0030 =< X, X =< 16#0039 -> true; +is_digit_9(_, X) when 16#06F0 =< X, X =< 16#06F9 -> true; +is_digit_9(_, _) -> false. + +is_digit_10(0, 0) -> false; +is_digit_10(X, _) when X < 16#066A, 16#0660 =< X -> true; +is_digit_10(X, _) when 16#0030 =< X, X =< 16#0039 -> true; +is_digit_10(X, _) when 16#06F0 =< X, X =< 16#06F9 -> true; +is_digit_10(_, _) -> false. + +is_digit_11(0, 0) -> false; +is_digit_11(X, _) when X =< 16#0669, 16#0660 =< X -> true; +is_digit_11(X, _) when 16#0030 =< X, X =< 16#0039 -> true; +is_digit_11(X, _) when 16#06F0 =< X, X =< 16#06F9 -> true; +is_digit_11(_, _) -> false. + rel_op_combinations_2(0, _) -> ok; rel_op_combinations_2(N, Range) -> @@ -1473,6 +1518,7 @@ rel_op_combinations_3(N, Red) -> Val = redundant_9(N), Val = redundant_10(N), Val = redundant_11(N), + Val = redundant_11(N), rel_op_combinations_3(N-1, Red). redundant_1(X) when X >= 51, X =< 80 -> 5*X; @@ -1527,6 +1573,10 @@ redundant_11(X) when X =:= 10 -> 2*X; redundant_11(X) when X >= 51, X =< 80 -> 5*X; redundant_11(_) -> none. +redundant_12(X) when X >= 50, X =< 80 -> 2*X; +redundant_12(X) when X < 51 -> 5*X; +redundant_12(_) -> none. + %% Test type tests on literal values. (From emulator test suites.) literal_type_tests(Config) when is_list(Config) -> case ?MODULE of @@ -1779,15 +1829,10 @@ t_tuple_size(Config) when is_list(Config) -> error = ludicrous_tuple_size({a,b,c}), error = ludicrous_tuple_size([a,b,c]), - %% Test the "unsafe case" - the register assigned the tuple size is - %% not killed. - DataDir = test_lib:get_data_dir(Config), - File = filename:join(DataDir, "guard_SUITE_tuple_size"), - {ok,Mod,Code} = compile:file(File, [from_asm,binary]), - code:load_binary(Mod, File, Code), - 14 = Mod:t({1,2,3,4}), - _ = code:delete(Mod), - _ = code:purge(Mod), + good_ip({1,2,3,4}), + good_ip({1,2,3,4,5,6,7,8}), + error = validate_ip({42,11}), + error = validate_ip(atom), ok. @@ -1805,6 +1850,16 @@ ludicrous_tuple_size(T) when tuple_size(T) =:= 16#FFFFFFFFFFFFFFFF -> ok; ludicrous_tuple_size(_) -> error. +good_ip(IP) -> + IP = validate_ip(IP). + +validate_ip(Value) when is_tuple(Value) andalso + ((size(Value) =:= 4) orelse (size(Value) =:= 8)) -> + %% size/1 (converted to tuple_size) used more than once. + Value; +validate_ip(_) -> + error. + %% %% The binary_part/2,3 guard BIFs %% @@ -2206,32 +2261,6 @@ maps() -> evidence(#{0 := Charge}) when 0; #{[] => Charge} == #{[] => 42} -> ok. -cover_beam_dead(_Config) -> - Mod = ?FUNCTION_NAME, - Attr = [], - Fs = [{function,test,1,2, - [{label,1}, - {line,[]}, - {func_info,{atom,Mod},{atom,test},1}, - {label,2}, - %% Cover beam_dead:turn_op/1 using swapped operand order. - {test,is_ne_exact,{f,3},[{integer,1},{x,0}]}, - {test,is_eq_exact,{f,1},[{atom,a},{x,0}]}, - {label,3}, - {move,{atom,ok},{x,0}}, - return]}], - Exp = [{test,1}], - Asm = {Mod,Exp,Attr,Fs,3}, - {ok,Mod,Beam} = compile:forms(Asm, [from_asm,binary,report]), - {module,Mod} = code:load_binary(Mod, Mod, Beam), - ok = Mod:test(1), - ok = Mod:test(a), - {'EXIT',_} = (catch Mod:test(other)), - true = code:delete(Mod), - _ = code:purge(Mod), - - ok. - %% Call this function to turn off constant propagation. id(I) -> I. diff --git a/lib/compiler/test/guard_SUITE_data/guard_SUITE_tuple_size.S b/lib/compiler/test/guard_SUITE_data/guard_SUITE_tuple_size.S deleted file mode 100644 index cffb792920..0000000000 --- a/lib/compiler/test/guard_SUITE_data/guard_SUITE_tuple_size.S +++ /dev/null @@ -1,30 +0,0 @@ -{module, guard_SUITE_tuple_size}. %% version = 0 - -{exports, [{t,1}]}. - -{attributes, []}. - -{labels, 5}. - - -{function, t, 1, 2}. - {label,1}. - {func_info,{atom,guard_SUITE_tuple_size},{atom,t},1}. - {label,2}. - {bif,tuple_size,{f,4},[{x,0}],{x,1}}. - {test,is_eq_exact,{f,4},[{x,1},{integer,4}]}. - {test,is_tuple,{f,3},[{x,0}]}. - {test,test_arity,{f,3},[{x,0},4]}. - {get_tuple_element,{x,0},0,{x,5}}. - {get_tuple_element,{x,0},1,{x,2}}. - {get_tuple_element,{x,0},2,{x,3}}. - {get_tuple_element,{x,0},3,{x,4}}. - {gc_bif,'+',{f,0},6,[{x,1},{x,2}],{x,0}}. - {gc_bif,'+',{f,0},6,[{x,0},{x,3}],{x,0}}. - {gc_bif,'+',{f,0},6,[{x,0},{x,4}],{x,0}}. - {gc_bif,'+',{f,0},6,[{x,0},{x,5}],{x,0}}. - return. - {label,3}. - {badmatch,{x,0}}. - {label,4}. - {jump,{f,1}}. diff --git a/lib/compiler/test/inline_SUITE.erl b/lib/compiler/test/inline_SUITE.erl index dcc703c3e1..aff1a56c47 100644 --- a/lib/compiler/test/inline_SUITE.erl +++ b/lib/compiler/test/inline_SUITE.erl @@ -42,13 +42,9 @@ groups() -> init_per_suite(Config) -> test_lib:recompile(?MODULE), - Pa = "-pa " ++ filename:dirname(code:which(?MODULE)), - {ok,Node} = start_node(compiler, Pa), - [{testing_node,Node}|Config]. + Config. -end_per_suite(Config) -> - Node = proplists:get_value(testing_node, Config), - test_server:stop_node(Node), +end_per_suite(_Config) -> ok. init_per_group(_GroupName, Config) -> @@ -89,26 +85,26 @@ attribute(Config) when is_list(Config) -> ?comp(maps_inline_test). try_inline(Mod, Config) -> - Node = proplists:get_value(testing_node, Config), Src = filename:join(proplists:get_value(data_dir, Config), atom_to_list(Mod)), Out = proplists:get_value(priv_dir,Config), %% Normal compilation. io:format("Compiling: ~s\n", [Src]), - {ok,Mod} = compile:file(Src, [{outdir,Out},report,bin_opt_info,clint]), + {ok,Mod} = compile:file(Src, [{outdir,Out},report, + bin_opt_info,clint,ssalint]), ct:timetrap({minutes,10}), - NormalResult = rpc:call(Node, ?MODULE, load_and_call, [Out,Mod]), + NormalResult = load_and_call(Out, Mod), %% Inlining. io:format("Compiling with old inliner: ~s\n", [Src]), {ok,Mod} = compile:file(Src, [{outdir,Out},report,bin_opt_info, - {inline,1000},clint]), + {inline,1000},clint,ssalint]), %% Run inlined code. ct:timetrap({minutes,10}), - OldInlinedResult = rpc:call(Node, ?MODULE, load_and_call, [Out,Mod]), + OldInlinedResult = load_and_call(Out, Mod), %% Compare results. compare(NormalResult, OldInlinedResult), @@ -117,11 +113,11 @@ try_inline(Mod, Config) -> %% Inlining. io:format("Compiling with new inliner: ~s\n", [Src]), {ok,Mod} = compile:file(Src, [{outdir,Out},report, - bin_opt_info,inline,clint]), + bin_opt_info,inline,clint,ssalint]), %% Run inlined code. ct:timetrap({minutes,10}), - InlinedResult = rpc:call(Node, ?MODULE, load_and_call, [Out,Mod]), + InlinedResult = load_and_call(Out, Mod), %% Compare results. compare(NormalResult, InlinedResult), @@ -130,6 +126,11 @@ try_inline(Mod, Config) -> %% Delete Beam file. ok = file:delete(filename:join(Out, atom_to_list(Mod)++code:objfile_extension())), + %% Delete loaded module. + _ = code:purge(Mod), + _ = code:delete(Mod), + _ = code:purge(Mod), + ok. compare(Same, Same) -> ok; @@ -143,12 +144,6 @@ compare([H1|_], [H2|_]) -> ct:fail(different); compare([], []) -> ok. -start_node(Name, Args) -> - case test_server:start_node(Name, slave, [{args,Args}]) of - {ok,Node} -> {ok, Node}; - Error -> ct:fail(Error) - end. - load_and_call(Out, Module) -> io:format("Loading...\n",[]), code:purge(Module), @@ -349,9 +344,8 @@ otp_7223_2({a}) -> 1. coverage(Config) when is_list(Config) -> - Mod = bsdecode, + Mod = attribute, Src = filename:join(proplists:get_value(data_dir, Config), Mod), - {ok,Mod,_} = compile:file(Src, [binary,report,{inline,0},clint]), - {ok,Mod,_} = compile:file(Src, [binary,report,{inline,20}, - verbose,clint]), + {ok,Mod,_} = compile:file(Src, [binary,report,{inline,0}, + clint,ssalint]), ok. diff --git a/lib/compiler/test/inline_SUITE_data/barnes2.erl b/lib/compiler/test/inline_SUITE_data/barnes2.erl index a986331060..49e9bdfb6b 100644 --- a/lib/compiler/test/inline_SUITE_data/barnes2.erl +++ b/lib/compiler/test/inline_SUITE_data/barnes2.erl @@ -6,7 +6,7 @@ ?MODULE() -> Stars = create_scenario(1000, 1.0), R = hd(loop(10,1000.0,Stars,0)), - Str = lists:flatten(io:lib_format("~s", [R])), + Str = lists:flatten(io_lib:format("~p", [R])), {R,Str =:= {1.00000,-1.92269e+4,-1.92269e+4,2.86459e-2,2.86459e-2}}. create_scenario(N, M) -> diff --git a/lib/compiler/test/map_SUITE.erl b/lib/compiler/test/map_SUITE.erl index 3e0ab78390..440b632381 100644 --- a/lib/compiler/test/map_SUITE.erl +++ b/lib/compiler/test/map_SUITE.erl @@ -70,7 +70,10 @@ t_bad_update/1, %% new in OTP 21 - t_reused_key_variable/1 + t_reused_key_variable/1, + + %% new in OTP 22 + t_mixed_clause/1,cover_beam_trim/1 ]). suite() -> []. @@ -124,7 +127,10 @@ all() -> t_bad_update, %% new in OTP 21 - t_reused_key_variable + t_reused_key_variable, + + %% new in OTP 22 + t_mixed_clause,cover_beam_trim ]. groups() -> []. @@ -1373,22 +1379,22 @@ map_usage(Def, Used) -> t_guard_sequence(Config) when is_list(Config) -> - {1, "a"} = map_guard_sequence_1(#{seq=>1,val=>id("a")}), - {2, "b"} = map_guard_sequence_1(#{seq=>2,val=>id("b")}), - {3, "c"} = map_guard_sequence_1(#{seq=>3,val=>id("c")}), - {4, "d"} = map_guard_sequence_1(#{seq=>4,val=>id("d")}), - {5, "e"} = map_guard_sequence_1(#{seq=>5,val=>id("e")}), - - {1,M1} = map_guard_sequence_2(M1 = id(#{a=>3})), - {2,M2} = map_guard_sequence_2(M2 = id(#{a=>4, b=>4})), - {3,gg,M3} = map_guard_sequence_2(M3 = id(#{a=>gg, b=>4})), - {4,sc,sc,M4} = map_guard_sequence_2(M4 = id(#{a=>sc, b=>3, c=>sc2})), - {5,kk,kk,M5} = map_guard_sequence_2(M5 = id(#{a=>kk, b=>other, c=>sc2})), - - %% error case - {'EXIT',{function_clause,_}} = (catch map_guard_sequence_1(#{seq=>6,val=>id("e")})), - {'EXIT',{function_clause,_}} = (catch map_guard_sequence_2(#{b=>5})), - ok. + {1, "a"} = map_guard_sequence_1(#{seq=>1,val=>id("a")}), + {2, "b"} = map_guard_sequence_1(#{seq=>2,val=>id("b")}), + {3, "c"} = map_guard_sequence_1(#{seq=>3,val=>id("c")}), + {4, "d"} = map_guard_sequence_1(#{seq=>4,val=>id("d")}), + {5, "e"} = map_guard_sequence_1(#{seq=>5,val=>id("e")}), + + {1,M1} = map_guard_sequence_2(M1 = id(#{a=>3})), + {2,M2} = map_guard_sequence_2(M2 = id(#{a=>4, b=>4})), + {3,gg,M3} = map_guard_sequence_2(M3 = id(#{a=>gg, b=>4})), + {4,sc,sc,M4} = map_guard_sequence_2(M4 = id(#{a=>sc, b=>3, c=>sc2})), + {5,kk,kk,M5} = map_guard_sequence_2(M5 = id(#{a=>kk, b=>other, c=>sc2})), + + %% error case + {'EXIT',{function_clause,_}} = (catch map_guard_sequence_1(#{seq=>6,val=>id("e")})), + {'EXIT',{function_clause,_}} = (catch map_guard_sequence_2(#{b=>5})), + ok. t_guard_sequence_large(Config) when is_list(Config) -> M0 = id(#{ 10=>a0,20=>b0,30=>"c0","40"=>"d0",<<"50">>=>"e0",{["00",03]}=>"10", @@ -1443,21 +1449,21 @@ t_guard_sequence_large(Config) when is_list(Config) -> 18=>a8,28=>b8,38=>"c8","48"=>"d8",<<"58">>=>"e8",{["08"]}=>"18", 19=>a9,29=>b9,39=>"c9","49"=>"d9",<<"59">>=>"e9",{["09"]}=>"19" } => "large map key 2" }), - {1, "a"} = map_guard_sequence_1(M0#{seq=>1,val=>id("a")}), - {2, "b"} = map_guard_sequence_1(M0#{seq=>2,val=>id("b")}), - {3, "c"} = map_guard_sequence_1(M0#{seq=>3,val=>id("c")}), - {4, "d"} = map_guard_sequence_1(M0#{seq=>4,val=>id("d")}), - {5, "e"} = map_guard_sequence_1(M0#{seq=>5,val=>id("e")}), + {1, "a"} = map_guard_sequence_1(M0#{seq=>1,val=>id("a")}), + {2, "b"} = map_guard_sequence_1(M0#{seq=>2,val=>id("b")}), + {3, "c"} = map_guard_sequence_1(M0#{seq=>3,val=>id("c")}), + {4, "d"} = map_guard_sequence_1(M0#{seq=>4,val=>id("d")}), + {5, "e"} = map_guard_sequence_1(M0#{seq=>5,val=>id("e")}), - {1,M1} = map_guard_sequence_2(M1 = id(M0#{a=>3})), - {2,M2} = map_guard_sequence_2(M2 = id(M0#{a=>4, b=>4})), - {3,gg,M3} = map_guard_sequence_2(M3 = id(M0#{a=>gg, b=>4})), - {4,sc,sc,M4} = map_guard_sequence_2(M4 = id(M0#{a=>sc, b=>3, c=>sc2})), - {5,kk,kk,M5} = map_guard_sequence_2(M5 = id(M0#{a=>kk, b=>other, c=>sc2})), + {1,M1} = map_guard_sequence_2(M1 = id(M0#{a=>3})), + {2,M2} = map_guard_sequence_2(M2 = id(M0#{a=>4, b=>4})), + {3,gg,M3} = map_guard_sequence_2(M3 = id(M0#{a=>gg, b=>4})), + {4,sc,sc,M4} = map_guard_sequence_2(M4 = id(M0#{a=>sc, b=>3, c=>sc2})), + {5,kk,kk,M5} = map_guard_sequence_2(M5 = id(M0#{a=>kk, b=>other, c=>sc2})), - {'EXIT',{function_clause,_}} = (catch map_guard_sequence_1(M0#{seq=>6,val=>id("e")})), - {'EXIT',{function_clause,_}} = (catch map_guard_sequence_2(M0#{b=>5})), - ok. + {'EXIT',{function_clause,_}} = (catch map_guard_sequence_1(M0#{seq=>6,val=>id("e")})), + {'EXIT',{function_clause,_}} = (catch map_guard_sequence_2(M0#{b=>5})), + ok. map_guard_sequence_1(#{seq:=1=Seq, val:=Val}) -> {Seq,Val}; map_guard_sequence_1(#{seq:=2=Seq, val:=Val}) -> {Seq,Val}; @@ -2079,7 +2085,7 @@ t_register_corruption(Config) when is_list(Config) -> {3,wanted,<<"value">>} = register_corruption_foo(wanted,M), ok. -register_corruption_foo(A,#{a := V1, b := V2}) -> +register_corruption_foo(_,#{a := V1, b := V2}) -> register_corruption_dummy_call(1,V1,V2); register_corruption_foo(A,#{b := V}) -> register_corruption_dummy_call(2,A,V); @@ -2161,6 +2167,31 @@ t_reused_key_variable(Config) when is_list(Config) -> ok end. +t_mixed_clause(_Config) -> + put(fool_inliner, x), + K = get(fool_inliner), + {42,100} = case #{K=>42,y=>100} of + #{x:=X,y:=Y} -> + {X,Y} + end, + nomatch = case #{K=>42,y=>100} of + #{x:=X,y:=0} -> + {X,Y}; + #{} -> + nomatch + end, + ok. + +cover_beam_trim(_Config) -> + val = do_cover_beam_trim(id, max, max, id, #{id=>val}), + ok. + +do_cover_beam_trim(Id, OldMax, Max, Id, M) -> + OldMax = id(Max), + #{Id:=Val} = id(M), + Val. + + %% aux rand_terms(0) -> []; diff --git a/lib/compiler/test/match_SUITE.erl b/lib/compiler/test/match_SUITE.erl index 72e5356a8d..94bfbb0efe 100644 --- a/lib/compiler/test/match_SUITE.erl +++ b/lib/compiler/test/match_SUITE.erl @@ -25,7 +25,7 @@ match_in_call/1,untuplify/1,shortcut_boolean/1,letify_guard/1, selectify/1,deselectify/1,underscore/1,match_map/1,map_vars_used/1, coverage/1,grab_bag/1,literal_binary/1, - unary_op/1]). + unary_op/1,eq_types/1,match_after_return/1]). -include_lib("common_test/include/ct.hrl"). @@ -40,7 +40,8 @@ groups() -> match_in_call,untuplify, shortcut_boolean,letify_guard,selectify,deselectify, underscore,match_map,map_vars_used,coverage, - grab_bag,literal_binary,unary_op]}]. + grab_bag,literal_binary,unary_op,eq_types, + match_after_return]}]. init_per_suite(Config) -> @@ -254,6 +255,8 @@ non_matching_aliases(_Config) -> none = mixed_aliases([d]), none = mixed_aliases({a,42}), none = mixed_aliases(42), + none = mixed_aliases(<<6789:16>>), + none = mixed_aliases(#{key=>value}), {'EXIT',{{badmatch,42},_}} = (catch nomatch_alias(42)), {'EXIT',{{badmatch,job},_}} = (catch entirely()), @@ -279,6 +282,16 @@ mixed_aliases(<<X:8>> = x) -> {a,X}; mixed_aliases([b] = <<X:8>>) -> {b,X}; mixed_aliases(<<X:8>> = {a,X}) -> {c,X}; mixed_aliases([X] = <<X:8>>) -> {d,X}; +mixed_aliases(<<X:16>> = X) -> {e,X}; +mixed_aliases(X = <<X:16>>) -> {f,X}; +mixed_aliases(<<X:16,_/binary>> = X) -> {g,X}; +mixed_aliases(X = <<X:16,_/binary>>) -> {h,X}; +mixed_aliases(X = #{key:=X}) -> {i,X}; +mixed_aliases(#{key:=X} = X) -> {j,X}; +mixed_aliases([X] = #{key:=X}) -> {k,X}; +mixed_aliases(#{key:=X} = [X]) -> {l,X}; +mixed_aliases({a,X} = #{key:=X}) -> {m,X}; +mixed_aliases(#{key:=X} = {a,X}) -> {n,X}; mixed_aliases(_) -> none. nomatch_alias(I) -> @@ -456,6 +469,7 @@ letify_guard(A, B) -> selectify(Config) when is_list(Config) -> integer = sel_different_types({r,42}), atom = sel_different_types({r,forty_two}), + float = sel_different_types({r,100.0}), none = sel_different_types({r,18}), {'EXIT',_} = (catch sel_different_types([a,b,c])), @@ -466,12 +480,15 @@ selectify(Config) when is_list(Config) -> integer42 = sel_same_value2(42), integer43 = sel_same_value2(43), error = sel_same_value2(44), + ok. sel_different_types({r,_}=T) when element(2, T) =:= forty_two -> atom; sel_different_types({r,_}=T) when element(2, T) =:= 42 -> integer; +sel_different_types({r,_}=T) when element(2, T) =:= 100.0 -> + float; sel_different_types({r,_}) -> none. @@ -489,9 +506,8 @@ sel_same_value2(V) when V =:= 42; V =:= 43 -> sel_same_value2(_) -> error. -%% Test deconstruction of select_val instructions in beam_peep into -%% regular tests with just one possible value left. Hitting proper cases -%% in beam_peep relies on unification of labels by beam_jump. +%% Test deconstruction of select_val instructions to regular tests +%% with zero or one values left. deselectify(Config) when is_list(Config) -> one_or_other = desel_tuple_arity({1}), @@ -512,7 +528,31 @@ deselectify(Config) when is_list(Config) -> one_or_other = dsel_atom_typecheck(one), two = dsel_atom_typecheck(two), - one_or_other = dsel_atom_typecheck(three). + one_or_other = dsel_atom_typecheck(three), + + %% Cover deconstruction of select_val instructions in + %% beam_peep. + + stop = dsel_peek_0(stop), + ignore = dsel_peek_0(ignore), + Config = dsel_peek_0(Config), + + stop = dsel_peek_1(stop, any), + Config = dsel_peek_1(ignore, Config), + other = dsel_peek_1(other, ignored), + + 0 = dsel_peek_2(0, any), + Config = dsel_peek_2(1, Config), + 2 = dsel_peek_2(2, ignored), + + true = dsel_peek_3(true), + false = dsel_peek_3(false), + {error,Config} = dsel_peek_3(Config), + + ok. + +%% The following will be optimized by the sharing optimizations +%% in beam_ssa_opt. desel_tuple_arity(Tuple) when is_tuple(Tuple) -> case Tuple of @@ -549,6 +589,39 @@ dsel_atom_typecheck(Val) when is_atom(Val) -> _ -> one_or_other end. +%% The following functions are carefully crafted so that the sharing +%% optimizations in beam_ssa_opt can't be applied. After applying the +%% beam_jump:eliminate_moves/1 optimization and beam_clean:clean_labels/1 +%% has unified labels, beam_peep is able to optimize these functions. + +dsel_peek_0(A0) -> + case id(A0) of + stop -> stop; + ignore -> ignore; + A -> A + end. + +dsel_peek_1(A0, B) -> + case id(A0) of + stop -> stop; + ignore -> B; + A -> A + end. + +dsel_peek_2(A0, B) -> + case id(A0) of + 0 -> 0; + 1 -> B; + A -> A + end. + +dsel_peek_3(A0) -> + case id(A0) of + true -> true; + false -> false; + Other -> {error,Other} + end. + underscore(Config) when is_list(Config) -> case Config of [] -> @@ -591,13 +664,26 @@ do_map_vars_used(X, Y, Map) -> Val end. +-record(coverage_id, {bool=false,id}). coverage(Config) when is_list(Config) -> %% Cover beam_dead. ok = coverage_1(x, a), ok = coverage_1(x, b), %% Cover sys_pre_expand. - ok = coverage_3("abc"). + ok = coverage_3("abc"), + + %% Cover beam_ssa_dead. + {expr,key} = coverage_4([literal,get], [[expr,key]]), + {expr,key} = coverage_4([expr,key], []), + + a = coverage_5([8,8,8], #coverage_id{bool=true}), + b = coverage_5([], #coverage_id{bool=true}), + + %% Cover beam_ssa_opt. + ok = coverage_6(), + + ok. coverage_1(B, Tag) -> case Tag of @@ -610,6 +696,37 @@ coverage_2(2, b, x) -> ok. coverage_3([$a]++[]++"bc") -> ok. +%% Cover beam_ssa_dead:eval_type_test_1(is_nonempty_list, Arg). +coverage_4([literal,get], [Expr]) -> + coverage_4(Expr, []); +coverage_4([Expr,Key], []) -> + {Expr,Key}. + +%% Cover beam_ssa_dead:eval_type_test_1(is_tagged_tuple, Arg). +coverage_5(Config, TermId) + when TermId =:= #coverage_id{bool=true}, + Config =:= [8,8,8] -> + a; +coverage_5(_Config, #coverage_id{bool=true}) -> + b. + +coverage_6() -> + X = 17, + case + case id(1) > 0 of + true -> + 17; + false -> + 42 + end + of + X -> + ok; + V -> + %% Cover beam_ssa_opt:make_literal/2. + error([error,X,V]) + end. + grab_bag(_Config) -> [_|T] = id([a,b,c]), [b,c] = id(T), @@ -754,5 +871,35 @@ unary_op_1(Vop@1) -> end end. +eq_types(_Config) -> + Ref = make_ref(), + Ref = eq_types(Ref, any), + ok. + +eq_types(A, B) -> + %% {put_tuple2,{y,0},{list,[{x,0},{x,1}]}}. + Term0 = {A, B}, + Term = id(Term0), + + %% {test,is_eq_exact,{f,3},[{y,0},{x,0}]}. + %% Here beam_validator must infer that {x,0} has the + %% same type as {y,0}. + Term = Term0, + + %% {get_tuple_element,{x,0},0,{x,0}}. + {Ref22,_} = Term, + + Ref22. + +match_after_return(Config) when is_list(Config) -> + %% The return type of the following call will never match the 'wont_happen' + %% clauses below, and the beam_ssa_type was clever enough to see that but + %% didn't remove the blocks, so it crashed when trying to extract A. + ok = case mar_test_tuple(erlang:unique_integer()) of + {gurka, never_matches, A} -> {wont_happen, A}; + _ -> ok + end. + +mar_test_tuple(I) -> {gurka, I}. id(I) -> I. diff --git a/lib/compiler/test/misc_SUITE.erl b/lib/compiler/test/misc_SUITE.erl index a1d931b994..a0b415ceaa 100644 --- a/lib/compiler/test/misc_SUITE.erl +++ b/lib/compiler/test/misc_SUITE.erl @@ -161,17 +161,16 @@ md5_1(Beam) -> %% Cover some code that handles internal errors. silly_coverage(Config) when is_list(Config) -> - %% sys_core_fold, sys_core_alias, sys_core_bsm, sys_core_setel, v3_kernel + %% sys_core_fold, sys_core_alias, sys_core_bsm, v3_kernel BadCoreErlang = {c_module,[], name,[],[], [{{c_var,[],{foo,2}},seriously_bad_body}]}, expect_error(fun() -> sys_core_fold:module(BadCoreErlang, []) end), expect_error(fun() -> sys_core_alias:module(BadCoreErlang, []) end), expect_error(fun() -> sys_core_bsm:module(BadCoreErlang, []) end), - expect_error(fun() -> sys_core_dsetel:module(BadCoreErlang, []) end), expect_error(fun() -> v3_kernel:module(BadCoreErlang, []) end), - %% v3_codegen + %% beam_kernel_to_ssa BadKernel = {k_mdef,[],?MODULE, [{foo,0}], [], @@ -179,7 +178,38 @@ silly_coverage(Config) when is_list(Config) -> {k,[],[],[]}, f,0,[], seriously_bad_body}]}, - expect_error(fun() -> v3_codegen:module(BadKernel, []) end), + expect_error(fun() -> beam_kernel_to_ssa:module(BadKernel, []) end), + + %% beam_ssa_lint + %% beam_ssa_recv + %% beam_ssa_share + %% beam_ssa_pre_codegen + %% beam_ssa_codegen + BadSSA = {b_module,#{},a,b,c, + [{b_function,#{func_info=>{mod,foo,0}},args,bad_blocks,0}]}, + expect_error(fun() -> beam_ssa_lint:module(BadSSA, []) end), + expect_error(fun() -> beam_ssa_recv:module(BadSSA, []) end), + expect_error(fun() -> beam_ssa_share:module(BadSSA, []) end), + expect_error(fun() -> beam_ssa_pre_codegen:module(BadSSA, []) end), + expect_error(fun() -> beam_ssa_codegen:module(BadSSA, []) end), + + %% beam_ssa_opt + BadSSABlocks = #{0 => {b_blk,#{},[bad_code],{b_ret,#{},arg}}}, + BadSSAOpt = {b_module,#{},a,[],c, + [{b_function,#{func_info=>{mod,foo,0}},[], + BadSSABlocks,0}]}, + expect_error(fun() -> beam_ssa_opt:module(BadSSAOpt, []) end), + + %% beam_ssa_lint, beam_ssa_pp + {error,[{_,Errors}]} = beam_ssa_lint:module(bad_ssa_lint_input(), []), + _ = [io:put_chars(Mod:format_error(Reason)) || + {Mod,Reason} <- Errors], + + %% Cover printing of annotations in beam_ssa_pp + PPAnno = #{func_info=>{mod,foo,0},other_anno=>value,map_anno=>#{k=>v}}, + PPBlocks = #{0=>{b_blk,#{},[],{b_ret,#{},{b_literal,42}}}}, + PP = {b_function,PPAnno,[],PPBlocks,0}, + io:put_chars(beam_ssa_pp:format_function(PP)), %% beam_a BeamAInput = {?MODULE,[{foo,0}],[], @@ -189,14 +219,6 @@ silly_coverage(Config) when is_list(Config) -> {label,2}|non_proper_list]}],99}, expect_error(fun() -> beam_a:module(BeamAInput, []) end), - %% beam_reorder - BlockInput = {?MODULE,[{foo,0}],[], - [{function,foo,0,2, - [{label,1}, - {func_info,{atom,?MODULE},{atom,foo},0}, - {label,2}|non_proper_list]}],99}, - expect_error(fun() -> beam_reorder:module(BlockInput, []) end), - %% beam_block BlockInput = {?MODULE,[{foo,0}],[], [{function,foo,0,2, @@ -205,19 +227,6 @@ silly_coverage(Config) when is_list(Config) -> {label,2}|non_proper_list]}],99}, expect_error(fun() -> beam_block:module(BlockInput, []) end), - %% beam_bs - BsInput = BlockInput, - expect_error(fun() -> beam_bs:module(BsInput, []) end), - - %% beam_type - TypeInput = {?MODULE,[{foo,0}],[], - [{function,foo,0,2, - [{label,1}, - {line,loc}, - {func_info,{atom,?MODULE},{atom,foo},0}, - {label,2}|non_proper_list]}],99}, - expect_error(fun() -> beam_type:module(TypeInput, []) end), - %% beam_except ExceptInput = {?MODULE,[{foo,0}],[], [{function,foo,0,2, @@ -227,15 +236,9 @@ silly_coverage(Config) when is_list(Config) -> {label,2}|non_proper_list]}],99}, expect_error(fun() -> beam_except:module(ExceptInput, []) end), - %% beam_dead. This is tricky. Our function must look OK to - %% beam_utils:clean_labels/1, but must crash beam_dead. - DeadInput = {?MODULE,[{foo,0}],[], - [{function,foo,0,2, - [{label,1}, - {func_info,{atom,?MODULE},{atom,foo},0}, - {label,2}, - {test,is_eq_exact,{f,1},[bad,operands]}]}],99}, - expect_error(fun() -> beam_dead:module(DeadInput, []) end), + %% beam_jump + JumpInput = BlockInput, + expect_error(fun() -> beam_jump:module(JumpInput, []) end), %% beam_clean CleanInput = {?MODULE,[{foo,0}],[], @@ -246,6 +249,10 @@ silly_coverage(Config) when is_list(Config) -> {jump,{f,42}}]}],99}, expect_error(fun() -> beam_clean:module(CleanInput, []) end), + %% beam_jump + TrimInput = BlockInput, + expect_error(fun() -> beam_trim:module(TrimInput, []) end), + %% beam_peep. This is tricky. Use a select instruction with %% an odd number of elements in the list to crash %% prune_redundant_values/2 but not beam_clean:clean_labels/1. @@ -253,48 +260,10 @@ silly_coverage(Config) when is_list(Config) -> [{function,foo,0,2, [{label,1}, {func_info,{atom,?MODULE},{atom,foo},0}, - {label,2},{select,op,r,{f,2},[{f,2}]}]}], + {label,2},{select,select_val,r,{f,2},[{f,2}]}]}], 2}, expect_error(fun() -> beam_peep:module(PeepInput, []) end), - %% beam_bsm. This is tricky. Our function must be sane enough to not crash - %% btb_index/1, but must crash the main optimization pass. - BsmInput = {?MODULE,[{foo,0}],[], - [{function,foo,0,2, - [{label,1}, - {func_info,{atom,?MODULE},{atom,foo},0}, - {label,2}, - {test,bs_get_binary2,{f,99},0,[{x,0},{atom,all},1,[]],{x,0}}, - {block,[a|b]}]}],0}, - expect_error(fun() -> beam_bsm:module(BsmInput, []) end), - - %% beam_receive. - ReceiveInput = {?MODULE,[{foo,0}],[], - [{function,foo,0,2, - [{label,1}, - {func_info,{atom,?MODULE},{atom,foo},0}, - {label,2}, - {call_ext,0,{extfunc,erlang,make_ref,0}}, - {block,[a|b]}]}],0}, - expect_error(fun() -> beam_receive:module(ReceiveInput, []) end), - - %% beam_record. - RecordInput = {?MODULE,[{foo,0}],[], - [{function,foo,1,2, - [{label,1}, - {func_info,{atom,?MODULE},{atom,foo},1}, - {label,2}, - {test,is_tuple,{f,1},[{x,0}]}, - {test,test_arity,{f,1},[{x,0},3]}, - {block,[{set,[{x,1}],[{x,0}],{get_tuple_element,0}}]}, - {test,is_eq_exact,{f,1},[{x,1},{atom,bar}]}, - {block,[{set,[{x,2}],[{x,0}],{get_tuple_element,1}}|a]}, - {test,is_eq_exact,{f,1},[{x,2},{integer,1}]}, - {block,[{set,[{x,0}],[{atom,ok}],move}]}, - return]}],0}, - - expect_error(fun() -> beam_record:module(RecordInput, []) end), - BeamZInput = {?MODULE,[{foo,0}],[], [{function,foo,0,2, [{label,1}, @@ -312,6 +281,31 @@ silly_coverage(Config) when is_list(Config) -> ok. +bad_ssa_lint_input() -> + {b_module,#{},t, + [{foobar,1},{module_info,0},{module_info,1}], + [], + [{b_function, + #{func_info => {t,foobar,1},location => {"t.erl",4}}, + [{b_var,0}], + #{0 => {b_blk,#{},[],{b_ret,#{},{b_var,'@undefined_var'}}}}, + 3}, + {b_function, + #{func_info => {t,module_info,0}}, + [], + #{0 => + {b_blk,#{}, + [{b_set,#{}, + {b_var,{'@ssa_ret',3}}, + call, + [{b_remote, + {b_literal,erlang}, + {b_literal,get_module_info}, + 1}, + {b_var,'@unknown_variable'}]}], + {b_ret,#{},{b_var,{'@ssa_ret',3}}}}}, + 4}]}. + expect_error(Fun) -> try Fun() of Any -> diff --git a/lib/compiler/test/receive_SUITE.erl b/lib/compiler/test/receive_SUITE.erl index 4219768d6f..0038eb1a4b 100644 --- a/lib/compiler/test/receive_SUITE.erl +++ b/lib/compiler/test/receive_SUITE.erl @@ -25,7 +25,8 @@ init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, export/1,recv/1,coverage/1,otp_7980/1,ref_opt/1, - wait/1,recv_in_try/1,double_recv/1]). + wait/1,recv_in_try/1,double_recv/1,receive_var_zero/1, + match_built_terms/1]). -include_lib("common_test/include/ct.hrl"). @@ -45,7 +46,8 @@ all() -> groups() -> [{p,test_lib:parallel(), [recv,coverage,otp_7980,ref_opt,export,wait, - recv_in_try,double_recv]}]. + recv_in_try,double_recv,receive_var_zero, + match_built_terms]}]. init_per_suite(Config) -> @@ -378,4 +380,51 @@ do_double_recv(_, Msg) -> error end. +%% Test 'after Z', when Z =:= 0 been propagated as an immediate by the type +%% optimization pass. +receive_var_zero(Config) when is_list(Config) -> + self() ! x, + self() ! y, + Z = zero(), + timeout = receive + z -> ok + after Z -> timeout + end, + timeout = receive + after Z -> timeout + end, + self() ! w, + receive + x -> ok; + Other -> + ct:fail({bad_message,Other}) + end. + +zero() -> 0. + +%% ERL-862; the validator would explode when a term was constructed in a +%% receive guard. + +-define(MATCH_BUILT_TERM(Ref, Expr), + (fun() -> + Ref = make_ref(), + A = id($a), + B = id($b), + Built = id(Expr), + self() ! {Ref, A, B}, + receive + {Ref, A, B} when Expr =:= Built -> + ok + after 5000 -> + ct:fail("Failed to match message with term built in " + "receive guard.") + end + end)()). + +match_built_terms(Config) when is_list(Config) -> + ?MATCH_BUILT_TERM(Ref, [A, B]), + ?MATCH_BUILT_TERM(Ref, {A, B}), + ?MATCH_BUILT_TERM(Ref, <<A, B>>), + ?MATCH_BUILT_TERM(Ref, #{ 1 => A, 2 => B}). + id(I) -> I. diff --git a/lib/compiler/test/receive_SUITE_data/ref_opt/no_5.erl b/lib/compiler/test/receive_SUITE_data/ref_opt/no_5.erl new file mode 100644 index 0000000000..4fbde3a83d --- /dev/null +++ b/lib/compiler/test/receive_SUITE_data/ref_opt/no_5.erl @@ -0,0 +1,38 @@ +-module(no_5). +-compile([export_all,nowarn_export_all]). + +?MODULE() -> + ok. + +%% Nested receives were not handled properly. + +confusing_recv_mark(Pid) -> + Ref = make_ref(), + %% There would be a recv_mark here. + MRef = erlang:monitor(process, Pid), + receive + Ref -> + %% And a recv_set here. + receive + MRef -> gurka + end; + MRef -> + gaffel + end. + +%% The optimization could potentially be improved to +%% handle matching of multiple refs, like this: + +proper_recv_mark(Pid) -> + %% Place the recv_mark before the creation of both refs. + Ref = make_ref(), + MRef = erlang:monitor(process, Pid), + %% Place the recv_set here. + receive + Ref -> + receive + MRef -> gurka + end; + MRef -> + gaffel + end. diff --git a/lib/compiler/test/receive_SUITE_data/ref_opt/yes_14.S b/lib/compiler/test/receive_SUITE_data/ref_opt/yes_14.S deleted file mode 100644 index fd14228135..0000000000 --- a/lib/compiler/test/receive_SUITE_data/ref_opt/yes_14.S +++ /dev/null @@ -1,71 +0,0 @@ -{module, yes_14}. %% version = 0 - -{exports, [{f,2},{module_info,0},{module_info,1},{yes_14,0}]}. - -{attributes, []}. - -{labels, 12}. - - -{function, yes_14, 0, 2}. - {label,1}. - {func_info,{atom,yes_14},{atom,yes_14},0}. - {label,2}. - {move,{atom,ok},{x,0}}. - return. - - -{function, f, 2, 4}. - {label,3}. - {func_info,{atom,yes_14},{atom,f},2}. - {label,4}. - {allocate_heap,2,3,2}. - {move,{x,0},{y,1}}. - {put_tuple,2,{y,0}}. - {put,{atom,data}}. - {put,{x,1}}. - {call_ext,0,{extfunc,erlang,make_ref,0}}. % Ref in [x0] - {test_heap,4,1}. - {put_tuple,3,{x,1}}. - {put,{atom,request}}. - {put,{x,0}}. - {put,{y,0}}. - {move,{x,0},{y,0}}. % Ref in [x0,y0] - {move,{y,1},{x,0}}. % Ref in [y0] - {kill,{y,1}}. - send. - {move,{y,0},{x,0}}. % Ref in [x0,y0] - {move,{x,0},{y,1}}. % Ref in [x0,y0,y1] - {label,5}. - {loop_rec,{f,7},{x,0}}. % Ref in [y0,y1] - {test,is_tuple,{f,6},[{x,0}]}. - {test,test_arity,{f,6},[{x,0},2]}. - {get_tuple_element,{x,0},0,{x,1}}. - {get_tuple_element,{x,0},1,{x,2}}. - {test,is_eq_exact,{f,6},[{x,1},{atom,reply}]}. - {test,is_eq_exact,{f,6},[{x,2},{y,1}]}. - remove_message. - {move,{atom,ok},{x,0}}. - {deallocate,2}. - return. - {label,6}. - {loop_rec_end,{f,5}}. - {label,7}. - {wait,{f,5}}. - - -{function, module_info, 0, 9}. - {label,8}. - {func_info,{atom,yes_14},{atom,module_info},0}. - {label,9}. - {move,{atom,yes_14},{x,0}}. - {call_ext_only,1,{extfunc,erlang,get_module_info,1}}. - - -{function, module_info, 1, 11}. - {label,10}. - {func_info,{atom,yes_14},{atom,module_info},1}. - {label,11}. - {move,{x,0},{x,1}}. - {move,{atom,yes_14},{x,0}}. - {call_ext_only,2,{extfunc,erlang,get_module_info,2}}. diff --git a/lib/compiler/test/receive_SUITE_data/ref_opt/yes_14.erl b/lib/compiler/test/receive_SUITE_data/ref_opt/yes_14.erl new file mode 100644 index 0000000000..aa47c02af9 --- /dev/null +++ b/lib/compiler/test/receive_SUITE_data/ref_opt/yes_14.erl @@ -0,0 +1,27 @@ +-module(yes_14). +-compile(export_all). + +?MODULE() -> + ok. + +do_call(Process, Request) -> + Mref = erlang:monitor(process, Process), + Process ! Request, + Local = case node(Process) of + Node when Node =:= node() -> true; + _Node -> false + end, + id(Local), + receive + {X,Y,Z} when Mref =/= X, Z =:= 42, Mref =:= Y -> + error; + {X,Y,_} when Mref =/= X, Mref =:= Y -> + error; + {Mref, Reply} -> + erlang:demonitor(Mref, [flush]), + {ok, Reply}; + {'DOWN', Mref, _, _, _} -> + error + end. + +id(I) -> I. diff --git a/lib/compiler/test/regressions_SUITE.erl b/lib/compiler/test/regressions_SUITE.erl index 9b0b9b0c38..39febf060f 100644 --- a/lib/compiler/test/regressions_SUITE.erl +++ b/lib/compiler/test/regressions_SUITE.erl @@ -23,7 +23,7 @@ -export([all/0,groups/0,init_per_testcase/2,end_per_testcase/2, init_per_group/2,end_per_group/2, - init_per_testcase/2,end_per_testcase/2, + init_per_suite/1,end_per_suite/1, suite/0]). -export([maps/1]). diff --git a/lib/compiler/test/test_lib.erl b/lib/compiler/test/test_lib.erl index 8954a9f5fb..3348c6e9ea 100644 --- a/lib/compiler/test/test_lib.erl +++ b/lib/compiler/test/test_lib.erl @@ -22,7 +22,8 @@ -include_lib("common_test/include/ct.hrl"). -compile({no_auto_import,[binary_part/2]}). -export([id/1,recompile/1,parallel/0,uniq/0,opt_opts/1,get_data_dir/1, - is_cloned_mod/1,smoke_disasm/1,p_run/2]). + is_cloned_mod/1,smoke_disasm/1,p_run/2, + highest_opcode/1]). %% Used by test case that override BIFs. -export([binary_part/2,binary/1]). @@ -50,12 +51,8 @@ smoke_disasm(File) when is_list(File) -> Res = beam_disasm:file(File), {beam_file,_Mod} = {element(1, Res),element(2, Res)}. -%% If we are running cover, we don't want to run test cases that -%% invokes the compiler in parallel, as doing so would probably -%% be slower than running them sequentially. - parallel() -> - case test_server:is_cover() orelse erlang:system_info(schedulers) =:= 1 of + case erlang:system_info(schedulers) =:= 1 of true -> []; false -> [parallel] end. @@ -70,18 +67,24 @@ uniq() -> opt_opts(Mod) -> Comp = Mod:module_info(compile), {options,Opts} = lists:keyfind(options, 1, Comp), - lists:filter(fun(no_copt) -> true; - (no_postopt) -> true; - (no_float_opt) -> true; - (no_new_funs) -> true; - (no_new_binaries) -> true; - (no_new_apply) -> true; - (no_gc_bifs) -> true; - (no_stack_trimming) -> true; - (debug_info) -> true; - (inline) -> true; - (_) -> false - end, Opts). + lists:filter(fun + (debug_info) -> true; + (inline) -> true; + (no_bsm3) -> true; + (no_bsm_opt) -> true; + (no_copt) -> true; + (no_fun_opt) -> true; + (no_module_opt) -> true; + (no_postopt) -> true; + (no_put_tuple2) -> true; + (no_recv_opt) -> true; + (no_share_opt) -> true; + (no_ssa_float) -> true; + (no_ssa_opt) -> true; + (no_stack_trimming) -> true; + (no_type_opt) -> true; + (_) -> false + end, Opts). %% Some test suites gets cloned (e.g. to "record_SUITE" to %% "record_no_opt_SUITE"), but the data directory is not cloned. @@ -91,37 +94,40 @@ get_data_dir(Config) -> Data0 = proplists:get_value(data_dir, Config), Opts = [{return,list}], Data1 = re:replace(Data0, "_no_opt_SUITE", "_SUITE", Opts), - Data = re:replace(Data1, "_post_opt_SUITE", "_SUITE", Opts), - re:replace(Data, "_inline_SUITE", "_SUITE", Opts). + Data2 = re:replace(Data1, "_post_opt_SUITE", "_SUITE", Opts), + Data3 = re:replace(Data2, "_inline_SUITE", "_SUITE", Opts), + Data4 = re:replace(Data3, "_r21_SUITE", "_SUITE", Opts), + Data = re:replace(Data4, "_no_module_opt_SUITE", "_SUITE", Opts), + re:replace(Data, "_no_ssa_opt_SUITE", "_SUITE", Opts). is_cloned_mod(Mod) -> is_cloned_mod_1(atom_to_list(Mod)). %% Test whether Mod is a cloned module. -is_cloned_mod_1("no_opt_SUITE") -> true; -is_cloned_mod_1("post_opt_SUITE") -> true; -is_cloned_mod_1("inline_SUITE") -> true; +is_cloned_mod_1("_no_opt_SUITE") -> true; +is_cloned_mod_1("_no_ssa_opt_SUITE") -> true; +is_cloned_mod_1("_post_opt_SUITE") -> true; +is_cloned_mod_1("_inline_SUITE") -> true; +is_cloned_mod_1("_21_SUITE") -> true; +is_cloned_mod_1("_no_module_opt_SUITE") -> true; is_cloned_mod_1([_|T]) -> is_cloned_mod_1(T); is_cloned_mod_1([]) -> false. +%% Return the highest opcode use in the BEAM module. + +highest_opcode(Beam) -> + {ok,{_Mod,[{"Code",Code}]}} = beam_lib:chunks(Beam, ["Code"]), + FormatNumber = 0, + <<16:32,FormatNumber:32,HighestOpcode:32,_/binary>> = Code, + HighestOpcode. + %% p_run(fun(Data) -> ok|error, List) -> ok %% Will fail the test case if there were any errors. p_run(Test, List) -> S = erlang:system_info(schedulers), - N = case test_server:is_cover() of - false -> - S + 1; - true -> - %% Cover is running. Using too many processes - %% could slow us down. Measurements on my computer - %% showed that using 4 parallel processes was - %% slightly faster than using 3. Using more than - %% 4 would not buy us much and could actually be - %% slower. - min(S, 4) - end, + N = S + 1, io:format("p_run: ~p parallel processes\n", [N]), p_run_loop(Test, List, N, [], 0, 0). diff --git a/lib/compiler/test/trycatch_SUITE.erl b/lib/compiler/test/trycatch_SUITE.erl index 8f9cd9ab1e..539f9d69fa 100644 --- a/lib/compiler/test/trycatch_SUITE.erl +++ b/lib/compiler/test/trycatch_SUITE.erl @@ -27,7 +27,8 @@ nested_horrid/1,last_call_optimization/1,bool/1, plain_catch_coverage/1,andalso_orelse/1,get_in_try/1, hockey/1,handle_info/1,catch_in_catch/1,grab_bag/1, - stacktrace/1,nested_stacktrace/1,raise/1]). + stacktrace/1,nested_stacktrace/1,raise/1, + no_return_in_try_block/1]). -include_lib("common_test/include/ct.hrl"). @@ -43,7 +44,8 @@ groups() -> nested_after,nested_horrid,last_call_optimization, bool,plain_catch_coverage,andalso_orelse,get_in_try, hockey,handle_info,catch_in_catch,grab_bag, - stacktrace,nested_stacktrace,raise]}]. + stacktrace,nested_stacktrace,raise, + no_return_in_try_block]}]. init_per_suite(Config) -> @@ -1287,5 +1289,26 @@ do_test_raise_4(Expr) -> erlang:raise(exit, {exception,C,E,Stk}, Stk) end. +no_return_in_try_block(Config) when is_list(Config) -> + 1.0 = no_return_in_try_block_1(0), + 1.0 = no_return_in_try_block_1(0.0), + + gurka = no_return_in_try_block_1(gurka), + [] = no_return_in_try_block_1([]), + + ok. + +no_return_in_try_block_1(H) -> + try + Float = if + is_number(H) -> float(H); + true -> no_return() + end, + Float + 1 + catch + throw:no_return -> H + end. + +no_return() -> throw(no_return). id(I) -> I. diff --git a/lib/compiler/test/warnings_SUITE.erl b/lib/compiler/test/warnings_SUITE.erl index 1c23eba06d..70b7100451 100644 --- a/lib/compiler/test/warnings_SUITE.erl +++ b/lib/compiler/test/warnings_SUITE.erl @@ -240,19 +240,7 @@ guard(Config) when is_list(Config) -> {4,sys_core_fold,nomatch_guard}, {6,sys_core_fold,no_clause_match}, {6,sys_core_fold,nomatch_guard}, - {6,sys_core_fold,{eval_failure,badarg}}, - {8,sys_core_fold,no_clause_match}, - {8,sys_core_fold,nomatch_guard}, - {8,sys_core_fold,{eval_failure,badarg}}, - {9,sys_core_fold,no_clause_match}, - {9,sys_core_fold,nomatch_guard}, - {9,sys_core_fold,{eval_failure,badarg}}, - {10,sys_core_fold,no_clause_match}, - {10,sys_core_fold,nomatch_guard}, - {10,sys_core_fold,{eval_failure,badarg}}, - {11,sys_core_fold,no_clause_match}, - {11,sys_core_fold,nomatch_guard}, - {11,sys_core_fold,{eval_failure,badarg}} + {6,sys_core_fold,{eval_failure,badarg}} ]}}], [] = run(Config, Ts), @@ -523,25 +511,43 @@ bin_opt_info(Config) when is_list(Config) -> <<>> -> ok end. + %% We use a tail in a BIF instruction, remote call, function + %% return, and an optimizable tail call for better coverage. + t2(<<A,B,T/bytes>>) -> + if + A > B -> t2(T); + A =< B -> T + end; + t2(<<_,T/bytes>>) when byte_size(T) < 4 -> + foo; t2(<<_,T/bytes>>) -> - split_binary(T, 4). + split_binary(T, 4). ">>, - Ts1 = [{bsm1, - Code, - [bin_opt_info], - {warnings, - [{4,sys_core_bsm,orig_bin_var_used_in_guard}, - {5,beam_bsm,{no_bin_opt,{{t1,1},no_suitable_bs_start_match}}}, - {9,beam_bsm,{no_bin_opt, - {binary_used_in,{extfunc,erlang,split_binary,2}}}} ]}}], - [] = run(Config, Ts1), + + Ws = (catch run_test(Config, Code, [bin_opt_info])), + + %% This is an inexact match since the pass reports exact instructions as + %% part of the warnings, which may include annotations that vary from run + %% to run. + {warnings, + [{5,beam_ssa_bsm,{unsuitable_call, + {{b_local,{b_literal,t1},1}, + {used_before_match, + {b_set,_,_,{bif,byte_size},[_]}}}}}, + {5,beam_ssa_bsm,{binary_created,_,_}}, + {11,beam_ssa_bsm,{binary_created,_,_}}, %% A =< B -> T + {13,beam_ssa_bsm,context_reused}, %% A > B -> t2(T); + {16,beam_ssa_bsm,{binary_created,_,_}}, %% when byte_size(T) < 4 -> + {19,beam_ssa_bsm,{remote_call, + {b_remote, + {b_literal,erlang}, + {b_literal,split_binary},2}}}, + {19,beam_ssa_bsm,{binary_created,_,_}} %% split_binary(T, 4) + ]} = Ws, %% For coverage: don't give the bin_opt_info option. - Ts2 = [{bsm2, - Code, - [], - []}], - [] = run(Config, Ts2), + [] = (catch run_test(Config, Code, [])), + ok. bin_construction(Config) when is_list(Config) -> @@ -747,7 +753,7 @@ maps_bin_opt_info(Config) when is_list(Config) -> M. ">>, [bin_opt_info], - {warnings,[{2,beam_bsm,bin_opt}]}}], + {warnings,[{3,beam_ssa_bsm,context_reused}]}}], [] = run(Config, Ts), ok. @@ -984,7 +990,6 @@ run(Config, Tests) -> end, lists:foldl(F, [], Tests). - %% Compiles a test module and returns the list of errors and warnings. run_test(Conf, Test0, Warnings) -> diff --git a/lib/configure.in.src b/lib/configure.in.src deleted file mode 100644 index d507a5c0dd..0000000000 --- a/lib/configure.in.src +++ /dev/null @@ -1,62 +0,0 @@ -dnl -dnl %CopyrightBegin% -dnl -dnl Copyright Ericsson AB 1999-2016. All Rights Reserved. -dnl -dnl Licensed under the Apache License, Version 2.0 (the "License"); -dnl you may not use this file except in compliance with the License. -dnl You may obtain a copy of the License at -dnl -dnl http://www.apache.org/licenses/LICENSE-2.0 -dnl -dnl Unless required by applicable law or agreed to in writing, software -dnl distributed under the License is distributed on an "AS IS" BASIS, -dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -dnl See the License for the specific language governing permissions and -dnl limitations under the License. -dnl -dnl %CopyrightEnd% -dnl - -dnl Turn off caching -define([AC_CACHE_LOAD], )dnl -define([AC_CACHE_SAVE], )dnl - -dnl Process this file with autoconf to produce a configure script. -AC_INIT - -dnl -dnl This is just to run configure in all applications that need it. -dnl - -if test -z "$ERL_TOP" || test ! -d $ERL_TOP ; then - AC_MSG_ERROR(You need to set the environment variable ERL_TOP!) -fi -erl_top=${ERL_TOP} -AC_CONFIG_AUX_DIRS($erl_top/erts/autoconf) - -AC_ARG_ENABLE(bootstrap-only, -[ --enable-bootstrap-only enable bootstrap only configuration], -[ if test "X$enableval" = "Xyes"; then - bootstrap_only=yes - else - bootstrap_only=no - fi -], -bootstrap_only=no) - -# Multiple versions of autoconf generates code that -# don't work on all platforms (e.g. SunOS 5.8) if -# sub directories are soft links. Internally at Ericsson -# some OTP application directories are soft links. -# An added "/." solves this problem. - -@BOOTSTRAP_CONFIGURE_APPS@ - -if test $bootstrap_only = no; then - -@NON_BOOTSTRAP_CONFIGURE_APPS@ - -fi - -AC_OUTPUT diff --git a/lib/crypto/c_src/Makefile.in b/lib/crypto/c_src/Makefile.in index 31124ba477..b6a65d7488 100644 --- a/lib/crypto/c_src/Makefile.in +++ b/lib/crypto/c_src/Makefile.in @@ -19,7 +19,6 @@ # include $(ERL_TOP)/make/target.mk include $(ERL_TOP)/make/$(TARGET)/otp.mk -include $(ERL_TOP)/make/$(TARGET)/otp_ded.mk # ---------------------------------------------------- # Application version @@ -31,23 +30,20 @@ VSN=$(CRYPTO_VSN) # The following variables differ between systems. # Set by configure. # ---------------------------------------------------- -CC = $(DED_CC) -LD = $(DED_LD) +CC = @DED_CC@ +LD = @DED_LD@ SHELL = /bin/sh -LIBS = $(DED_LIBS) -LDFLAGS += $(DED_LDFLAGS) -CFLAGS = $(DED_CFLAGS) +LIBS = @DED_LIBS@ +LDFLAGS += @DED_LDFLAGS@ +CFLAGS = @DED_CFLAGS@ @SSL_FLAGS@ # From erts/configure SSL_LIBDIR = @SSL_LIBDIR@ SSL_INCLUDE = @SSL_INCLUDE@ SSL_CRYPTO_LIBNAME = @SSL_CRYPTO_LIBNAME@ SSL_SSL_LIBNAME = @SSL_SSL_LIBNAME@ -SSL_FLAGS = @SSL_FLAGS@ - -INCLUDES = $(SSL_INCLUDE) $(DED_INCLUDES) -CFLAGS += $(SSL_FLAGS) +INCLUDES = $(SSL_INCLUDE) @DED_INCLUDE@ ifeq ($(TYPE),debug) TYPEMARKER = .debug @@ -70,7 +66,38 @@ RELSYSDIR = $(RELEASE_PATH)/lib/crypto-$(VSN) # ---------------------------------------------------- # Misc Macros # ---------------------------------------------------- -CRYPTO_OBJS = $(OBJDIR)/crypto$(TYPEMARKER).o + +PRIVDIR = ../priv +OBJDIR = $(PRIVDIR)/obj/$(TARGET) +LIBDIR = $(PRIVDIR)/lib/$(TARGET) + +CRYPTO_OBJS = $(OBJDIR)/crypto$(TYPEMARKER).o \ + $(OBJDIR)/aead$(TYPEMARKER).o \ + $(OBJDIR)/aes$(TYPEMARKER).o \ + $(OBJDIR)/algorithms$(TYPEMARKER).o \ + $(OBJDIR)/api_ng$(TYPEMARKER).o \ + $(OBJDIR)/atoms$(TYPEMARKER).o \ + $(OBJDIR)/bn$(TYPEMARKER).o \ + $(OBJDIR)/cipher$(TYPEMARKER).o \ + $(OBJDIR)/cmac$(TYPEMARKER).o \ + $(OBJDIR)/dh$(TYPEMARKER).o \ + $(OBJDIR)/digest$(TYPEMARKER).o \ + $(OBJDIR)/dss$(TYPEMARKER).o \ + $(OBJDIR)/ec$(TYPEMARKER).o \ + $(OBJDIR)/ecdh$(TYPEMARKER).o \ + $(OBJDIR)/eddsa$(TYPEMARKER).o \ + $(OBJDIR)/engine$(TYPEMARKER).o \ + $(OBJDIR)/evp$(TYPEMARKER).o \ + $(OBJDIR)/fips$(TYPEMARKER).o \ + $(OBJDIR)/hash$(TYPEMARKER).o \ + $(OBJDIR)/hmac$(TYPEMARKER).o \ + $(OBJDIR)/info$(TYPEMARKER).o \ + $(OBJDIR)/math$(TYPEMARKER).o \ + $(OBJDIR)/pkey$(TYPEMARKER).o \ + $(OBJDIR)/poly1305$(TYPEMARKER).o \ + $(OBJDIR)/rand$(TYPEMARKER).o \ + $(OBJDIR)/rsa$(TYPEMARKER).o \ + $(OBJDIR)/srp$(TYPEMARKER).o CALLBACK_OBJS = $(OBJDIR)/crypto_callback$(TYPEMARKER).o NIF_MAKEFILE = $(PRIVDIR)/Makefile CRYPTO_STATIC_OBJS = $(OBJDIR)/crypto_static$(TYPEMARKER).o\ @@ -80,19 +107,10 @@ NIF_ARCHIVE = $(LIBDIR)/crypto$(TYPEMARKER).a TEST_ENGINE_OBJS = $(OBJDIR)/otp_test_engine$(TYPEMARKER).o -ifeq ($(findstring win32,$(TARGET)), win32) -NIF_LIB = $(LIBDIR)/crypto$(TYPEMARKER).dll -CALLBACK_LIB = $(LIBDIR)/crypto_callback$(TYPEMARKER).dll -TEST_ENGINE_LIB = $(LIBDIR)/otp_test_engine$(TYPEMARKER).dll -else -NIF_LIB = $(LIBDIR)/crypto$(TYPEMARKER).so -CALLBACK_LIB = $(LIBDIR)/crypto_callback$(TYPEMARKER).so -TEST_ENGINE_LIB = $(LIBDIR)/otp_test_engine$(TYPEMARKER).so -endif +NIF_LIB = $(LIBDIR)/crypto$(TYPEMARKER).@DED_EXT@ +CALLBACK_LIB = $(LIBDIR)/crypto_callback$(TYPEMARKER).@DED_EXT@ +TEST_ENGINE_LIB = $(LIBDIR)/otp_test_engine$(TYPEMARKER).@DED_EXT@ -ifeq ($(HOST_OS),) -HOST_OS := $(shell $(ERL_TOP)/erts/autoconf/config.guess) -endif DYNAMIC_CRYPTO_LIB=@SSL_DYNAMIC_ONLY@ ifeq ($(DYNAMIC_CRYPTO_LIB),yes) @@ -125,7 +143,7 @@ RANLIB=true endif ALL_CFLAGS = $(TYPE_FLAGS) $(EXTRA_FLAGS) $(INCLUDES) -ALL_STATIC_CFLAGS = $(DED_STATIC_CFLAGS) $(INCLUDES) +ALL_STATIC_CFLAGS = @DED_STATIC_CFLAGS@ $(INCLUDES) # ---------------------------------------------------- # Targets @@ -180,32 +198,21 @@ $(LIBDIR)/crypto_callback$(TYPEMARKER).dll: $(CALLBACK_OBJS) endif -clean: -ifeq ($(findstring win32,$(TARGET)), win32) - rm -f $(LIBDIR)/crypto.dll - rm -f $(LIBDIR)/crypto.debug.dll - rm -f $(LIBDIR)/crypto_callback.dll - rm -f $(LIBDIR)/crypto_callback.debug.dll - rm -f $(LIBDIR)/otp_test_engine.dll -else - rm -f $(LIBDIR)/crypto.so - rm -f $(LIBDIR)/crypto.debug.so - rm -f $(LIBDIR)/crypto.valgrind.so - rm -f $(LIBDIR)/crypto_callback.so - rm -f $(LIBDIR)/crypto_callback.debug.so - rm -f $(LIBDIR)/crypto_callback.valgrind.so - rm -f $(LIBDIR)/otp_test_engine.so -endif - rm -f $(OBJDIR)/crypto.o - rm -f $(OBJDIR)/crypto_static.o - rm -f $(OBJDIR)/crypto.debug.o - rm -f $(OBJDIR)/crypto_static.debug.o - rm -f $(OBJDIR)/crypto.valgrind.o - rm -f $(OBJDIR)/crypto_static.valgrind.o - rm -f $(OBJDIR)/crypto_callback.o - rm -f $(OBJDIR)/crypto_callback.debug.o - rm -f $(OBJDIR)/crypto_callback.valgrind.o - rm -f $(OBJDIR)/otp_test_engine.o +CLEAN_OBJS_RAW = $(CRYPTO_OBJS) $(CALLBACK_OBJS) $(CRYPTO_STATIC_OBJS) $(TEST_ENGINE_OBJS) +CLEAN_OBJS_O = $(patsubst %.debug.o,%.o,$(CLEAN_OBJS_RAW:.valgrind.o=.o)) + +CLEAN_LIBS_RAW = $(NIF_LIB) $(CALLBACK_LIB) $(TEST_ENGINE_LIB) +CLEAN_LIBS_SO = $(patsubst %.debug.@DED_EXT@,%.@DED_EXT@,$(CLEAN_LIBS_RAW:.valgrind.@DED_EXT@=.@DED_EXT@)) + +clean_dynamic_libs: + rm -f $(CLEAN_LIBS_SO) + rm -f $(foreach T,.valgrind.@DED_EXT@ .debug.@DED_EXT@,$(CLEAN_LIBS_SO:.@DED_EXT@=$T)) + +clean_objs: + rm -f $(CLEAN_OBJS_O) + rm -f $(foreach T,.valgrind.o .debug.o,$(CLEAN_OBJS_O:.o=$T)) + +clean: clean_objs clean_dynamic_libs rm -f core *~ docs: diff --git a/lib/crypto/c_src/aead.c b/lib/crypto/c_src/aead.c new file mode 100644 index 0000000000..ab0e609130 --- /dev/null +++ b/lib/crypto/c_src/aead.c @@ -0,0 +1,182 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "aead.h" +#include "aes.h" +#include "cipher.h" + + + +ERL_NIF_TERM aead_cipher(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* + (Type,Key,Iv,AAD,In,TagLen,true) + (Type,Key,Iv,AAD,In,Tag,false) + */ +#if defined(HAVE_AEAD) + const struct cipher_type_t *cipherp; + EVP_CIPHER_CTX *ctx = NULL; + const EVP_CIPHER *cipher = NULL; + ErlNifBinary key, iv, aad, in, tag; + unsigned int tag_len; + unsigned char *outp, *tagp, *tag_data; + ERL_NIF_TERM type, out, out_tag, ret, encflg_arg; + int len, encflg; + + encflg_arg = argv[6]; + + /* Fetch the flag telling if we are going to encrypt (=true) or decrypt (=false) */ + if (encflg_arg == atom_true) + encflg = 1; + else if (encflg_arg == atom_false) + encflg = 0; + else if (encflg_arg == atom_undefined) + /* For compat funcs in crypto.erl */ + encflg = -1; + else + { + ret = EXCP_BADARG(env, "Bad enc flag"); + goto done; + } + + type = argv[0]; + + if (!enif_is_atom(env, type)) + {ret = EXCP_BADARG(env, "non-atom cipher type"); goto done;} + if (!enif_inspect_iolist_as_binary(env, argv[1], &key)) + {ret = EXCP_BADARG(env, "non-binary key"); goto done;} + if (!enif_inspect_iolist_as_binary(env, argv[2], &iv)) + {ret = EXCP_BADARG(env, "non-binary iv"); goto done;} + if (!enif_inspect_iolist_as_binary(env, argv[3], &in)) + {ret = EXCP_BADARG(env, "non-binary text"); goto done;} + if (!enif_inspect_iolist_as_binary(env, argv[4], &aad)) + {ret = EXCP_BADARG(env, "non-binary AAD"); goto done;} + + if (encflg) { + if (!enif_get_uint(env, argv[5], &tag_len)) + {ret = EXCP_BADARG(env, "Bad Tag length"); goto done;} + tag_data = NULL; + } else { + if (!enif_inspect_iolist_as_binary(env, argv[5], &tag)) + {ret = EXCP_BADARG(env, "non-binary Tag"); goto done;} + tag_len = tag.size; + tag_data = tag.data; + } + + if (tag_len > INT_MAX + || key.size > INT_MAX + || iv.size > INT_MAX + || in.size > INT_MAX + || aad.size > INT_MAX) + {ret = EXCP_BADARG(env, "binary too long"); goto done;} + + if ((cipherp = get_cipher_type(type, key.size)) == NULL) + {ret = EXCP_BADARG(env, "Unknown cipher"); goto done;} + if (cipherp->flags & NON_EVP_CIPHER) + {ret = EXCP_BADARG(env, "Bad cipher"); goto done;} + if (! (cipherp->flags & AEAD_CIPHER) ) + {ret = EXCP_BADARG(env, "Not aead cipher"); goto done;} + if ((cipher = cipherp->cipher.p) == NULL) + {ret = EXCP_NOTSUP(env, "Cipher not supported in this libcrypto version"); goto done;} + +#if defined(HAVE_GCM_EVP_DECRYPT_BUG) + if ( !encflg && (cipherp->flags & GCM_MODE)) + return aes_gcm_decrypt_NO_EVP(env, argc, argv); +#endif + + if ((ctx = EVP_CIPHER_CTX_new()) == NULL) + {ret = EXCP_ERROR(env, "Can't allocate ctx"); goto done;} + + if (EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, encflg) != 1) + {ret = EXCP_ERROR(env, "CipherInit failed"); goto done;} + if (EVP_CIPHER_CTX_ctrl(ctx, cipherp->extra.aead.ctx_ctrl_set_ivlen, (int)iv.size, NULL) != 1) + {ret = EXCP_BADARG(env, "Bad IV length"); goto done;} + +#if defined(HAVE_CCM) + if (cipherp->flags & CCM_MODE) { + if (EVP_CIPHER_CTX_ctrl(ctx, cipherp->extra.aead.ctx_ctrl_set_tag, (int)tag_len, tag_data) != 1) + {ret = EXCP_BADARG(env, "Can't set tag"); goto done;} + if (EVP_CipherInit_ex(ctx, NULL, NULL, key.data, iv.data, -1) != 1) + {ret = EXCP_BADARG(env, "Can't set key or iv"); goto done;} + if (EVP_CipherUpdate(ctx, NULL, &len, NULL, (int)in.size) != 1) + {ret = EXCP_BADARG(env, "Can't set text size"); goto done;} + } else +#endif + { + if (EVP_CipherInit_ex(ctx, NULL, NULL, key.data, iv.data, -1) != 1) + {ret = EXCP_BADARG(env, "Can't set key or iv"); goto done;} + } + + if (EVP_CipherUpdate(ctx, NULL, &len, aad.data, (int)aad.size) != 1) + {ret = EXCP_BADARG(env, "Can't set AAD"); goto done;} + + if ((outp = enif_make_new_binary(env, in.size, &out)) == NULL) + {ret = EXCP_ERROR(env, "Can't make 'Out' binary"); goto done;} + + if (EVP_CipherUpdate(ctx, outp, &len, in.data, (int)in.size) != 1) + { + if (encflg) + ret = EXCP_BADARG(env, "Can't set in-text"); + else + /* Decrypt error */ + ret = atom_error; + goto done; + } + + if (encflg) + { + if (EVP_CipherFinal_ex(ctx, outp/*+len*/, &len) != 1) + {ret = EXCP_ERROR(env, "Encrypt error"); goto done;} + + if ((tagp = enif_make_new_binary(env, tag_len, &out_tag)) == NULL) + {ret = EXCP_ERROR(env, "Can't make 'Out' binary"); goto done;} + + if (EVP_CIPHER_CTX_ctrl(ctx, cipherp->extra.aead.ctx_ctrl_get_tag, (int)tag_len, tagp) != 1) + {ret = EXCP_ERROR(env, "Can't get Tag"); goto done;} + + ret = enif_make_tuple2(env, out, out_tag); + } + else + { +#if defined(HAVE_GCM) + if (cipherp->flags & GCM_MODE) { + if (EVP_CIPHER_CTX_ctrl(ctx, cipherp->extra.aead.ctx_ctrl_set_tag, (int)tag_len, tag.data) != 1) + /* Decrypt error */ + {ret = atom_error; goto done;} + if (EVP_DecryptFinal_ex(ctx, outp+len, &len) != 1) + /* Decrypt error */ + {ret = atom_error; goto done;} + } +#endif + ret = out; + } + + CONSUME_REDS(env, in); + +done: + if (ctx) + EVP_CIPHER_CTX_free(ctx); + return ret; + +#else + return EXCP_NOTSUP(env, "Unsupported Cipher"); +#endif +} + + diff --git a/lib/crypto/c_src/aead.h b/lib/crypto/c_src/aead.h new file mode 100644 index 0000000000..2ec7a8a930 --- /dev/null +++ b/lib/crypto/c_src/aead.h @@ -0,0 +1,28 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_AEAD_H__ +#define E_AEAD_H__ 1 + +#include "common.h" + +ERL_NIF_TERM aead_cipher(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_AEAD_H__ */ diff --git a/lib/crypto/c_src/aes.c b/lib/crypto/c_src/aes.c new file mode 100644 index 0000000000..4b01e629f9 --- /dev/null +++ b/lib/crypto/c_src/aes.c @@ -0,0 +1,302 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "aes.h" +#include "cipher.h" + +ERL_NIF_TERM aes_cfb_8_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Key, IVec, Data, IsEncrypt) */ + ErlNifBinary key, ivec, text; + AES_KEY aes_key; + unsigned char ivec_clone[16]; /* writable copy */ + int new_ivlen = 0; + ERL_NIF_TERM ret; + unsigned char *outp; + + CHECK_NO_FIPS_MODE(); + + ASSERT(argc == 4); + + if (!enif_inspect_iolist_as_binary(env, argv[0], &key)) + goto bad_arg; + if (key.size != 16 && key.size != 24 && key.size != 32) + goto bad_arg; + if (!enif_inspect_binary(env, argv[1], &ivec)) + goto bad_arg; + if (ivec.size != 16) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[2], &text)) + goto bad_arg; + + memcpy(ivec_clone, ivec.data, 16); + + /* NOTE: This function returns 0 on success unlike most OpenSSL functions */ + if (AES_set_encrypt_key(key.data, (int)key.size * 8, &aes_key) != 0) + goto err; + if ((outp = enif_make_new_binary(env, text.size, &ret)) == NULL) + goto err; + AES_cfb8_encrypt((unsigned char *) text.data, + outp, + text.size, &aes_key, ivec_clone, &new_ivlen, + (argv[3] == atom_true)); + CONSUME_REDS(env,text); + return ret; + + bad_arg: + err: + return enif_make_badarg(env); +} + +ERL_NIF_TERM aes_cfb_128_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Key, IVec, Data, IsEncrypt) */ + ErlNifBinary key, ivec, text; + AES_KEY aes_key; + unsigned char ivec_clone[16]; /* writable copy */ + int new_ivlen = 0; + ERL_NIF_TERM ret; + unsigned char *outp; + + ASSERT(argc == 4); + + if (!enif_inspect_iolist_as_binary(env, argv[0], &key)) + goto bad_arg; + if (key.size != 16 && key.size != 24 && key.size != 32) + goto bad_arg; + if (!enif_inspect_binary(env, argv[1], &ivec)) + goto bad_arg; + if (ivec.size != 16) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[2], &text)) + goto bad_arg; + + memcpy(ivec_clone, ivec.data, 16); + + /* NOTE: This function returns 0 on success unlike most OpenSSL functions */ + if (AES_set_encrypt_key(key.data, (int)key.size * 8, &aes_key) != 0) + goto err; + + if ((outp = enif_make_new_binary(env, text.size, &ret)) == NULL) + goto err; + AES_cfb128_encrypt((unsigned char *) text.data, + outp, + text.size, &aes_key, ivec_clone, &new_ivlen, + (argv[3] == atom_true)); + CONSUME_REDS(env,text); + return ret; + + bad_arg: + err: + return enif_make_badarg(env); +} + +ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Key, IVec, Data, IsEncrypt) */ +#ifdef HAVE_AES_IGE + ErlNifBinary key_bin, ivec_bin, data_bin; + AES_KEY aes_key; + unsigned char ivec[32]; + int type; + unsigned char* ret_ptr; + ERL_NIF_TERM ret; + + CHECK_NO_FIPS_MODE(); + + ASSERT(argc == 4); + + if (!enif_inspect_iolist_as_binary(env, argv[0], &key_bin)) + goto bad_arg; + if (key_bin.size != 16 && key_bin.size != 32) + goto bad_arg; + if (!enif_inspect_binary(env, argv[1], &ivec_bin)) + goto bad_arg; + if (ivec_bin.size != 32) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[2], &data_bin)) + goto bad_arg; + if (data_bin.size % 16 != 0) + goto bad_arg; + + if (argv[3] == atom_true) { + type = AES_ENCRYPT; + /* NOTE: This function returns 0 on success unlike most OpenSSL functions */ + if (AES_set_encrypt_key(key_bin.data, (int)key_bin.size * 8, &aes_key) != 0) + goto err; + } + else { + type = AES_DECRYPT; + /* NOTE: This function returns 0 on success unlike most OpenSSL functions */ + if (AES_set_decrypt_key(key_bin.data, (int)key_bin.size * 8, &aes_key) != 0) + goto err; + } + + if ((ret_ptr = enif_make_new_binary(env, data_bin.size, &ret)) == NULL) + goto err; + + memcpy(ivec, ivec_bin.data, 32); /* writable copy */ + + AES_ige_encrypt(data_bin.data, ret_ptr, data_bin.size, &aes_key, ivec, type); + + CONSUME_REDS(env,data_bin); + return ret; + + bad_arg: + err: + return enif_make_badarg(env); + +#else + return atom_notsup; +#endif +} + + +#if !defined(HAVE_EVP_AES_CTR) +ERL_NIF_TERM aes_ctr_stream_encrypt_compat(ErlNifEnv* env, const ERL_NIF_TERM state_arg, const ERL_NIF_TERM data_arg) +{/* ({Key, IVec, ECount, Num}, Data) */ + ErlNifBinary key_bin, ivec_bin, text_bin, ecount_bin; + AES_KEY aes_key; + unsigned int num; + ERL_NIF_TERM ret, num2_term, cipher_term, ivec2_term, ecount2_term, new_state_term; + int state_arity; + const ERL_NIF_TERM *state_term; + unsigned char * ivec2_buf; + unsigned char * ecount2_buf; + unsigned char *outp; + + if (!enif_get_tuple(env, state_arg, &state_arity, &state_term)) + goto bad_arg; + if (state_arity != 4) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, state_term[0], &key_bin)) + goto bad_arg; + if (key_bin.size > INT_MAX / 8) + goto bad_arg; + if (!enif_inspect_binary(env, state_term[1], &ivec_bin)) + goto bad_arg; + if (ivec_bin.size != 16) + goto bad_arg; + if (!enif_inspect_binary(env, state_term[2], &ecount_bin)) + goto bad_arg; + if (ecount_bin.size != AES_BLOCK_SIZE) + goto bad_arg; + if (!enif_get_uint(env, state_term[3], &num)) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, data_arg, &text_bin)) + goto bad_arg; + + /* NOTE: This function returns 0 on success unlike most OpenSSL functions */ + if (AES_set_encrypt_key(key_bin.data, (int)key_bin.size * 8, &aes_key) != 0) + goto bad_arg; + + if ((ivec2_buf = enif_make_new_binary(env, ivec_bin.size, &ivec2_term)) == NULL) + goto err; + if ((ecount2_buf = enif_make_new_binary(env, ecount_bin.size, &ecount2_term)) == NULL) + goto err; + + memcpy(ivec2_buf, ivec_bin.data, 16); + memcpy(ecount2_buf, ecount_bin.data, ecount_bin.size); + + if ((outp = enif_make_new_binary(env, text_bin.size, &cipher_term)) == NULL) + goto err; + + AES_ctr128_encrypt((unsigned char *) text_bin.data, + outp, + text_bin.size, &aes_key, ivec2_buf, ecount2_buf, &num); + + num2_term = enif_make_uint(env, num); + new_state_term = enif_make_tuple4(env, state_term[0], ivec2_term, ecount2_term, num2_term); + ret = enif_make_tuple2(env, new_state_term, cipher_term); + CONSUME_REDS(env,text_bin); + return ret; + + bad_arg: + err: + return enif_make_badarg(env); +} +#endif /* !HAVE_EVP_AES_CTR */ + +#ifdef HAVE_GCM_EVP_DECRYPT_BUG +ERL_NIF_TERM aes_gcm_decrypt_NO_EVP(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type,Key,Iv,AAD,In,Tag) */ + GCM128_CONTEXT *ctx = NULL; + ErlNifBinary key, iv, aad, in, tag; + AES_KEY aes_key; + unsigned char *outp; + ERL_NIF_TERM out, ret; + + ASSERT(argc == 6); + + if (!enif_inspect_iolist_as_binary(env, argv[1], &key)) + goto bad_arg; + if (key.size > INT_MAX / 8) + goto bad_arg; + if (!enif_inspect_binary(env, argv[2], &iv)) + goto bad_arg; + if (iv.size == 0) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[3], &aad)) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[4], &in)) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[5], &tag)) + goto bad_arg; + + /* NOTE: This function returns 0 on success unlike most OpenSSL functions */ + if (AES_set_encrypt_key(key.data, (int)key.size * 8, &aes_key) != 0) + goto bad_arg; + + if ((ctx = CRYPTO_gcm128_new(&aes_key, (block128_f)AES_encrypt)) == NULL) + goto err; + + CRYPTO_gcm128_setiv(ctx, iv.data, iv.size); + + /* NOTE: This function returns 0 on success unlike most OpenSSL functions */ + if (CRYPTO_gcm128_aad(ctx, aad.data, aad.size) != 0) + goto err; + + if ((outp = enif_make_new_binary(env, in.size, &out)) == NULL) + goto err; + + /* NOTE: This function returns 0 on success unlike most OpenSSL functions */ + if (CRYPTO_gcm128_decrypt(ctx, in.data, outp, in.size) != 0) + goto err; + + /* calculate and check the tag */ + /* NOTE: This function returns 0 on success unlike most OpenSSL functions */ + if (CRYPTO_gcm128_finish(ctx, tag.data, tag.size) != 0) + goto err; + + CONSUME_REDS(env, in); + ret = out; + goto done; + + bad_arg: + ret = enif_make_badarg(env); + goto done; + + err: + ret = atom_error; + + done: + if (ctx) + CRYPTO_gcm128_release(ctx); + return ret; +} +#endif /* HAVE_GCM_EVP_DECRYPT_BUG */ + diff --git a/lib/crypto/c_src/aes.h b/lib/crypto/c_src/aes.h new file mode 100644 index 0000000000..c0b2b91f8d --- /dev/null +++ b/lib/crypto/c_src/aes.h @@ -0,0 +1,38 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_AES_H__ +#define E_AES_H__ 1 + +#include "common.h" + +ERL_NIF_TERM aes_cfb_8_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM aes_cfb_128_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#if !defined(HAVE_EVP_AES_CTR) +ERL_NIF_TERM aes_ctr_stream_encrypt_compat(ErlNifEnv* env, const ERL_NIF_TERM state_arg, const ERL_NIF_TERM data_arg); +#endif + +#ifdef HAVE_GCM_EVP_DECRYPT_BUG +ERL_NIF_TERM aes_gcm_decrypt_NO_EVP(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +#endif + +#endif /* E_AES_H__ */ diff --git a/lib/crypto/c_src/algorithms.c b/lib/crypto/c_src/algorithms.c new file mode 100644 index 0000000000..75cddeb1e9 --- /dev/null +++ b/lib/crypto/c_src/algorithms.c @@ -0,0 +1,320 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "algorithms.h" +#include "cipher.h" + +static unsigned int algo_hash_cnt, algo_hash_fips_cnt; +static ERL_NIF_TERM algo_hash[14]; /* increase when extending the list */ +static unsigned int algo_pubkey_cnt, algo_pubkey_fips_cnt; +static ERL_NIF_TERM algo_pubkey[12]; /* increase when extending the list */ +static unsigned int algo_mac_cnt, algo_mac_fips_cnt; +static ERL_NIF_TERM algo_mac[3]; /* increase when extending the list */ +static unsigned int algo_curve_cnt, algo_curve_fips_cnt; +static ERL_NIF_TERM algo_curve[89]; /* increase when extending the list */ +static unsigned int algo_rsa_opts_cnt, algo_rsa_opts_fips_cnt; +static ERL_NIF_TERM algo_rsa_opts[11]; /* increase when extending the list */ + +void init_algorithms_types(ErlNifEnv* env) +{ + // Validated algorithms first + algo_hash_cnt = 0; + algo_hash[algo_hash_cnt++] = atom_sha; +#ifdef HAVE_SHA224 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha224"); +#endif +#ifdef HAVE_SHA256 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha256"); +#endif +#ifdef HAVE_SHA384 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha384"); +#endif +#ifdef HAVE_SHA512 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha512"); +#endif +#ifdef HAVE_SHA3_224 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha3_224"); +#endif +#ifdef HAVE_SHA3_256 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha3_256"); +#endif +#ifdef HAVE_SHA3_384 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha3_384"); +#endif +#ifdef HAVE_SHA3_512 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha3_512"); +#endif +#ifdef HAVE_BLAKE2 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "blake2b"); + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "blake2s"); +#endif + + // Non-validated algorithms follow + algo_hash_fips_cnt = algo_hash_cnt; +#ifdef HAVE_MD4 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "md4"); +#endif +#ifdef HAVE_MD5 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "md5"); +#endif +#ifdef HAVE_RIPEMD160 + algo_hash[algo_hash_cnt++] = enif_make_atom(env, "ripemd160"); +#endif + + algo_pubkey_cnt = 0; + algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "rsa"); +#ifdef HAVE_DSA + algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "dss"); +#endif +#ifdef HAVE_DH + algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "dh"); +#endif +#if defined(HAVE_EC) +#if !defined(OPENSSL_NO_EC2M) + algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "ec_gf2m"); +#endif + algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "ecdsa"); + algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "ecdh"); +#endif + // Non-validated algorithms follow + algo_pubkey_fips_cnt = algo_pubkey_cnt; + // Don't know if Edward curves are fips validated +#if defined(HAVE_EDDSA) + algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "eddsa"); +#endif + algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "srp"); + + + // Validated algorithms first + algo_mac_cnt = 0; + algo_mac[algo_mac_cnt++] = enif_make_atom(env,"hmac"); +#ifdef HAVE_CMAC + algo_mac[algo_mac_cnt++] = enif_make_atom(env,"cmac"); +#endif +#ifdef HAVE_POLY1305 + algo_mac[algo_mac_cnt++] = enif_make_atom(env,"poly1305"); +#endif + // Non-validated algorithms follow + algo_mac_fips_cnt = algo_mac_cnt; + + // Validated algorithms first + algo_curve_cnt = 0; +#if defined(HAVE_EC) + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp160k1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp160r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp160r2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp192r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp192k1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp224k1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp224r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp256k1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp256r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp384r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp521r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime192v1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime192v2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime192v3"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime239v1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime239v2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime239v3"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime256v1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls7"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls9"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls12"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP160r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP160t1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP192r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP192t1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP224r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP224t1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP256r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP256t1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP320r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP320t1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP384r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP384t1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP512r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP512t1"); +#if !defined(OPENSSL_NO_EC2M) + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect163k1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect163r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect163r2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect193r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect193r2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect233k1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect233r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect239k1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect283k1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect283r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect409k1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect409r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect571k1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect571r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb163v1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb163v2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb163v3"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb176v1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb191v1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb191v2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb191v3"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb208w1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb239v1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb239v2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb239v3"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb272w1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb304w1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb359v1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb368w1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb431r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls3"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls5"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls10"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls11"); +#endif +#endif + // Non-validated algorithms follow + algo_curve_fips_cnt = algo_curve_cnt; +#if defined(HAVE_EC) + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp112r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp112r2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp128r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp128r2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls6"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls8"); +#if !defined(OPENSSL_NO_EC2M) + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect113r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect113r2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect131r1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect131r2"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls1"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls4"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"ipsec3"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"ipsec4"); +#endif +#endif + //-- +#ifdef HAVE_EDDSA + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"ed25519"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"ed448"); +#endif +#ifdef HAVE_ED_CURVE_DH + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"x25519"); + algo_curve[algo_curve_cnt++] = enif_make_atom(env,"x448"); +#endif + + // Validated algorithms first + algo_rsa_opts_cnt = 0; +#ifdef HAS_EVP_PKEY_CTX +# ifdef HAVE_RSA_PKCS1_PSS_PADDING + algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_pkcs1_pss_padding"); + algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_pss_saltlen"); +# endif +# ifdef HAVE_RSA_MGF1_MD + algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_mgf1_md"); +# endif +# ifdef HAVE_RSA_OAEP_PADDING + algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_pkcs1_oaep_padding"); +# endif +# ifdef HAVE_RSA_OAEP_MD + algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_oaep_label"); + algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_oaep_md"); +# endif + algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"signature_md"); +#endif + algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_pkcs1_padding"); + algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_x931_padding"); +#ifdef HAVE_RSA_SSLV23_PADDING + algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_sslv23_padding"); +#endif + algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_no_padding"); + algo_rsa_opts_fips_cnt = algo_rsa_opts_cnt; + + + // Check that the max number of algos is updated + ASSERT(algo_hash_cnt <= sizeof(algo_hash)/sizeof(ERL_NIF_TERM)); + ASSERT(algo_pubkey_cnt <= sizeof(algo_pubkey)/sizeof(ERL_NIF_TERM)); + ASSERT(algo_mac_cnt <= sizeof(algo_mac)/sizeof(ERL_NIF_TERM)); + ASSERT(algo_curve_cnt <= sizeof(algo_curve)/sizeof(ERL_NIF_TERM)); + ASSERT(algo_rsa_opts_cnt <= sizeof(algo_rsa_opts)/sizeof(ERL_NIF_TERM)); +} + + +ERL_NIF_TERM hash_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + unsigned int cnt = +#ifdef FIPS_SUPPORT + FIPS_mode() ? algo_hash_fips_cnt : +#endif + algo_hash_cnt; + + return enif_make_list_from_array(env, algo_hash, cnt); +} + +ERL_NIF_TERM pubkey_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + unsigned int cnt = +#ifdef FIPS_SUPPORT + FIPS_mode() ? algo_pubkey_fips_cnt : +#endif + algo_pubkey_cnt; + + return enif_make_list_from_array(env, algo_pubkey, cnt); +} + + +ERL_NIF_TERM cipher_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + return cipher_types_as_list(env); /* Exclude old api ciphers */ +} + +ERL_NIF_TERM mac_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + unsigned int cnt = +#ifdef FIPS_SUPPORT + FIPS_mode() ? algo_mac_fips_cnt : +#endif + algo_mac_cnt; + + return enif_make_list_from_array(env, algo_mac, cnt); +} + + +ERL_NIF_TERM curve_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + unsigned int cnt = +#ifdef FIPS_SUPPORT + FIPS_mode() ? algo_curve_fips_cnt : +#endif + algo_curve_cnt; + + return enif_make_list_from_array(env, algo_curve, cnt); +} + + +ERL_NIF_TERM rsa_opts_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + unsigned int cnt = +#ifdef FIPS_SUPPORT + FIPS_mode() ? algo_rsa_opts_fips_cnt : +#endif + algo_rsa_opts_cnt; + + return enif_make_list_from_array(env, algo_rsa_opts, cnt); +} diff --git a/lib/crypto/c_src/algorithms.h b/lib/crypto/c_src/algorithms.h new file mode 100644 index 0000000000..4ad8b56db8 --- /dev/null +++ b/lib/crypto/c_src/algorithms.h @@ -0,0 +1,35 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_ALGORITHMS_H__ +#define E_ALGORITHMS_H__ 1 + +#include "common.h" + +void init_algorithms_types(ErlNifEnv* env); + +ERL_NIF_TERM hash_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM pubkey_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM cipher_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM mac_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM curve_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM rsa_opts_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_ALGORITHMS_H__ */ diff --git a/lib/crypto/c_src/api_ng.c b/lib/crypto/c_src/api_ng.c new file mode 100644 index 0000000000..3408ba1b88 --- /dev/null +++ b/lib/crypto/c_src/api_ng.c @@ -0,0 +1,569 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "api_ng.h" +#include "aes.h" +#include "cipher.h" + +/* + * A unified set of functions for encryption/decryption. + * + */ +ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM ng_crypto_one_time(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#ifdef HAVE_ECB_IVEC_BUG + /* <= 0.9.8l returns faulty ivec length */ +# define GET_IV_LEN(Ciph) ((Ciph)->flags & ECB_BUG_0_9_8L) ? 0 : EVP_CIPHER_iv_length((Ciph)->cipher.p) +#else +# define GET_IV_LEN(Ciph) EVP_CIPHER_iv_length((Ciph)->cipher.p) +#endif + +/*************************************************************************/ +/* Get the arguments for the initialization of the EVP_CIPHER_CTX. Check */ +/* them and initialize that context. */ +/*************************************************************************/ +static int get_init_args(ErlNifEnv* env, + struct evp_cipher_ctx *ctx_res, + const ERL_NIF_TERM cipher_arg, + const ERL_NIF_TERM key_arg, + const ERL_NIF_TERM ivec_arg, + const ERL_NIF_TERM encflg_arg, + const struct cipher_type_t **cipherp, + ERL_NIF_TERM *return_term) +{ + int ivec_len; + ErlNifBinary key_bin; + ErlNifBinary ivec_bin; + int encflg; + + ctx_res->ctx = NULL; /* For testing if *ctx should be freed after errors */ + + /* Fetch the flag telling if we are going to encrypt (=true) or decrypt (=false) */ + if (encflg_arg == atom_true) + encflg = 1; + else if (encflg_arg == atom_false) + encflg = 0; + else if (encflg_arg == atom_undefined) + /* For compat funcs in crypto.erl */ + encflg = -1; + else + { + *return_term = EXCP_BADARG(env, "Bad enc flag"); + goto err; + } + + /* Fetch the key */ + if (!enif_inspect_iolist_as_binary(env, key_arg, &key_bin)) + { + *return_term = EXCP_BADARG(env, "Bad key"); + goto err; + } + + /* Fetch cipher type */ + if (!enif_is_atom(env, cipher_arg)) + { + *return_term = EXCP_BADARG(env, "Cipher id is not an atom"); + goto err; + } + + if (!(*cipherp = get_cipher_type(cipher_arg, key_bin.size))) + { + if (!get_cipher_type_no_key(cipher_arg)) + *return_term = EXCP_BADARG(env, "Unknown cipher"); + else + *return_term = EXCP_BADARG(env, "Bad key size"); + goto err; + } + + if ((*cipherp)->flags & AEAD_CIPHER) + { + *return_term = EXCP_BADARG(env, "Missing arguments for this cipher"); + goto err; + } + + + if (FORBIDDEN_IN_FIPS(*cipherp)) + { + *return_term = EXCP_NOTSUP(env, "Forbidden in FIPS"); + goto err; + } + + /* Get ivec_len for this cipher (if we found one) */ +#if !defined(HAVE_EVP_AES_CTR) + /* This code is for historic OpenSSL where EVP_aes_*_ctr is not defined.... */ + if ((*cipherp)->cipher.p) { + /* Not aes_ctr compatibility code since EVP_* + was defined and assigned to (*cipherp)->cipher.p */ + ivec_len = GET_IV_LEN(*cipherp); + } else { + /* No EVP_* was found */ + if ((*cipherp)->flags & AES_CTR_COMPAT) + /* Use aes_ctr compatibility code later */ + ivec_len = 16; + else { + /* Unsupported crypto */ + *return_term = EXCP_NOTSUP(env, "Cipher not supported in this libcrypto version"); + goto err; + } + } +#else + /* Normal code */ + if (!((*cipherp)->cipher.p)) { + *return_term = EXCP_NOTSUP(env, "Cipher not supported in this libcrypto version"); + goto err; + } + ivec_len = GET_IV_LEN(*cipherp); +#endif + + /* (*cipherp)->cipher.p != NULL and ivec_len has a value */ + + /* Fetch IV */ + if (ivec_len && (ivec_arg != atom_undefined)) { + if (!enif_inspect_iolist_as_binary(env, ivec_arg, &ivec_bin)) + { + *return_term = EXCP_BADARG(env, "Bad iv type"); + goto err; + } + + if (ivec_len != ivec_bin.size) + { + *return_term = EXCP_BADARG(env, "Bad iv size"); + goto err; + } + } + + ctx_res->iv_len = ivec_len; + +#if !defined(HAVE_EVP_AES_CTR) + if (!((*cipherp)->cipher.p) + && ((*cipherp)->flags & AES_CTR_COMPAT) + ) { + /* Must use aes_ctr compatibility code */ + ERL_NIF_TERM ecount_bin; + unsigned char *outp; + if ((outp = enif_make_new_binary(env, AES_BLOCK_SIZE, &ecount_bin)) == NULL) { + *return_term = EXCP_ERROR(env, "Can't allocate ecount_bin"); + goto err; + } + memset(outp, 0, AES_BLOCK_SIZE); + + ctx_res->env = enif_alloc_env(); + if (!ctx_res->env) { + *return_term = EXCP_ERROR(env, "Can't allocate env"); + goto err; + } + ctx_res->state = + enif_make_copy(ctx_res->env, + enif_make_tuple4(env, key_arg, ivec_arg, ecount_bin, enif_make_int(env, 0))); + goto success; + } else { + /* Flag for subsequent calls that no aes_ctr compatibility code should be called */ + ctx_res->state = atom_undefined; + ctx_res->env = NULL; + } +#endif + + /* Initialize the EVP_CIPHER_CTX */ + + ctx_res->ctx = EVP_CIPHER_CTX_new(); + if (! ctx_res->ctx) + { + *return_term = EXCP_ERROR(env, "Can't allocate context"); + goto err; + } + + if (!EVP_CipherInit_ex(ctx_res->ctx, (*cipherp)->cipher.p, NULL, NULL, NULL, encflg)) + { + *return_term = EXCP_ERROR(env, "Can't initialize context, step 1"); + goto err; + } + + if (!EVP_CIPHER_CTX_set_key_length(ctx_res->ctx, (int)key_bin.size)) + { + *return_term = EXCP_ERROR(env, "Can't initialize context, key_length"); + goto err; + } + +#ifdef HAVE_RC2 + if (EVP_CIPHER_type((*cipherp)->cipher.p) == NID_rc2_cbc) { + if (key_bin.size > INT_MAX / 8) { + *return_term = EXCP_BADARG(env, "To large rc2_cbc key"); + goto err; + } + if (!EVP_CIPHER_CTX_ctrl(ctx_res->ctx, EVP_CTRL_SET_RC2_KEY_BITS, (int)key_bin.size * 8, NULL)) { + *return_term = EXCP_ERROR(env, "ctrl rc2_cbc key"); + goto err; + } + } +#endif + + if (ivec_arg == atom_undefined || ivec_len == 0) + { + if (!EVP_CipherInit_ex(ctx_res->ctx, NULL, NULL, key_bin.data, NULL, -1)) { + *return_term = EXCP_ERROR(env, "Can't initialize key"); + goto err; + } + } + else + if (!EVP_CipherInit_ex(ctx_res->ctx, NULL, NULL, key_bin.data, ivec_bin.data, -1)) + { + *return_term = EXCP_ERROR(env, "Can't initialize key or iv"); + goto err; + } + + EVP_CIPHER_CTX_set_padding(ctx_res->ctx, 0); + + *return_term = atom_ok; + +#if !defined(HAVE_EVP_AES_CTR) + success: +#endif + return 1; + + err: + if (ctx_res->ctx) EVP_CIPHER_CTX_free(ctx_res->ctx); + return 0; +} + +/*************************************************************************/ +/* Get the arguments for the EVP_CipherUpdate function, and call it. */ +/*************************************************************************/ + +static int get_update_args(ErlNifEnv* env, + struct evp_cipher_ctx *ctx_res, + const ERL_NIF_TERM indata_arg, + ERL_NIF_TERM *return_term) +{ + ErlNifBinary in_data_bin, out_data_bin; + int out_len, block_size; + + if (!enif_inspect_binary(env, indata_arg, &in_data_bin) ) + { + *return_term = EXCP_BADARG(env, "Bad 2:nd arg"); + goto err; + } + + ASSERT(in_data_bin.size <= INT_MAX); + +#if !defined(HAVE_EVP_AES_CTR) + if (ctx_res->state != atom_undefined) { + ERL_NIF_TERM state0, newstate_and_outdata; + const ERL_NIF_TERM *tuple_argv; + int tuple_argc; + + state0 = enif_make_copy(env, ctx_res->state); + + if (enif_get_tuple(env, state0, &tuple_argc, &tuple_argv) && (tuple_argc == 4)) { + /* A compatibility state term */ + /* encrypt and decrypt is performed by calling the same function */ + newstate_and_outdata = aes_ctr_stream_encrypt_compat(env, state0, indata_arg); + + if (enif_get_tuple(env, newstate_and_outdata, &tuple_argc, &tuple_argv) && (tuple_argc == 2)) { + /* newstate_and_outdata = {NewState, OutData} */ + ctx_res->state = enif_make_copy(ctx_res->env, tuple_argv[0]); + /* Return the OutData (from the newstate_and_outdata tuple) only: */ + *return_term = tuple_argv[1]; + } + } + } else +#endif + { + block_size = EVP_CIPHER_CTX_block_size(ctx_res->ctx); + + if (!enif_alloc_binary((size_t)in_data_bin.size+block_size, &out_data_bin)) + { + *return_term = EXCP_ERROR(env, "Can't allocate outdata"); + goto err; + } + + if (!EVP_CipherUpdate(ctx_res->ctx, out_data_bin.data, &out_len, in_data_bin.data, in_data_bin.size)) + { + *return_term = EXCP_ERROR(env, "Can't update"); + goto err; + } + + if (!enif_realloc_binary(&out_data_bin, (size_t)out_len)) + { + *return_term = EXCP_ERROR(env, "Can't reallocate"); + goto err; + } + + CONSUME_REDS(env, in_data_bin); + /* return the result text as a binary: */ + *return_term = enif_make_binary(env, &out_data_bin); + } + + /* success: */ + return 1; + + err: + return 0; +} + +/*************************************************************************/ +/* Initialize the state for (de/en)cryption */ +/*************************************************************************/ + +ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Cipher, Key, IVec, Encrypt) % if no IV for the Cipher, set IVec = <<>> + */ + struct evp_cipher_ctx *ctx_res = NULL; + const struct cipher_type_t *cipherp; + ERL_NIF_TERM ret; + int encflg; + + if (enif_is_atom(env, argv[0])) { + if ((ctx_res = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) + return EXCP_ERROR(env, "Can't allocate resource"); + + if (!get_init_args(env, ctx_res, argv[0], argv[1], argv[2], argv[argc-1], + &cipherp, &ret)) + /* Error msg in &ret */ + goto ret; + + ret = enif_make_resource(env, ctx_res); + if(ctx_res) enif_release_resource(ctx_res); + + } else if (enif_get_resource(env, argv[0], (ErlNifResourceType*)evp_cipher_ctx_rtype, (void**)&ctx_res)) { + /* Fetch the flag telling if we are going to encrypt (=true) or decrypt (=false) */ + if (argv[3] == atom_true) + encflg = 1; + else if (argv[3] == atom_false) + encflg = 0; + else { + ret = EXCP_BADARG(env, "Bad enc flag"); + goto ret; + } + if (ctx_res->ctx) { + /* It is *not* a ctx_res for the compatibility handling of non-EVP aes_ctr */ + if (!EVP_CipherInit_ex(ctx_res->ctx, NULL, NULL, NULL, NULL, encflg)) { + ret = EXCP_ERROR(env, "Can't initialize encflag"); + goto ret; + } + } + ret = argv[0]; + } else { + ret = EXCP_BADARG(env, "Bad 1:st arg"); + goto ret; + } + + ret: + return ret; +} + + +/*************************************************************************/ +/* Encrypt/decrypt */ +/*************************************************************************/ + +#if !defined(HAVE_EVP_CIPHER_CTX_COPY) +/* + The EVP_CIPHER_CTX_copy is not available in older cryptolibs although + the function is needed. + Instead of implement it in-place, we have a copy here as a compatibility + function +*/ + +int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in); + +int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in) +{ + if ((in == NULL) || (in->cipher == NULL)) + { + return 0; + } +#ifdef HAS_ENGINE_SUPPORT + /* Make sure it's safe to copy a cipher context using an ENGINE */ + if (in->engine && !ENGINE_init(in->engine)) + return 0; +#endif + + EVP_CIPHER_CTX_cleanup(out); + memcpy(out,in,sizeof *out); + + if (in->cipher_data && in->cipher->ctx_size) + { + out->cipher_data=OPENSSL_malloc(in->cipher->ctx_size); + if (!out->cipher_data) + return 0; + memcpy(out->cipher_data,in->cipher_data,in->cipher->ctx_size); + } + +#if defined(EVP_CIPH_CUSTOM_COPY) && defined(EVP_CTRL_COPY) + if (in->cipher->flags & EVP_CIPH_CUSTOM_COPY) + return in->cipher->ctrl((EVP_CIPHER_CTX *)in, EVP_CTRL_COPY, 0, out); +#endif + return 1; +} +/****** End of compatibility function ******/ +#endif + + +ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Context, Data [, IV]) */ + struct evp_cipher_ctx *ctx_res; + struct evp_cipher_ctx ctx_res_copy; + ERL_NIF_TERM ret; + + ctx_res_copy.ctx = NULL; + + if (!enif_get_resource(env, argv[0], (ErlNifResourceType*)evp_cipher_ctx_rtype, (void**)&ctx_res)) + return EXCP_BADARG(env, "Bad 1:st arg"); + + if (argc == 3) { + ErlNifBinary ivec_bin; + + memcpy(&ctx_res_copy, ctx_res, sizeof ctx_res_copy); +#if !defined(HAVE_EVP_AES_CTR) + if (ctx_res_copy.state == atom_undefined) + /* Not going to use aes_ctr compat functions */ +#endif + { + ctx_res_copy.ctx = EVP_CIPHER_CTX_new(); + + if (!EVP_CIPHER_CTX_copy(ctx_res_copy.ctx, ctx_res->ctx)) { + ret = EXCP_ERROR(env, "Can't copy ctx_res"); + goto err; + } + } + + ctx_res = &ctx_res_copy; + + if (!enif_inspect_iolist_as_binary(env, argv[2], &ivec_bin)) + { + ret = EXCP_BADARG(env, "Bad iv type"); + goto err; + } + + if (ctx_res_copy.iv_len != ivec_bin.size) + { + ret = EXCP_BADARG(env, "Bad iv size"); + goto err; + } + +#if !defined(HAVE_EVP_AES_CTR) + if ((ctx_res_copy.state != atom_undefined) ) { + /* replace the iv in state with argv[2] */ + ERL_NIF_TERM state0; + const ERL_NIF_TERM *tuple_argv; + int tuple_argc; + state0 = enif_make_copy(env, ctx_res_copy.state); + if (enif_get_tuple(env, state0, &tuple_argc, &tuple_argv) && (tuple_argc == 4)) { + /* A compatibility state term */ + ctx_res_copy.state = enif_make_tuple4(env, tuple_argv[0], argv[2], tuple_argv[2], tuple_argv[3]); + } + } else +#endif + if (!EVP_CipherInit_ex(ctx_res_copy.ctx, NULL, NULL, NULL, ivec_bin.data, -1)) + { + ret = EXCP_ERROR(env, "Can't set iv"); + goto err; + } + + get_update_args(env, &ctx_res_copy, argv[1], &ret); + } else + get_update_args(env, ctx_res, argv[1], &ret); + + err: + if (ctx_res_copy.ctx) + EVP_CIPHER_CTX_free(ctx_res_copy.ctx); + + return ret; /* Both success and error */ +} + + +ERL_NIF_TERM ng_crypto_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Context, Data [, IV]) */ + ErlNifBinary data_bin; + + ASSERT(argc <= 3); + + if (!enif_inspect_binary(env, argv[1], &data_bin)) + return EXCP_BADARG(env, "expected binary as data"); + + if (data_bin.size > INT_MAX) + return EXCP_BADARG(env, "to long data"); + + /* Run long jobs on a dirty scheduler to not block the current emulator thread */ + if (data_bin.size > MAX_BYTES_TO_NIF) { + return enif_schedule_nif(env, "ng_crypto_update", + ERL_NIF_DIRTY_JOB_CPU_BOUND, + ng_crypto_update, argc, argv); + } + + return ng_crypto_update(env, argc, argv); +} + +/*************************************************************************/ +/* One shot */ +/*************************************************************************/ + +ERL_NIF_TERM ng_crypto_one_time(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Cipher, Key, IVec, Data, Encrypt) */ + struct evp_cipher_ctx ctx_res; + const struct cipher_type_t *cipherp; + ERL_NIF_TERM ret; + + ctx_res.ctx = NULL; +#if !defined(HAVE_EVP_AES_CTR) + ctx_res.env = NULL; +#endif + + if (!get_init_args(env, &ctx_res, argv[0], argv[1], argv[2], argv[4], &cipherp, &ret)) + goto ret; + + get_update_args(env, &ctx_res, argv[3], &ret); + + ret: + if (ctx_res.ctx) + EVP_CIPHER_CTX_free(ctx_res.ctx); + +#if !defined(HAVE_EVP_AES_CTR) + if (ctx_res.env) + enif_free_env(ctx_res.env); +#endif + + return ret; +} + + +ERL_NIF_TERM ng_crypto_one_time_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Cipher, Key, IVec, Data, Encrypt) % if no IV for the Cipher, set IVec = <<>> + */ + ErlNifBinary data_bin; + + ASSERT(argc == 5); + + if (!enif_inspect_binary(env, argv[3], &data_bin)) + return EXCP_BADARG(env, "expected binary as data"); + + if (data_bin.size > INT_MAX) + return EXCP_BADARG(env, "to long data"); + + /* Run long jobs on a dirty scheduler to not block the current emulator thread */ + if (data_bin.size > MAX_BYTES_TO_NIF) { + return enif_schedule_nif(env, "ng_crypto_one_time", + ERL_NIF_DIRTY_JOB_CPU_BOUND, + ng_crypto_one_time, argc, argv); + } + + return ng_crypto_one_time(env, argc, argv); +} diff --git a/lib/crypto/c_src/api_ng.h b/lib/crypto/c_src/api_ng.h new file mode 100644 index 0000000000..aaf67524ae --- /dev/null +++ b/lib/crypto/c_src/api_ng.h @@ -0,0 +1,30 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_API_NG_H__ +#define E_API_NG_H__ 1 + +#include "common.h" + +ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM ng_crypto_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM ng_crypto_one_time_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_AES_H__ */ diff --git a/lib/crypto/c_src/atoms.c b/lib/crypto/c_src/atoms.c new file mode 100644 index 0000000000..059c14690f --- /dev/null +++ b/lib/crypto/c_src/atoms.c @@ -0,0 +1,252 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "atoms.h" + +ERL_NIF_TERM atom_true; +ERL_NIF_TERM atom_false; +ERL_NIF_TERM atom_sha; +ERL_NIF_TERM atom_error; +ERL_NIF_TERM atom_rsa_pkcs1_padding; +ERL_NIF_TERM atom_rsa_pkcs1_oaep_padding; +ERL_NIF_TERM atom_rsa_no_padding; +ERL_NIF_TERM atom_signature_md; +ERL_NIF_TERM atom_undefined; + +ERL_NIF_TERM atom_ok; +ERL_NIF_TERM atom_none; +ERL_NIF_TERM atom_notsup; +ERL_NIF_TERM atom_badarg; +ERL_NIF_TERM atom_digest; +#ifdef FIPS_SUPPORT +ERL_NIF_TERM atom_enabled; +ERL_NIF_TERM atom_not_enabled; +#else +ERL_NIF_TERM atom_not_supported; +#endif + +ERL_NIF_TERM atom_type; +ERL_NIF_TERM atom_size; +ERL_NIF_TERM atom_block_size; +ERL_NIF_TERM atom_key_length; +ERL_NIF_TERM atom_iv_length; +ERL_NIF_TERM atom_mode; +ERL_NIF_TERM atom_ecb_mode; +ERL_NIF_TERM atom_cbc_mode; +ERL_NIF_TERM atom_cfb_mode; +ERL_NIF_TERM atom_ofb_mode; +ERL_NIF_TERM atom_ctr_mode; +ERL_NIF_TERM atom_gcm_mode; +ERL_NIF_TERM atom_ccm_mode; +ERL_NIF_TERM atom_xts_mode; +ERL_NIF_TERM atom_wrap_mode; +ERL_NIF_TERM atom_ocb_mode; +ERL_NIF_TERM atom_stream_cipher; + +#if defined(HAVE_EC) +ERL_NIF_TERM atom_prime_field; +ERL_NIF_TERM atom_characteristic_two_field; +ERL_NIF_TERM atom_tpbasis; +ERL_NIF_TERM atom_ppbasis; +ERL_NIF_TERM atom_onbasis; +#endif + +ERL_NIF_TERM atom_aes_cfb8; +ERL_NIF_TERM atom_aes_cfb128; +ERL_NIF_TERM atom_aes_ige256; +#ifdef HAVE_GCM +ERL_NIF_TERM atom_aes_gcm; +#endif +#ifdef HAVE_CCM +ERL_NIF_TERM atom_aes_ccm; +#endif + +ERL_NIF_TERM atom_rsa; +ERL_NIF_TERM atom_dss; +ERL_NIF_TERM atom_ecdsa; + +#ifdef HAVE_ED_CURVE_DH +ERL_NIF_TERM atom_x25519; +ERL_NIF_TERM atom_x448; +#endif + +ERL_NIF_TERM atom_eddsa; +#ifdef HAVE_EDDSA +ERL_NIF_TERM atom_ed25519; +ERL_NIF_TERM atom_ed448; +#endif + +ERL_NIF_TERM atom_rsa_mgf1_md; +ERL_NIF_TERM atom_rsa_oaep_label; +ERL_NIF_TERM atom_rsa_oaep_md; +ERL_NIF_TERM atom_rsa_pad; /* backwards compatibility */ +ERL_NIF_TERM atom_rsa_padding; +ERL_NIF_TERM atom_rsa_pkcs1_pss_padding; +#ifdef HAVE_RSA_SSLV23_PADDING +ERL_NIF_TERM atom_rsa_sslv23_padding; +#endif +ERL_NIF_TERM atom_rsa_x931_padding; +ERL_NIF_TERM atom_rsa_pss_saltlen; + +#ifdef HAVE_BLAKE2 +ERL_NIF_TERM atom_blake2b; +ERL_NIF_TERM atom_blake2s; +#endif + +#ifdef HAS_ENGINE_SUPPORT + +ERL_NIF_TERM atom_engine_method_rsa; +ERL_NIF_TERM atom_engine_method_dsa; +ERL_NIF_TERM atom_engine_method_dh; +ERL_NIF_TERM atom_engine_method_rand; +ERL_NIF_TERM atom_engine_method_ecdh; +ERL_NIF_TERM atom_engine_method_ecdsa; +ERL_NIF_TERM atom_engine_method_ciphers; +ERL_NIF_TERM atom_engine_method_digests; +ERL_NIF_TERM atom_engine_method_store; +ERL_NIF_TERM atom_engine_method_pkey_meths; +ERL_NIF_TERM atom_engine_method_pkey_asn1_meths; +ERL_NIF_TERM atom_engine_method_ec; + +ERL_NIF_TERM atom_engine; +ERL_NIF_TERM atom_key_id; +ERL_NIF_TERM atom_password; +#endif + +int init_atoms(ErlNifEnv *env, const ERL_NIF_TERM fips_mode, const ERL_NIF_TERM load_info) { + atom_true = enif_make_atom(env,"true"); + atom_false = enif_make_atom(env,"false"); + /* Enter FIPS mode */ + if (fips_mode == atom_true) { +#ifdef FIPS_SUPPORT + if (!FIPS_mode_set(1)) { +#else + { +#endif + PRINTF_ERR0("CRYPTO: Could not setup FIPS mode"); + return 0; + } + } else if (fips_mode != atom_false) { + PRINTF_ERR1("CRYPTO: Invalid load_info '%T'", load_info); + return 0; + } + + atom_sha = enif_make_atom(env,"sha"); + atom_error = enif_make_atom(env,"error"); + atom_rsa_pkcs1_padding = enif_make_atom(env,"rsa_pkcs1_padding"); + atom_rsa_pkcs1_oaep_padding = enif_make_atom(env,"rsa_pkcs1_oaep_padding"); + atom_rsa_no_padding = enif_make_atom(env,"rsa_no_padding"); + atom_signature_md = enif_make_atom(env,"signature_md"); + atom_undefined = enif_make_atom(env,"undefined"); + atom_ok = enif_make_atom(env,"ok"); + atom_none = enif_make_atom(env,"none"); + atom_notsup = enif_make_atom(env,"notsup"); + atom_badarg = enif_make_atom(env,"badarg"); + atom_digest = enif_make_atom(env,"digest"); + + atom_type = enif_make_atom(env,"type"); + atom_size = enif_make_atom(env,"size"); + atom_block_size = enif_make_atom(env,"block_size"); + atom_key_length = enif_make_atom(env,"key_length"); + atom_iv_length = enif_make_atom(env,"iv_length"); + atom_mode = enif_make_atom(env,"mode"); + atom_ecb_mode = enif_make_atom(env,"ecb_mode"); + atom_cbc_mode = enif_make_atom(env,"cbc_mode"); + atom_cfb_mode = enif_make_atom(env,"cfb_mode"); + atom_ofb_mode = enif_make_atom(env,"ofb_mode"); + atom_ctr_mode = enif_make_atom(env,"ctr_mode"); + atom_gcm_mode = enif_make_atom(env,"gcm_mode"); + atom_ccm_mode = enif_make_atom(env,"ccm_mode"); + atom_xts_mode = enif_make_atom(env,"xts_mode"); + atom_wrap_mode = enif_make_atom(env,"wrap_mode"); + atom_ocb_mode = enif_make_atom(env,"ocb_mode"); + atom_stream_cipher = enif_make_atom(env,"stream_cipher"); + +#if defined(HAVE_EC) + atom_prime_field = enif_make_atom(env,"prime_field"); + atom_characteristic_two_field = enif_make_atom(env,"characteristic_two_field"); + atom_tpbasis = enif_make_atom(env,"tpbasis"); + atom_ppbasis = enif_make_atom(env,"ppbasis"); + atom_onbasis = enif_make_atom(env,"onbasis"); +#endif + + atom_aes_cfb8 = enif_make_atom(env, "aes_cfb8"); + atom_aes_cfb128 = enif_make_atom(env, "aes_cfb128"); + atom_aes_ige256 = enif_make_atom(env, "aes_ige256"); +#ifdef HAVE_GCM + atom_aes_gcm = enif_make_atom(env, "aes_gcm"); +#endif +#ifdef HAVE_CCM + atom_aes_ccm = enif_make_atom(env, "aes_ccm"); +#endif + +#ifdef FIPS_SUPPORT + atom_enabled = enif_make_atom(env,"enabled"); + atom_not_enabled = enif_make_atom(env,"not_enabled"); +#else + atom_not_supported = enif_make_atom(env,"not_supported"); +#endif + atom_rsa = enif_make_atom(env,"rsa"); + atom_dss = enif_make_atom(env,"dss"); + atom_ecdsa = enif_make_atom(env,"ecdsa"); + +#ifdef HAVE_ED_CURVE_DH + atom_x25519 = enif_make_atom(env,"x25519"); + atom_x448 = enif_make_atom(env,"x448"); +#endif + atom_eddsa = enif_make_atom(env,"eddsa"); +#ifdef HAVE_EDDSA + atom_ed25519 = enif_make_atom(env,"ed25519"); + atom_ed448 = enif_make_atom(env,"ed448"); +#endif + atom_rsa_mgf1_md = enif_make_atom(env,"rsa_mgf1_md"); + atom_rsa_oaep_label = enif_make_atom(env,"rsa_oaep_label"); + atom_rsa_oaep_md = enif_make_atom(env,"rsa_oaep_md"); + atom_rsa_pad = enif_make_atom(env,"rsa_pad"); /* backwards compatibility */ + atom_rsa_padding = enif_make_atom(env,"rsa_padding"); + atom_rsa_pkcs1_pss_padding = enif_make_atom(env,"rsa_pkcs1_pss_padding"); +#ifdef HAVE_RSA_SSLV23_PADDING + atom_rsa_sslv23_padding = enif_make_atom(env,"rsa_sslv23_padding"); +#endif + atom_rsa_x931_padding = enif_make_atom(env,"rsa_x931_padding"); + atom_rsa_pss_saltlen = enif_make_atom(env,"rsa_pss_saltlen"); + +#ifdef HAS_ENGINE_SUPPORT + + atom_engine_method_rsa = enif_make_atom(env,"engine_method_rsa"); + atom_engine_method_dsa = enif_make_atom(env,"engine_method_dsa"); + atom_engine_method_dh = enif_make_atom(env,"engine_method_dh"); + atom_engine_method_rand = enif_make_atom(env,"engine_method_rand"); + atom_engine_method_ecdh = enif_make_atom(env,"engine_method_ecdh"); + atom_engine_method_ecdsa = enif_make_atom(env,"engine_method_ecdsa"); + atom_engine_method_store = enif_make_atom(env,"engine_method_store"); + atom_engine_method_ciphers = enif_make_atom(env,"engine_method_ciphers"); + atom_engine_method_digests = enif_make_atom(env,"engine_method_digests"); + atom_engine_method_pkey_meths = enif_make_atom(env,"engine_method_pkey_meths"); + atom_engine_method_pkey_asn1_meths = enif_make_atom(env,"engine_method_pkey_asn1_meths"); + atom_engine_method_ec = enif_make_atom(env,"engine_method_ec"); + + atom_engine = enif_make_atom(env,"engine"); + atom_key_id = enif_make_atom(env,"key_id"); + atom_password = enif_make_atom(env,"password"); +#endif + + return 1; +} diff --git a/lib/crypto/c_src/atoms.h b/lib/crypto/c_src/atoms.h new file mode 100644 index 0000000000..f5913de96f --- /dev/null +++ b/lib/crypto/c_src/atoms.h @@ -0,0 +1,134 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_ATOMS_H__ +#define E_ATOMS_H__ 1 + +#include <erl_nif.h> +#include "openssl_config.h" + +extern ERL_NIF_TERM atom_true; +extern ERL_NIF_TERM atom_false; +extern ERL_NIF_TERM atom_sha; +extern ERL_NIF_TERM atom_error; +extern ERL_NIF_TERM atom_rsa_pkcs1_padding; +extern ERL_NIF_TERM atom_rsa_pkcs1_oaep_padding; +extern ERL_NIF_TERM atom_rsa_no_padding; +extern ERL_NIF_TERM atom_signature_md; +extern ERL_NIF_TERM atom_undefined; + +extern ERL_NIF_TERM atom_ok; +extern ERL_NIF_TERM atom_none; +extern ERL_NIF_TERM atom_notsup; +extern ERL_NIF_TERM atom_badarg; +extern ERL_NIF_TERM atom_digest; +#ifdef FIPS_SUPPORT +extern ERL_NIF_TERM atom_enabled; +extern ERL_NIF_TERM atom_not_enabled; +#else +extern ERL_NIF_TERM atom_not_supported; +#endif + +extern ERL_NIF_TERM atom_type; +extern ERL_NIF_TERM atom_size; +extern ERL_NIF_TERM atom_block_size; +extern ERL_NIF_TERM atom_key_length; +extern ERL_NIF_TERM atom_iv_length; +extern ERL_NIF_TERM atom_mode; +extern ERL_NIF_TERM atom_ecb_mode; +extern ERL_NIF_TERM atom_cbc_mode; +extern ERL_NIF_TERM atom_cfb_mode; +extern ERL_NIF_TERM atom_ofb_mode; +extern ERL_NIF_TERM atom_ctr_mode; +extern ERL_NIF_TERM atom_gcm_mode; +extern ERL_NIF_TERM atom_ccm_mode; +extern ERL_NIF_TERM atom_xts_mode; +extern ERL_NIF_TERM atom_wrap_mode; +extern ERL_NIF_TERM atom_ocb_mode; +extern ERL_NIF_TERM atom_stream_cipher; + +#if defined(HAVE_EC) +extern ERL_NIF_TERM atom_prime_field; +extern ERL_NIF_TERM atom_characteristic_two_field; +extern ERL_NIF_TERM atom_tpbasis; +extern ERL_NIF_TERM atom_ppbasis; +extern ERL_NIF_TERM atom_onbasis; +#endif + +extern ERL_NIF_TERM atom_aes_cfb8; +extern ERL_NIF_TERM atom_aes_cfb128; +extern ERL_NIF_TERM atom_aes_ige256; +#ifdef HAVE_GCM +extern ERL_NIF_TERM atom_aes_gcm; +#endif +#ifdef HAVE_CCM +extern ERL_NIF_TERM atom_aes_ccm; +#endif + +extern ERL_NIF_TERM atom_rsa; +extern ERL_NIF_TERM atom_dss; +extern ERL_NIF_TERM atom_ecdsa; + +#ifdef HAVE_ED_CURVE_DH +extern ERL_NIF_TERM atom_x25519; +extern ERL_NIF_TERM atom_x448; +#endif + +extern ERL_NIF_TERM atom_eddsa; +#ifdef HAVE_EDDSA +extern ERL_NIF_TERM atom_ed25519; +extern ERL_NIF_TERM atom_ed448; +#endif + +extern ERL_NIF_TERM atom_rsa_mgf1_md; +extern ERL_NIF_TERM atom_rsa_oaep_label; +extern ERL_NIF_TERM atom_rsa_oaep_md; +extern ERL_NIF_TERM atom_rsa_pad; /* backwards compatibility */ +extern ERL_NIF_TERM atom_rsa_padding; +extern ERL_NIF_TERM atom_rsa_pkcs1_pss_padding; +#ifdef HAVE_RSA_SSLV23_PADDING +extern ERL_NIF_TERM atom_rsa_sslv23_padding; +#endif +extern ERL_NIF_TERM atom_rsa_x931_padding; +extern ERL_NIF_TERM atom_rsa_pss_saltlen; + +#ifdef HAS_ENGINE_SUPPORT + +extern ERL_NIF_TERM atom_engine_method_rsa; +extern ERL_NIF_TERM atom_engine_method_dsa; +extern ERL_NIF_TERM atom_engine_method_dh; +extern ERL_NIF_TERM atom_engine_method_rand; +extern ERL_NIF_TERM atom_engine_method_ecdh; +extern ERL_NIF_TERM atom_engine_method_ecdsa; +extern ERL_NIF_TERM atom_engine_method_ciphers; +extern ERL_NIF_TERM atom_engine_method_digests; +extern ERL_NIF_TERM atom_engine_method_store; +extern ERL_NIF_TERM atom_engine_method_pkey_meths; +extern ERL_NIF_TERM atom_engine_method_pkey_asn1_meths; +extern ERL_NIF_TERM atom_engine_method_ec; + +extern ERL_NIF_TERM atom_engine; +extern ERL_NIF_TERM atom_key_id; +extern ERL_NIF_TERM atom_password; +#endif + +int init_atoms(ErlNifEnv *env, const ERL_NIF_TERM fips_mode, const ERL_NIF_TERM load_info); + +#endif /* E_ATOMS_H__ */ diff --git a/lib/crypto/c_src/bn.c b/lib/crypto/c_src/bn.c new file mode 100644 index 0000000000..34ed4f7ebc --- /dev/null +++ b/lib/crypto/c_src/bn.c @@ -0,0 +1,186 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "bn.h" + + +int get_bn_from_mpint(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp) +{ + BIGNUM *ret; + ErlNifBinary bin; + int sz; + + if (!enif_inspect_binary(env, term, &bin)) + goto err; + if (bin.size > INT_MAX - 4) + goto err; + + ERL_VALGRIND_ASSERT_MEM_DEFINED(bin.data, bin.size); + + if (bin.size < 4) + goto err; + sz = (int)bin.size - 4; + if (get_int32(bin.data) != sz) + goto err; + + if ((ret = BN_bin2bn(bin.data+4, sz, NULL)) == NULL) + goto err; + + *bnp = ret; + return 1; + + err: + return 0; +} + +int get_bn_from_bin(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp) +{ + BIGNUM *ret; + ErlNifBinary bin; + + if (!enif_inspect_binary(env, term, &bin)) + goto err; + if (bin.size > INT_MAX) + goto err; + + ERL_VALGRIND_ASSERT_MEM_DEFINED(bin.data, bin.size); + + if ((ret = BN_bin2bn(bin.data, (int)bin.size, NULL)) == NULL) + goto err; + + *bnp = ret; + return 1; + + err: + return 0; +} + +ERL_NIF_TERM bin_from_bn(ErlNifEnv* env, const BIGNUM *bn) +{ + int bn_len; + unsigned char *bin_ptr; + ERL_NIF_TERM term; + + /* Copy the bignum into an erlang binary. */ + if ((bn_len = BN_num_bytes(bn)) < 0) + goto err; + if ((bin_ptr = enif_make_new_binary(env, (size_t)bn_len, &term)) == NULL) + goto err; + + if (BN_bn2bin(bn, bin_ptr) < 0) + goto err; + + return term; + + err: + return atom_error; +} + +ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Base,Exponent,Modulo,bin_hdr) */ + BIGNUM *bn_base = NULL, *bn_exponent = NULL, *bn_modulo = NULL, *bn_result = NULL; + BN_CTX *bn_ctx = NULL; + unsigned char* ptr; + int dlen; + unsigned bin_hdr; /* return type: 0=plain binary, 4: mpint */ + unsigned extra_byte; + ERL_NIF_TERM ret; + + ASSERT(argc == 4); + + if (!get_bn_from_bin(env, argv[0], &bn_base)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[1], &bn_exponent)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[2], &bn_modulo)) + goto bad_arg; + if (!enif_get_uint(env, argv[3], &bin_hdr)) + goto bad_arg; + if (bin_hdr != 0 && bin_hdr != 4) + goto bad_arg; + + if ((bn_result = BN_new()) == NULL) + goto err; + if ((bn_ctx = BN_CTX_new()) == NULL) + goto err; + + if (!BN_mod_exp(bn_result, bn_base, bn_exponent, bn_modulo, bn_ctx)) + goto err; + + dlen = BN_num_bytes(bn_result); + if (dlen < 0 || dlen > INT_MAX / 8) + goto bad_arg; + extra_byte = bin_hdr && BN_is_bit_set(bn_result, dlen * 8 - 1); + + if ((ptr = enif_make_new_binary(env, bin_hdr + extra_byte + (unsigned int)dlen, &ret)) == NULL) + goto err; + + if (bin_hdr) { + put_uint32(ptr, extra_byte + (unsigned int)dlen); + ptr[4] = 0; /* extra zeroed byte to ensure a positive mpint */ + ptr += bin_hdr + extra_byte; + } + + BN_bn2bin(bn_result, ptr); + goto done; + + bad_arg: + err: + ret = enif_make_badarg(env); + + done: + if (bn_base) + BN_free(bn_base); + if (bn_exponent) + BN_free(bn_exponent); + if (bn_modulo) + BN_free(bn_modulo); + if (bn_result) + BN_free(bn_result); + if (bn_ctx) + BN_CTX_free(bn_ctx); + return ret; +} + +#ifdef HAVE_EC +ERL_NIF_TERM bn2term(ErlNifEnv* env, const BIGNUM *bn) +{ + int dlen; + unsigned char* ptr; + ERL_NIF_TERM ret; + + if (bn == NULL) + return atom_undefined; + + dlen = BN_num_bytes(bn); + if (dlen < 0) + goto err; + if ((ptr = enif_make_new_binary(env, (size_t)dlen, &ret)) == NULL) + goto err; + + BN_bn2bin(bn, ptr); + + ERL_VALGRIND_MAKE_MEM_DEFINED(ptr, dlen); + return ret; + + err: + return enif_make_badarg(env); +} +#endif diff --git a/lib/crypto/c_src/bn.h b/lib/crypto/c_src/bn.h new file mode 100644 index 0000000000..332b06e79d --- /dev/null +++ b/lib/crypto/c_src/bn.h @@ -0,0 +1,36 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_BN_H__ +#define E_BN_H__ 1 + +#include "common.h" + +ERL_NIF_TERM bin_from_bn(ErlNifEnv* env, const BIGNUM *bn); +ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#ifdef HAVE_EC +ERL_NIF_TERM bn2term(ErlNifEnv* env, const BIGNUM *bn); +#endif + +int get_bn_from_mpint(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp); +int get_bn_from_bin(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp); + +#endif /* E_BN_H__ */ diff --git a/lib/crypto/c_src/check_erlang.cocci b/lib/crypto/c_src/check_erlang.cocci new file mode 100644 index 0000000000..b2a981f2ac --- /dev/null +++ b/lib/crypto/c_src/check_erlang.cocci @@ -0,0 +1,196 @@ +// %CopyrightBegin% +// +// Copyright Doug Hogan 2019. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% + +// Coccinelle script to help verify Erlang calls. +// http://coccinelle.lip6.fr +// https://github.com/coccinelle/coccinelle +// +// These work with the Erlang code because it has a rigid coding pattern. +// $ spatch.opt --all-includes -sp_file check_erlang.cocci -dir . + +// Make sure resources are cleaned up properly in all paths. +// Need 'strict' so it's also checked in error handling paths. +@enif_alloc_resource@ +type T; +identifier CTX, L; +identifier virtual.enif_alloc_resource, virtual.enif_release_resource; +position p, pr; +@@ + + T *CTX = NULL; + + ... + if ((CTX = enif_alloc_resource(...)@p) == NULL) + goto L; + + ... when strict, forall + if (CTX) + enif_release_resource(CTX)@pr; + + +// After calling enif_alloc_binary(), you must either release it with +// enif_release_binary() or transfer ownership to Erlang via enif_make_binary(). +@enif_alloc_binary@ +expression SZ; +identifier BIN, RET, ENV, X, L; +identifier TUPLE =~ "^enif_make_tuple[0-9]+$"; +identifier virtual.enif_alloc_binary, virtual.enif_make_binary; +identifier virtual.enif_release_binary; +position pa, pm, pr; +@@ + +// This construct is used in engine.c +( + if (!enif_alloc_binary(SZ, &BIN)@pa) + goto L; + + ... when strict, forall + return +( + enif_make_binary(ENV, &BIN)@pm +| + TUPLE(..., enif_make_binary(ENV, &BIN)@pm)@pm +); + +| +// This is the typical way we allocate and use binaries. + int X = 0; + + ... + if (!enif_alloc_binary(SZ, &BIN)@pa) + goto L; + X = 1; + + ... when strict, forall +( + RET = enif_make_binary(ENV, &BIN)@pm; + X = 0; +| + if (X) + enif_release_binary(&BIN)@pr; +| + return enif_make_binary(ENV, &BIN)@pm; +) +) + +// TODO: These don't have single checks that handle all cases. +// +// enif_consume_timeslice returns 1 if exhausted or else 0 +// enif_has_pending_exception returns true if exception pending + +@erlang_check_void@ +identifier FUNCVOID =~ "^(enif_mutex_destroy|enif_mutex_lock|enif_mutex_unlock|enif_rwlock_destroy|enif_rwlock_rlock|enif_rwlock_runlock|enif_rwlock_rwlock|enif_rwlock_rwunlock|enif_system_info)$"; +position p; +@@ + + FUNCVOID(...)@p; + + +@erlang_check_null@ +expression X; +identifier L; +identifier FUNCNULL =~ "^(enif_alloc|enif_alloc_resource|enif_dlopen|enif_dlsym|enif_make_new_binary|enif_mutex_create|enif_open_resource_type|enif_realloc|enif_rwlock_create)$"; +position p; +@@ + +( + if ((X = FUNCNULL(...)@p) == NULL) + goto L; +| + X = FUNCNULL(...)@p; + if (X == NULL) + goto L; +| + return FUNCNULL(...)@p; +) + + +@erlang_check_not@ +identifier L; +identifier FUNCNOT =~ "^(enif_alloc_binary|enif_get_int|enif_get_list_cell|enif_get_list_length|enif_get_long|enif_get_map_value|enif_get_resource|enif_get_tuple|enif_get_uint|enif_get_ulong|enif_inspect_binary|enif_inspect_iolist_as_binary|enif_is_atom|enif_is_binary|enif_is_current_process_alive|enif_is_empty_list|enif_is_list|enif_is_map|enif_is_tuple|enif_realloc_binary)$"; +position p; +@@ + +( + if (!FUNCNOT(...)@p) + goto L; +| + return FUNCNOT(...)@p; +) + + +@erlang_check_null_free@ +expression X; +identifier FUNCFREE =~ "^(enif_free|enif_free_env|enif_free_iovec|enif_release_binary|enif_release_resource)$"; +position p; +@@ + + if ( +( + X +| + X != NULL +) + ) + FUNCFREE(X)@p; + + +@erlang_check_new@ +expression RET; +identifier FUNCNEW =~ "^(enif_make_atom|enif_make_badarg|enif_make_binary|enif_make_int|enif_make_list|enif_make_list_from_array|enif_make_resource|enif_make_tuple|enif_raise_exception|enif_schedule_nif|enif_thread_self)$"; +position p; +@@ + +( + RET = FUNCNEW(...)@p; +| + return FUNCNEW(...)@p; +) + + +// Flag any calls that aren't part of the above pattern. +@enif_alloc_not_free@ + +identifier FUNCVOID =~ "^(enif_mutex_destroy|enif_mutex_lock|enif_mutex_unlock|enif_rwlock_destroy|enif_rwlock_rlock|enif_rwlock_runlock|enif_rwlock_rwlock|enif_rwlock_rwunlock|enif_system_info)$"; +position pvoid != {erlang_check_void.p,enif_alloc_binary.pr}; + +identifier FUNCNULL =~ "^(enif_alloc|enif_alloc_resource|enif_dlopen|enif_dlsym|enif_make_new_binary|enif_mutex_create|enif_open_resource_type|enif_realloc|enif_rwlock_create)$"; +position pnull != {erlang_check_null.p,enif_alloc_resource.p}; + +identifier FUNCNOT =~ "^(enif_alloc_binary|enif_get_int|enif_get_list_cell|enif_get_list_length|enif_get_long|enif_get_map_value|enif_get_resource|enif_get_tuple|enif_get_uint|enif_get_ulong|enif_inspect_binary|enif_inspect_iolist_as_binary|enif_is_atom|enif_is_binary|enif_is_current_process_alive|enif_is_empty_list|enif_is_list|enif_is_map|enif_is_tuple|enif_realloc_binary)$"; +position pnot != {erlang_check_not.p,enif_alloc_binary.pa}; + +identifier FUNCNEW =~ "^(enif_make_atom|enif_make_badarg|enif_make_binary|enif_make_int|enif_make_list|enif_make_list_from_array|enif_make_resource|enif_make_tuple|enif_raise_exception|enif_schedule_nif|enif_thread_self)$"; +position pnew != {erlang_check_new.p,enif_alloc_binary.pm}; + +identifier FUNCFREE =~ "^(enif_free|enif_free_env|enif_free_iovec|enif_release_binary|enif_release_resource)$"; +position pfree != {enif_alloc_resource.pr,enif_alloc_binary.pr,erlang_check_null_free.p}; + +@@ + +( +* FUNCVOID(...)@pvoid +| +* FUNCNULL(...)@pnull +| +* FUNCNOT(...)@pnot +| +* FUNCNEW(...)@pnew +| +* FUNCFREE(...)@pfree +) diff --git a/lib/crypto/c_src/check_openssl.cocci b/lib/crypto/c_src/check_openssl.cocci new file mode 100644 index 0000000000..75d1a6e44b --- /dev/null +++ b/lib/crypto/c_src/check_openssl.cocci @@ -0,0 +1,281 @@ +// %CopyrightBegin% +// +// Copyright Doug Hogan 2019. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% + +// Coccinelle script to help verify the subset of OpenSSL calls used by Erlang. +// http://coccinelle.lip6.fr +// https://github.com/coccinelle/coccinelle +// +// These work with the Erlang code because it has a rigid coding pattern. +// $ spatch.opt --all-includes -sp_file check_openssl.cocci -dir . + +// TODO: These APIs may not have a single check that covers all cases +// or may not be necessary to check. +// +// BN_GENCB_get_arg +// BN_bn2bin +// BN_cmp +// BN_is_bit_set +// BN_is_negative +// BN_is_zero +// BN_num_bits +// DH_get0_key +// DH_size +// EC_GROUP_get_degree +// EC_KEY_get0_group +// EC_KEY_get0_private_key +// EC_KEY_get0_public_key +// EC_KEY_get_conv_form +// EVP_CIPHER_block_size +// EVP_CIPHER_iv_length +// EVP_CIPHER_type +// EVP_MD_CTX_md +// EVP_MD_size +// EVP_aes_128_cbc +// EVP_aes_128_ccm +// EVP_aes_128_cfb128 +// EVP_aes_128_cfb8 +// EVP_aes_128_ctr +// EVP_aes_128_ecb +// EVP_aes_128_gcm +// EVP_aes_192_cbc +// EVP_aes_192_ccm +// EVP_aes_192_ctr +// EVP_aes_192_ecb +// EVP_aes_192_gcm +// EVP_aes_256_cbc +// EVP_aes_256_ccm +// EVP_aes_256_ctr +// EVP_aes_256_ecb +// EVP_aes_256_gcm +// EVP_bf_cbc +// EVP_bf_cfb64 +// EVP_bf_ecb +// EVP_bf_ofb +// EVP_chacha20 +// EVP_chacha20_poly1305 +// EVP_des_cbc +// EVP_des_cfb8 +// EVP_des_ecb +// EVP_des_ede3_cbc +// EVP_des_ede3_cfb8 +// EVP_md4 +// EVP_md5 +// EVP_rc2_cbc +// EVP_ripemd160 +// EVP_sha1 +// EVP_sha224 +// EVP_sha256 +// EVP_sha384 +// EVP_sha3_224 +// EVP_sha3_256 +// EVP_sha3_384 +// EVP_sha3_512 +// EVP_sha512 +// OpenSSL_version +// OpenSSL_version_num +// PEM_read_PrivateKey +// PEM_read_PUBKEY +// RSA_size + +// Unusual API for OpenSSL: 0 or positive on success and negative value(s) on error. +@openssl_check_negative@ +identifier FUNCNEG =~ "^(DH_compute_key|RSA_padding_check_SSLv23)$"; +expression X; +identifier L; +position p; +@@ + + if ( +( + FUNCNEG(...)@p < 0 +| + (X = FUNCNEG(...)@p) < 0 +) + ) + goto L; + +// Unusual API for OpenSSL: positive on success or else error +@openssl_check_positive@ +identifier FUNCPOS =~ "^(ECDH_compute_key|EVP_CIPHER_asn1_to_param|EVP_CIPHER_param_to_asn1|EVP_PKEY_CTX_ctrl|RSA_pkey_ctx_ctrl)$"; +identifier L; +expression X; +position p; +@@ + + if ( +( + FUNCPOS(...)@p < 1 +| + (X = FUNCPOS(...)@p) < 1 +) + ) + goto L; + +// Unusual API for OpenSSL: 0=success. +@openssl_check_0@ +identifier L; +expression X; +identifier FUNC0 =~ "^(AES_set_decrypt_key|AES_set_encrypt_key|CRYPTO_gcm128_aad|CRYPTO_gcm128_decrypt|CRYPTO_gcm128_finish)$"; +position p; +@@ + + if ( +( + FUNC0(...)@p != 0 +| + (X = FUNC0(...)@p) != 0 +) + ) + goto L; + +// These do not necessarily allocate resources but they may return NULL. +@openssl_check_null@ +expression X; +identifier L; +identifier FUNCNULL =~ "^(BN_CTX_new|BN_GENCB_new|BN_MONT_CTX_new|BN_bin2bn|BN_dup|BN_generate_prime|BN_new|CMAC_CTX_new|CRYPTO_clear_realloc|CRYPTO_gcm128_new|CRYPTO_malloc|CRYPTO_realloc|CRYPTO_zalloc|DH_generate_parameters|DH_new|DSA_new|EC_GROUP_dup|EC_GROUP_get0_generator|EC_GROUP_method_of|EC_GROUP_new_curve_GFm|EC_GROUP_new_curve_GFp|EC_KEY_copy|EC_KEY_dup|EC_KEY_get0_engine|EC_KEY_new|EC_KEY_new_by_curve_name|EC_POINT_bn2point|EC_POINT_dup|EC_POINT_new|EC_POINT_point2bn|ENGINE_by_id|ENGINE_get_cipher_engine|ENGINE_get_default_DH|ENGINE_get_default_DSA|ENGINE_get_default_RAND|ENGINE_get_default_RSA|ENGINE_get_digest_engine|ENGINE_get_first|ENGINE_get_id|ENGINE_get_last|ENGINE_get_name|ENGINE_get_next|ENGINE_get_prev|ENGINE_load_private_key|ENGINE_load_public_key|ENGINE_new|EVP_CIPHER_CTX_new|EVP_MAC_CTX_new|EVP_MAC_CTX_new_id|EVP_MD_CTX_new|EVP_MD_meth_new|EVP_PKEY_CTX_new|EVP_PKEY_CTX_new_id|EVP_PKEY_get1_DH|EVP_PKEY_get1_DSA|EVP_PKEY_get1_EC_KEY|EVP_PKEY_get1_RSA|EVP_PKEY_new|EVP_PKEY_new_raw_private_key|EVP_PKEY_new_raw_public_key|EVP_get_cipherbyname|EVP_get_cipherbynid|EVP_get_cipherbyobj|EVP_get_macbyname|EVP_get_macbynid|EVP_get_macbyobj|HMAC|HMAC_CTX_new|OPENSSL_buf2hexstr|OPENSSL_clear_realloc|OPENSSL_hexstr2buf|OPENSSL_malloc|OPENSSL_realloc|OPENSSL_strdup|OPENSSL_strndup|OPENSSL_zalloc|RSA_meth_dup|RSA_meth_new|RSA_new)$"; +position p; +@@ + +( + if ((X = FUNCNULL(...)@p) == NULL) + goto L; +| + X = FUNCNULL(...)@p; + if (X == NULL) + goto L; +) + +// non-zero=success, 0=failure. These can be safely used with ! +@openssl_check_not@ +expression X; +identifier L; +identifier FUNCNOT =~ "^(BN_add|BN_div|BN_exp|BN_from_montgomery|BN_gcd|BN_generate_prime_ex|BN_mod|BN_mod_add|BN_mod_exp|BN_mod_mul|BN_mod_mul_montgomery|BN_mod_sqr|BN_mod_sub|BN_mul|BN_nnmod|BN_priv_rand|BN_priv_rand_range|BN_pseudo_rand|BN_pseudo_rand_range|BN_rand|BN_rand_range|BN_set_bit|BN_set_word|BN_sqr|BN_sub|BN_to_montgomery|CMAC_Final|CMAC_Init|CMAC_Update|CRYPTO_set_mem_debug|CRYPTO_set_mem_functions|DH_check|DH_check_ex|DH_check_params|DH_check_pub_key_ex|DH_generate_key|DH_generate_parameters_ex|DH_set0_key|DH_set0_pqg|DH_set_length|DSA_set0_key|DSA_set0_pqg|EC_GROUP_check|EC_GROUP_check_discriminant|EC_GROUP_copy|EC_GROUP_get_curve_name|EC_GROUP_get_pentanomial_basis|EC_GROUP_get_trinomial_basis|EC_GROUP_precompute_mult|EC_GROUP_set_generator|EC_GROUP_set_seed|EC_KEY_check_key|EC_KEY_generate_key|EC_KEY_key2buf|EC_KEY_oct2key|EC_KEY_oct2priv|EC_KEY_precompute_mult|EC_KEY_priv2buf|EC_KEY_priv2oct|EC_KEY_set_group|EC_KEY_set_private_key|EC_KEY_set_public_key|EC_KEY_set_public_key_affine_coordinates|EC_KEY_up_ref|EC_POINT_add|EC_POINT_copy|EC_POINT_dbl|EC_POINT_get_Jprojective_coordinates_GFp|EC_POINT_get_affine_coordinates_GF2m|EC_POINT_get_affine_coordinates_GFp|EC_POINT_invert|EC_POINT_make_affine|EC_POINT_mul|EC_POINT_oct2point|EC_POINT_point2oct|EC_POINT_set_Jprojective_coordinates_GFp|EC_POINT_set_affine_coordinates_GF2m|EC_POINT_set_affine_coordinates_GFp|EC_POINT_set_compressed_coordinates_GF2m|EC_POINT_set_compressed_coordinates_GFp|EC_POINT_set_to_infinity|EC_POINTs_make_affine|EC_POINTs_mul|ENGINE_add|ENGINE_ctrl_cmd|ENGINE_ctrl_cmd_string|ENGINE_finish|ENGINE_free|ENGINE_init|ENGINE_register_DH|ENGINE_register_DSA|ENGINE_register_EC|ENGINE_register_RAND|ENGINE_register_RSA|ENGINE_register_all_complete|ENGINE_register_ciphers|ENGINE_register_complete|ENGINE_register_digests|ENGINE_register_pkey_asn1_meths|ENGINE_register_pkey_meths|ENGINE_remove|ENGINE_set_RSA|ENGINE_set_default|ENGINE_set_default_DH|ENGINE_set_default_DSA|ENGINE_set_default_EC|ENGINE_set_default_RAND|ENGINE_set_default_RSA|ENGINE_set_digests|ENGINE_set_id|ENGINE_set_init_function|ENGINE_set_load_privkey_function|ENGINE_set_load_pubkey_function|ENGINE_set_name|ENGINE_up_ref|HMAC_CTX_copy|HMAC_CTX_reset|HMAC_Final|HMAC_Init_ex|HMAC_Update|MD2_Init|MD2_Update|MD2_Final|MD4_Init|MD4_Update|MD4_Final|MD5_Init|MD5_Update|MD5_Final|OPENSSL_init_crypto|OPENSSL_mem_debug_pop|OPENSSL_mem_debug_push|RSA_generate_key_ex|RSA_generate_multi_prime_key|RSA_meth_set_finish|RSA_meth_set_sign|RSA_meth_set_verify|RSA_padding_add_SSLv23|RSA_set0_crt_params|RSA_set0_factors|RSA_set0_key|RSA_set0_multi_prime_params)$"; +position p; +@@ + + if ( +( + !FUNCNOT(...)@p +| + !(X = FUNCNOT)@p +) + ) + goto L; + +// 1=success. These may have == 0 or <= 0 or non-one failure so we explicitly check for success. +// Since some EVP_* functions use failure == 0 and others use <= 0, we consolidate all +// EVP_* calls into here so it's less error prone. In such cases, they all use 1 for success. +@openssl_check_1@ +expression X; +identifier L; +identifier FUNC1 =~ "^(EVP_CIPHER_CTX_copy|EVP_CIPHER_CTX_ctrl|EVP_CIPHER_CTX_rand_key|EVP_CIPHER_CTX_reset|EVP_CIPHER_CTX_set_key_length|EVP_CIPHER_CTX_set_padding|EVP_CipherFinal_ex|EVP_CipherInit_ex|EVP_CipherUpdate|EVP_DecryptFinal_ex|EVP_DecryptInit_ex|EVP_DecryptUpdate|EVP_Digest|EVP_DigestFinal|EVP_DigestFinal_ex|EVP_DigestInit|EVP_DigestInit_ex|EVP_DigestSign|EVP_DigestSignInit|EVP_DigestSignUpdate|EVP_DigestSignaFinal|EVP_DigestUpdate|EVP_DigestVerify|EVP_DigestVerifyInit|EVP_EncryptFinal_ex|EVP_EncryptInit_ex|EVP_EncryptUpdate|EVP_MAC_CTX_copy|EVP_MAC_ctrl|EVP_MAC_ctrl_str|EVP_MAC_hex2ctrl|EVP_MAC_init|EVP_MAC_reset|EVP_MAC_str2ctrl|EVP_MAC_update|EVP_MD_CTX_copy|EVP_MD_CTX_copy_ex|EVP_MD_CTX_ctrl|EVP_MD_meth_set_app_datasize|EVP_MD_meth_set_cleanup|EVP_MD_meth_set_copy|EVP_MD_meth_set_ctrl|EVP_MD_meth_set_final|EVP_MD_meth_set_flags|EVP_MD_meth_set_init|EVP_MD_meth_set_input_blocksize|EVP_MD_meth_set_result_size|EVP_MD_meth_set_update|EVP_PKEY_CTX_set_rsa_mgf1_md|EVP_PKEY_CTX_set_rsa_padding|EVP_PKEY_CTX_set_rsa_pss_saltlen|EVP_PKEY_CTX_set_signature|EVP_PKEY_assign|EVP_PKEY_assign_DSA|EVP_PKEY_assign_EC_KEY|EVP_PKEY_assign_RSA|EVP_PKEY_decrypt|EVP_PKEY_decrypt_init|EVP_PKEY_derive|EVP_PKEY_derive_init|EVP_PKEY_derive_set_peer|EVP_PKEY_encrypt|EVP_PKEY_encrypt_init|EVP_PKEY_get1_DH|EVP_PKEY_get_raw_private_key|EVP_PKEY_get_raw_public_key|EVP_PKEY_keygen|EVP_PKEY_keygen_init|EVP_PKEY_set1_DH|EVP_PKEY_sign|EVP_PKEY_sign_init|EVP_PKEY_verify|EVP_PKEY_verify_init|EVP_PKEY_verify_recover|EVP_PKEY_verify_recover_init|EVP_add_mac|RAND_bytes|RAND_priv_bytes)$"; +position p; +@@ + + if ( +( + FUNC1(...)@p != 1 +| + (X = FUNC1(...)@p) != 1 +) + ) + goto L; + + +// These are void but here for completeness +@openssl_void@ +identifier FUNCVOID =~ "^(AES_cfb128_encrypt|AES_cfb8_encrypt|AES_ige_encrypt|BN_GENCB_set|DSA_get0_key|DSA_get0_pqg|EC_GROUP_set_asn1_flag|EC_GROUP_set_point_conversion_form|ENGINE_get_static_state|ENGINE_unregister_DH|ENGINE_unregister_DSA|ENGINE_unregister_EC|ENGINE_unregister_RAND|ENGINE_unregister_RSA|ENGINE_unregister_ciphers|ENGINE_unregister_digests|ENGINE_unregister_pkey_asn1_meths|ENGINE_unregister_pkey_meths|OpenSSL_add_all_ciphers|OpenSSL_add_all_digests|RAND_seed|RC4|RC4_set_key|RSA_get0_crt_params|RSA_get0_factors|RSA_get0_key)$"; +position p; +@@ + + FUNCVOID(...)@p; + + +// Traditionally, OpenSSL didn't adhere to the semantics of free() calls +// allowing for NULL. However, they have been changing it over time. +// Since Erlang allows for unmaintained versions of OpenSSL, be conservative +// and assume the worst. +@openssl_free@ +expression X; +identifier FUNCFREE =~ "^(BN_CTX_free|BN_GENCB_free|BN_clear_free|BN_free|CMAC_CTX_free|CRYPTO_free|DH_free|DSA_free|EC_GROUP_free|EC_KEY_free|EC_POINT_free|EVP_CIPHER_CTX_free|EVP_MD_CTX_free|EVP_PKEY_CTX_free|EVP_PKEY_free|HMAC_CTX_free|RSA_free|RSA_meth_free)$"; +position p; +@@ + + if ( +( + X +| + X != NULL +) + ) + FUNCFREE(X)@p; + + +// NOTE: Keep these in sync with the above definitions! +// +// Find all of the cases that we haven't marked safe positions of. +// +// This will flag a few false positives because the code isn't using the +// standard pattern. +// +// NOTE: You have to copy the regexps because there doesn't appear to be a way in +// coccinelle to reference a regexp identifier from another rule properly. +@openssl_check_NOT_SAFE@ + +identifier FUNCNEG =~ "^(DH_compute_key|RSA_padding_check_SSLv23)$"; +position pneg != openssl_check_negative.p; + +identifier FUNCPOS =~ "^(ECDH_compute_key|EVP_CIPHER_asn1_to_param|EVP_CIPHER_param_to_asn1|EVP_PKEY_CTX_ctrl|RSA_pkey_ctx_ctrl)$"; +position ppos != openssl_check_positive.p; + +identifier FUNC0 =~ "^(AES_set_decrypt_key|AES_set_encrypt_key|CRYPTO_gcm128_aad|CRYPTO_gcm128_decrypt|CRYPTO_gcm128_finish)$"; +position p0 != openssl_check_0.p; + +identifier FUNCNULL =~ "^(BN_CTX_new|BN_GENCB_new|BN_MONT_CTX_new|BN_bin2bn|BN_dup|BN_generate_prime|BN_new|CMAC_CTX_new|CRYPTO_clear_realloc|CRYPTO_gcm128_new|CRYPTO_malloc|CRYPTO_realloc|CRYPTO_zalloc|DH_generate_parameters|DH_new|DSA_new|EC_GROUP_dup|EC_GROUP_get0_generator|EC_GROUP_method_of|EC_GROUP_new_curve_GFm|EC_GROUP_new_curve_GFp|EC_KEY_copy|EC_KEY_dup|EC_KEY_get0_engine|EC_KEY_new|EC_KEY_new_by_curve_name|EC_POINT_bn2point|EC_POINT_dup|EC_POINT_new|EC_POINT_point2bn|ENGINE_by_id|ENGINE_get_cipher_engine|ENGINE_get_default_DH|ENGINE_get_default_DSA|ENGINE_get_default_RAND|ENGINE_get_default_RSA|ENGINE_get_digest_engine|ENGINE_get_first|ENGINE_get_id|ENGINE_get_last|ENGINE_get_name|ENGINE_get_next|ENGINE_get_prev|ENGINE_load_private_key|ENGINE_load_public_key|ENGINE_new|EVP_CIPHER_CTX_new|EVP_MAC_CTX_new|EVP_MAC_CTX_new_id|EVP_MD_CTX_new|EVP_MD_meth_new|EVP_PKEY_CTX_new|EVP_PKEY_CTX_new_id|EVP_PKEY_get1_DH|EVP_PKEY_get1_DSA|EVP_PKEY_get1_EC_KEY|EVP_PKEY_get1_RSA|EVP_PKEY_new|EVP_PKEY_new_raw_private_key|EVP_PKEY_new_raw_public_key|EVP_get_cipherbyname|EVP_get_cipherbynid|EVP_get_cipherbyobj|EVP_get_macbyname|EVP_get_macbynid|EVP_get_macbyobj|HMAC|HMAC_CTX_new|OPENSSL_buf2hexstr|OPENSSL_clear_realloc|OPENSSL_hexstr2buf|OPENSSL_malloc|OPENSSL_realloc|OPENSSL_strdup|OPENSSL_strndup|OPENSSL_zalloc|RSA_meth_dup|RSA_meth_new|RSA_new)$"; +position pnull != openssl_check_null.p; + +identifier FUNCNOT =~ "^(BN_add|BN_div|BN_exp|BN_from_montgomery|BN_gcd|BN_generate_prime_ex|BN_mod|BN_mod_add|BN_mod_exp|BN_mod_mul|BN_mod_mul_montgomery|BN_mod_sqr|BN_mod_sub|BN_mul|BN_nnmod|BN_priv_rand|BN_priv_rand_range|BN_pseudo_rand|BN_pseudo_rand_range|BN_rand|BN_rand_range|BN_set_bit|BN_set_word|BN_sqr|BN_sub|BN_to_montgomery|CMAC_Final|CMAC_Init|CMAC_Update|CRYPTO_set_mem_debug|CRYPTO_set_mem_functions|DH_check|DH_check_ex|DH_check_params|DH_check_pub_key_ex|DH_generate_key|DH_generate_parameters_ex|DH_set0_key|DH_set0_pqg|DH_set_length|DSA_set0_key|DSA_set0_pqg|EC_GROUP_check|EC_GROUP_check_discriminant|EC_GROUP_copy|EC_GROUP_get_curve_name|EC_GROUP_get_pentanomial_basis|EC_GROUP_get_trinomial_basis|EC_GROUP_precompute_mult|EC_GROUP_set_generator|EC_GROUP_set_seed|EC_KEY_check_key|EC_KEY_generate_key|EC_KEY_key2buf|EC_KEY_oct2key|EC_KEY_oct2priv|EC_KEY_precompute_mult|EC_KEY_priv2buf|EC_KEY_priv2oct|EC_KEY_set_group|EC_KEY_set_private_key|EC_KEY_set_public_key|EC_KEY_set_public_key_affine_coordinates|EC_KEY_up_ref|EC_POINT_add|EC_POINT_copy|EC_POINT_dbl|EC_POINT_get_Jprojective_coordinates_GFp|EC_POINT_get_affine_coordinates_GF2m|EC_POINT_get_affine_coordinates_GFp|EC_POINT_invert|EC_POINT_make_affine|EC_POINT_mul|EC_POINT_oct2point|EC_POINT_point2oct|EC_POINT_set_Jprojective_coordinates_GFp|EC_POINT_set_affine_coordinates_GF2m|EC_POINT_set_affine_coordinates_GFp|EC_POINT_set_compressed_coordinates_GF2m|EC_POINT_set_compressed_coordinates_GFp|EC_POINT_set_to_infinity|EC_POINTs_make_affine|EC_POINTs_mul|ENGINE_add|ENGINE_ctrl_cmd|ENGINE_ctrl_cmd_string|ENGINE_finish|ENGINE_free|ENGINE_init|ENGINE_register_DH|ENGINE_register_DSA|ENGINE_register_EC|ENGINE_register_RAND|ENGINE_register_RSA|ENGINE_register_all_complete|ENGINE_register_ciphers|ENGINE_register_complete|ENGINE_register_digests|ENGINE_register_pkey_asn1_meths|ENGINE_register_pkey_meths|ENGINE_remove|ENGINE_set_RSA|ENGINE_set_default|ENGINE_set_default_DH|ENGINE_set_default_DSA|ENGINE_set_default_EC|ENGINE_set_default_RAND|ENGINE_set_default_RSA|ENGINE_set_digests|ENGINE_set_id|ENGINE_set_init_function|ENGINE_set_load_privkey_function|ENGINE_set_load_pubkey_function|ENGINE_set_name|ENGINE_up_ref|HMAC_CTX_copy|HMAC_CTX_reset|HMAC_Final|HMAC_Init_ex|HMAC_Update|MD2_Init|MD2_Update|MD2_Final|MD4_Init|MD4_Update|MD4_Final|MD5_Init|MD5_Update|MD5_Final|OPENSSL_init_crypto|OPENSSL_mem_debug_pop|OPENSSL_mem_debug_push|RSA_generate_key_ex|RSA_generate_multi_prime_key|RSA_meth_set_finish|RSA_meth_set_sign|RSA_meth_set_verify|RSA_padding_add_SSLv23|RSA_set0_crt_params|RSA_set0_factors|RSA_set0_key|RSA_set0_multi_prime_params)$"; +position pnot != openssl_check_not.p; + +identifier FUNC1 =~ "^(EVP_CIPHER_CTX_copy|EVP_CIPHER_CTX_ctrl|EVP_CIPHER_CTX_rand_key|EVP_CIPHER_CTX_reset|EVP_CIPHER_CTX_set_key_length|EVP_CIPHER_CTX_set_padding|EVP_CipherFinal_ex|EVP_CipherInit_ex|EVP_CipherUpdate|EVP_DecryptFinal_ex|EVP_DecryptInit_ex|EVP_DecryptUpdate|EVP_Digest|EVP_DigestFinal|EVP_DigestFinal_ex|EVP_DigestInit|EVP_DigestInit_ex|EVP_DigestSign|EVP_DigestSignInit|EVP_DigestSignUpdate|EVP_DigestSignaFinal|EVP_DigestUpdate|EVP_DigestVerify|EVP_DigestVerifyInit|EVP_EncryptFinal_ex|EVP_EncryptInit_ex|EVP_EncryptUpdate|EVP_MAC_CTX_copy|EVP_MAC_ctrl|EVP_MAC_ctrl_str|EVP_MAC_hex2ctrl|EVP_MAC_init|EVP_MAC_reset|EVP_MAC_str2ctrl|EVP_MAC_update|EVP_MD_CTX_copy|EVP_MD_CTX_copy_ex|EVP_MD_CTX_ctrl|EVP_MD_meth_set_app_datasize|EVP_MD_meth_set_cleanup|EVP_MD_meth_set_copy|EVP_MD_meth_set_ctrl|EVP_MD_meth_set_final|EVP_MD_meth_set_flags|EVP_MD_meth_set_init|EVP_MD_meth_set_input_blocksize|EVP_MD_meth_set_result_size|EVP_MD_meth_set_update|EVP_PKEY_CTX_set_rsa_mgf1_md|EVP_PKEY_CTX_set_rsa_padding|EVP_PKEY_CTX_set_rsa_pss_saltlen|EVP_PKEY_CTX_set_signature|EVP_PKEY_assign|EVP_PKEY_assign_DSA|EVP_PKEY_assign_EC_KEY|EVP_PKEY_assign_RSA|EVP_PKEY_decrypt|EVP_PKEY_decrypt_init|EVP_PKEY_derive|EVP_PKEY_derive_init|EVP_PKEY_derive_set_peer|EVP_PKEY_encrypt|EVP_PKEY_encrypt_init|EVP_PKEY_get1_DH|EVP_PKEY_get_raw_private_key|EVP_PKEY_get_raw_public_key|EVP_PKEY_keygen|EVP_PKEY_keygen_init|EVP_PKEY_set1_DH|EVP_PKEY_sign|EVP_PKEY_sign_init|EVP_PKEY_verify|EVP_PKEY_verify_init|EVP_PKEY_verify_recover|EVP_PKEY_verify_recover_init|EVP_add_mac|RAND_bytes|RAND_priv_bytes)$"; +position p1 != openssl_check_1.p; + +identifier FUNCVOID =~ "^(AES_cfb128_encrypt|AES_cfb8_encrypt|AES_ige_encrypt|BN_GENCB_set|DSA_get0_key|DSA_get0_pqg|EC_GROUP_set_asn1_flag|EC_GROUP_set_point_conversion_form|ENGINE_get_static_state|ENGINE_unregister_DH|ENGINE_unregister_DSA|ENGINE_unregister_EC|ENGINE_unregister_RAND|ENGINE_unregister_RSA|ENGINE_unregister_ciphers|ENGINE_unregister_digests|ENGINE_unregister_pkey_asn1_meths|ENGINE_unregister_pkey_meths|OpenSSL_add_all_ciphers|OpenSSL_add_all_digests|RAND_seed|RC4|RC4_set_key|RSA_get0_crt_params|RSA_get0_factors|RSA_get0_key)$"; +position pvoid != openssl_void.p; + +identifier FUNCFREE =~ "^(BN_CTX_free|BN_GENCB_free|BN_clear_free|BN_free|CMAC_CTX_free|CRYPTO_free|DH_free|DSA_free|EC_GROUP_free|EC_KEY_free|EC_POINT_free|EVP_CIPHER_CTX_free|EVP_MD_CTX_free|EVP_PKEY_CTX_free|EVP_PKEY_free|HMAC_CTX_free|RSA_free|RSA_meth_free)$"; +position pfree != openssl_free.p; +@@ + +( +* FUNCNEG(...)@pneg +| +* FUNCPOS(...)@ppos +| +* FUNCNULL(...)@pnull +| +* FUNC0(...)@p0 +| +* FUNC1(...)@p1 +| +* FUNCNOT(...)@pnot +| +* FUNCVOID(...)@pvoid +| +* FUNCFREE(...)@pfree +) diff --git a/lib/crypto/c_src/cipher.c b/lib/crypto/c_src/cipher.c new file mode 100644 index 0000000000..00072af632 --- /dev/null +++ b/lib/crypto/c_src/cipher.c @@ -0,0 +1,344 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "cipher.h" + +#ifdef HAVE_DES +#define COND_NO_DES_PTR(Ptr) (Ptr) +#else +#define COND_NO_DES_PTR(Ptr) (NULL) +#endif + +static struct cipher_type_t cipher_types[] = +{ +#ifdef HAVE_RC2 + {{"rc2_cbc"}, {&EVP_rc2_cbc}, 0, NO_FIPS_CIPHER}, +#else + {{"rc2_cbc"}, {NULL}, 0, NO_FIPS_CIPHER}, +#endif +#ifdef HAVE_RC4 + {{"rc4"}, {&EVP_rc4}, 0, NO_FIPS_CIPHER}, +#else + {{"rc4"}, {NULL}, 0, NO_FIPS_CIPHER}, +#endif + {{"des_cbc"}, {COND_NO_DES_PTR(&EVP_des_cbc)}, 0, NO_FIPS_CIPHER}, + {{"des_cfb"}, {COND_NO_DES_PTR(&EVP_des_cfb8)}, 0, NO_FIPS_CIPHER}, + {{"des_ecb"}, {COND_NO_DES_PTR(&EVP_des_ecb)}, 0, NO_FIPS_CIPHER | ECB_BUG_0_9_8L}, + + {{"des_ede3_cbc"}, {COND_NO_DES_PTR(&EVP_des_ede3_cbc)}, 0, 0}, + +#ifdef HAVE_DES_ede3_cfb_encrypt + {{"des_ede3_cfb"}, {COND_NO_DES_PTR(&EVP_des_ede3_cfb8)}, 0, 0}, +#else + {{"des_ede3_cfb"}, {NULL}, 0, 0}, +#endif + +#ifdef HAVE_BF + {{"blowfish_cbc"}, {&EVP_bf_cbc}, 0, NO_FIPS_CIPHER}, + {{"blowfish_cfb64"}, {&EVP_bf_cfb64}, 0, NO_FIPS_CIPHER}, + {{"blowfish_ofb64"}, {&EVP_bf_ofb}, 0, NO_FIPS_CIPHER}, + {{"blowfish_ecb"}, {&EVP_bf_ecb}, 0, NO_FIPS_CIPHER | ECB_BUG_0_9_8L}, +#else + {{"blowfish_cbc"}, {NULL}, 0, 0}, + {{"blowfish_cfb64"}, {NULL}, 0, 0}, + {{"blowfish_ofb64"}, {NULL}, 0, 0}, + {{"blowfish_ecb"}, {NULL}, 0, 0}, +#endif + + {{"aes_128_cbc"}, {&EVP_aes_128_cbc}, 16, 0}, + {{"aes_192_cbc"}, {&EVP_aes_192_cbc}, 24, 0}, + {{"aes_256_cbc"}, {&EVP_aes_256_cbc}, 32, 0}, + + {{"aes_128_cfb8"}, {&EVP_aes_128_cfb8}, 16, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_192_cfb8"}, {&EVP_aes_192_cfb8}, 24, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_256_cfb8"}, {&EVP_aes_256_cfb8}, 32, NO_FIPS_CIPHER | AES_CFBx}, + + {{"aes_128_cfb128"}, {&EVP_aes_128_cfb128}, 16, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_192_cfb128"}, {&EVP_aes_192_cfb128}, 24, NO_FIPS_CIPHER | AES_CFBx}, + {{"aes_256_cfb128"}, {&EVP_aes_256_cfb128}, 32, NO_FIPS_CIPHER | AES_CFBx}, + + {{"aes_128_ecb"}, {&EVP_aes_128_ecb}, 16, ECB_BUG_0_9_8L}, + {{"aes_192_ecb"}, {&EVP_aes_192_ecb}, 24, ECB_BUG_0_9_8L}, + {{"aes_256_ecb"}, {&EVP_aes_256_ecb}, 32, ECB_BUG_0_9_8L}, + +#if defined(HAVE_EVP_AES_CTR) + {{"aes_128_ctr"}, {&EVP_aes_128_ctr}, 16, 0}, + {{"aes_192_ctr"}, {&EVP_aes_192_ctr}, 24, 0}, + {{"aes_256_ctr"}, {&EVP_aes_256_ctr}, 32, 0}, +#else + {{"aes_128_ctr"}, {NULL}, 16, AES_CTR_COMPAT}, + {{"aes_192_ctr"}, {NULL}, 24, AES_CTR_COMPAT}, + {{"aes_256_ctr"}, {NULL}, 32, AES_CTR_COMPAT}, +#endif + +#if defined(HAVE_CHACHA20) + {{"chacha20"}, {&EVP_chacha20}, 32, NO_FIPS_CIPHER}, +#else + {{"chacha20"}, {NULL}, 0, NO_FIPS_CIPHER}, +#endif + + /*==== AEAD ciphers ====*/ +#if defined(HAVE_CHACHA20_POLY1305) + {{"chacha20_poly1305"}, {&EVP_chacha20_poly1305}, 0, NO_FIPS_CIPHER | AEAD_CIPHER, {{EVP_CTRL_AEAD_SET_IVLEN,EVP_CTRL_AEAD_GET_TAG,EVP_CTRL_AEAD_SET_TAG}}}, +#else + {{"chacha20_poly1305"}, {NULL}, 0, NO_FIPS_CIPHER | AEAD_CIPHER, {{0,0,0}}}, +#endif + +#if defined(HAVE_GCM) + {{"aes_128_gcm"}, {&EVP_aes_128_gcm}, 16, AEAD_CIPHER|GCM_MODE, {{EVP_CTRL_GCM_SET_IVLEN,EVP_CTRL_GCM_GET_TAG,EVP_CTRL_GCM_SET_TAG}}}, + {{"aes_192_gcm"}, {&EVP_aes_192_gcm}, 24, AEAD_CIPHER|GCM_MODE, {{EVP_CTRL_GCM_SET_IVLEN,EVP_CTRL_GCM_GET_TAG,EVP_CTRL_GCM_SET_TAG}}}, + {{"aes_256_gcm"}, {&EVP_aes_256_gcm}, 32, AEAD_CIPHER|GCM_MODE, {{EVP_CTRL_GCM_SET_IVLEN,EVP_CTRL_GCM_GET_TAG,EVP_CTRL_GCM_SET_TAG}}}, +#else + {{"aes_128_gcm"}, {NULL}, 16, AEAD_CIPHER|GCM_MODE, {{0,0,0}}}, + {{"aes_192_gcm"}, {NULL}, 24, AEAD_CIPHER|GCM_MODE, {{0,0,0}}}, + {{"aes_256_gcm"}, {NULL}, 32, AEAD_CIPHER|GCM_MODE, {{0,0,0}}}, +#endif + +#if defined(HAVE_CCM) + {{"aes_128_ccm"}, {&EVP_aes_128_ccm}, 16, AEAD_CIPHER|CCM_MODE, {{EVP_CTRL_CCM_SET_IVLEN,EVP_CTRL_CCM_GET_TAG,EVP_CTRL_CCM_SET_TAG}}}, + {{"aes_192_ccm"}, {&EVP_aes_192_ccm}, 24, AEAD_CIPHER|CCM_MODE, {{EVP_CTRL_CCM_SET_IVLEN,EVP_CTRL_CCM_GET_TAG,EVP_CTRL_CCM_SET_TAG}}}, + {{"aes_256_ccm"}, {&EVP_aes_256_ccm}, 32, AEAD_CIPHER|CCM_MODE, {{EVP_CTRL_CCM_SET_IVLEN,EVP_CTRL_CCM_GET_TAG,EVP_CTRL_CCM_SET_TAG}}}, +#else + {{"aes_128_ccm"}, {NULL}, 16, AEAD_CIPHER|CCM_MODE, {{0,0,0}}}, + {{"aes_192_ccm"}, {NULL}, 24, AEAD_CIPHER|CCM_MODE, {{0,0,0}}}, + {{"aes_256_ccm"}, {NULL}, 32, AEAD_CIPHER|CCM_MODE, {{0,0,0}}}, +#endif + + /*==== Specialy handled ciphers, only for inclusion in algorithm's list ====*/ +#ifdef HAVE_AES_IGE + {{"aes_ige256"}, {NULL}, 0, NO_FIPS_CIPHER | NON_EVP_CIPHER}, +#endif + + /*==== End of list ==== */ + + {{NULL},{NULL},0,0} +}; + +ErlNifResourceType* evp_cipher_ctx_rtype; + +static size_t num_cipher_types = 0; + +static void evp_cipher_ctx_dtor(ErlNifEnv* env, struct evp_cipher_ctx* ctx) { + if (ctx == NULL) + return; + + if (ctx->ctx) + EVP_CIPHER_CTX_free(ctx->ctx); + +#if !defined(HAVE_EVP_AES_CTR) + if (ctx->env) + enif_free_env(ctx->env); +#endif +} + +int init_cipher_ctx(ErlNifEnv *env) { + evp_cipher_ctx_rtype = enif_open_resource_type(env, NULL, "EVP_CIPHER_CTX", + (ErlNifResourceDtor*) evp_cipher_ctx_dtor, + ERL_NIF_RT_CREATE|ERL_NIF_RT_TAKEOVER, + NULL); + if (evp_cipher_ctx_rtype == NULL) + goto err; + + return 1; + + err: + PRINTF_ERR0("CRYPTO: Could not open resource type 'EVP_CIPHER_CTX'"); + return 0; +} + +void init_cipher_types(ErlNifEnv* env) +{ + struct cipher_type_t* p = cipher_types; + + num_cipher_types = 0; + for (p = cipher_types; p->type.str; p++) { + num_cipher_types++; + p->type.atom = enif_make_atom(env, p->type.str); + if (p->cipher.funcp) + p->cipher.p = p->cipher.funcp(); + } + p->type.atom = atom_false; /* end marker */ + + qsort(cipher_types, num_cipher_types, sizeof(cipher_types[0]), cmp_cipher_types); +} + +const struct cipher_type_t* get_cipher_type(ERL_NIF_TERM type, size_t key_len) +{ + struct cipher_type_t key; + + key.type.atom = type; + key.key_len = key_len; + + return bsearch(&key, cipher_types, num_cipher_types, sizeof(cipher_types[0]), cmp_cipher_types); +} + + +int cmp_cipher_types(const void *keyp, const void *elemp) { + const struct cipher_type_t *key = keyp; + const struct cipher_type_t *elem = elemp; + + if (key->type.atom < elem->type.atom) return -1; + else if (key->type.atom > elem->type.atom) return 1; + else /* key->type.atom == elem->type.atom */ + if (!elem->key_len || key->key_len == elem->key_len) return 0; + else if (key->key_len < elem->key_len) return -1; + else return 1; +} + + +ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type) */ + const struct cipher_type_t *cipherp; + const EVP_CIPHER *cipher; + ERL_NIF_TERM ret, ret_mode; + unsigned type; + unsigned long mode; + + if ((cipherp = get_cipher_type_no_key(argv[0])) == NULL) + return enif_make_badarg(env); + + if (FORBIDDEN_IN_FIPS(cipherp)) + return enif_raise_exception(env, atom_notsup); + if ((cipher = cipherp->cipher.p) == NULL) + return enif_raise_exception(env, atom_notsup); + + ret = enif_make_new_map(env); + + type = EVP_CIPHER_type(cipher); + enif_make_map_put(env, ret, atom_type, + type == NID_undef ? atom_undefined : enif_make_int(env, type), + &ret); + + enif_make_map_put(env, ret, atom_key_length, + enif_make_int(env, EVP_CIPHER_key_length(cipher)), &ret); + enif_make_map_put(env, ret, atom_iv_length, + enif_make_int(env, EVP_CIPHER_iv_length(cipher)), &ret); + enif_make_map_put(env, ret, atom_block_size, + enif_make_int(env, EVP_CIPHER_block_size(cipher)), &ret); + + mode = EVP_CIPHER_mode(cipher); + switch (mode) { + case EVP_CIPH_ECB_MODE: + ret_mode = atom_ecb_mode; + break; + + case EVP_CIPH_CBC_MODE: + ret_mode = atom_cbc_mode; + break; + + case EVP_CIPH_CFB_MODE: + ret_mode = atom_cfb_mode; + break; + + case EVP_CIPH_OFB_MODE: + ret_mode = atom_ofb_mode; + break; + +#ifdef EVP_CIPH_CTR_MODE + case EVP_CIPH_CTR_MODE: + ret_mode = atom_ctr_mode; + break; +#endif + +#ifdef EVP_CIPH_GCM_MODE + case EVP_CIPH_GCM_MODE: + ret_mode = atom_gcm_mode; + break; +#endif + +#ifdef EVP_CIPH_CCM_MODE + case EVP_CIPH_CCM_MODE: + ret_mode = atom_ccm_mode; + break; +#endif + +#ifdef EVP_CIPH_XTS_MODE + case EVP_CIPH_XTS_MODE: + ret_mode = atom_xts_mode; + break; +#endif + +#ifdef EVP_CIPH_WRAP_MODE + case EVP_CIPH_WRAP_MODE: + ret_mode = atom_wrap_mode; + break; +#endif + +#ifdef EVP_CIPH_OCB_MODE + case EVP_CIPH_OCB_MODE: + ret_mode = atom_ocb_mode; + break; +#endif + + case EVP_CIPH_STREAM_CIPHER: + ret_mode = atom_stream_cipher; + break; + + default: + ret_mode = atom_undefined; + break; + } + + enif_make_map_put(env, ret, atom_mode, ret_mode, &ret); + + return ret; +} + +const struct cipher_type_t* get_cipher_type_no_key(ERL_NIF_TERM type) +{ + struct cipher_type_t key; + + key.type.atom = type; + + return bsearch(&key, cipher_types, num_cipher_types, sizeof(cipher_types[0]), cmp_cipher_types_no_key); +} + +int cmp_cipher_types_no_key(const void *keyp, const void *elemp) { + const struct cipher_type_t *key = keyp; + const struct cipher_type_t *elem = elemp; + + if (key->type.atom < elem->type.atom) return -1; + else if (key->type.atom > elem->type.atom) return 1; + else /* key->type.atom == elem->type.atom */ return 0; +} + + +ERL_NIF_TERM cipher_types_as_list(ErlNifEnv* env) +{ + struct cipher_type_t* p; + ERL_NIF_TERM prev, hd; + + hd = enif_make_list(env, 0); + prev = atom_undefined; + + for (p = cipher_types; (p->type.atom & (p->type.atom != atom_false)); p++) { + if ((prev == p->type.atom) || + FORBIDDEN_IN_FIPS(p) ) + continue; + + if ((p->cipher.p != NULL) || + (p->type.atom == atom_aes_ige256)) /* Special handling. Bad indeed... */ + { + hd = enif_make_list_cell(env, p->type.atom, hd); + } + } + + return hd; +} diff --git a/lib/crypto/c_src/cipher.h b/lib/crypto/c_src/cipher.h new file mode 100644 index 0000000000..0e51c410eb --- /dev/null +++ b/lib/crypto/c_src/cipher.h @@ -0,0 +1,84 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_CIPHER_H__ +#define E_CIPHER_H__ 1 + +#include "common.h" + +struct cipher_type_t { + union { + const char* str; /* before init */ + ERL_NIF_TERM atom; /* after init */ + }type; + union { + const EVP_CIPHER* (*funcp)(void); /* before init, NULL if notsup */ + const EVP_CIPHER* p; /* after init, NULL if notsup */ + }cipher; + size_t key_len; /* != 0 to also match on key_len */ + unsigned flags; + union { + struct aead_ctrl {int ctx_ctrl_set_ivlen, ctx_ctrl_get_tag, ctx_ctrl_set_tag;} aead; + } extra; +}; + +/* masks in the flags field if cipher_type_t */ +#define NO_FIPS_CIPHER 1 +#define AES_CFBx 2 +#define ECB_BUG_0_9_8L 4 +#define AEAD_CIPHER 8 +#define NON_EVP_CIPHER 16 +#define AES_CTR_COMPAT 32 +#define CCM_MODE 64 +#define GCM_MODE 128 + + +#ifdef FIPS_SUPPORT +/* May have FIPS support, must check dynamically if it is enabled */ +# define FORBIDDEN_IN_FIPS(P) (((P)->flags & NO_FIPS_CIPHER) && FIPS_mode()) +#else +/* No FIPS support since the symbol FIPS_SUPPORT is undefined */ +# define FORBIDDEN_IN_FIPS(P) 0 +#endif + +extern ErlNifResourceType* evp_cipher_ctx_rtype; +struct evp_cipher_ctx { + EVP_CIPHER_CTX* ctx; + int iv_len; +#if !defined(HAVE_EVP_AES_CTR) + ErlNifEnv* env; + ERL_NIF_TERM state; +#endif +}; + +ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +int init_cipher_ctx(ErlNifEnv *env); + +void init_cipher_types(ErlNifEnv* env); +const struct cipher_type_t* get_cipher_type_no_key(ERL_NIF_TERM type); +const struct cipher_type_t* get_cipher_type(ERL_NIF_TERM type, size_t key_len); + +int cmp_cipher_types(const void *keyp, const void *elemp); +int cmp_cipher_types_no_key(const void *keyp, const void *elemp); + +ERL_NIF_TERM cipher_types_as_list(ErlNifEnv* env); + +#endif /* E_CIPHER_H__ */ diff --git a/lib/crypto/c_src/cmac.c b/lib/crypto/c_src/cmac.c new file mode 100644 index 0000000000..49e67ccf29 --- /dev/null +++ b/lib/crypto/c_src/cmac.c @@ -0,0 +1,88 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "cmac.h" +#include "cipher.h" + +ERL_NIF_TERM cmac_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type, Key, Data) */ +#if defined(HAVE_CMAC) + const struct cipher_type_t *cipherp; + const EVP_CIPHER *cipher; + CMAC_CTX *ctx = NULL; + ErlNifBinary key; + ErlNifBinary data; + ERL_NIF_TERM ret; + size_t ret_size; + unsigned char *outp; + int cipher_len; + + ASSERT(argc == 3); + + if (!enif_inspect_iolist_as_binary(env, argv[1], &key)) + goto bad_arg; + if ((cipherp = get_cipher_type(argv[0], key.size)) == NULL) + goto bad_arg; + if (cipherp->flags & (NON_EVP_CIPHER | AEAD_CIPHER)) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[2], &data)) + goto bad_arg; + + if (FORBIDDEN_IN_FIPS(cipherp)) + return enif_raise_exception(env, atom_notsup); + if ((cipher = cipherp->cipher.p) == NULL) + return enif_raise_exception(env, atom_notsup); + + if ((ctx = CMAC_CTX_new()) == NULL) + goto err; + if (!CMAC_Init(ctx, key.data, key.size, cipher, NULL)) + goto err; + if (!CMAC_Update(ctx, data.data, data.size)) + goto err; + if ((cipher_len = EVP_CIPHER_block_size(cipher)) < 0) + goto err; + if ((outp = enif_make_new_binary(env, (size_t)cipher_len, &ret)) == NULL) + goto err; + if (!CMAC_Final(ctx, outp, &ret_size)) + goto err; + + ASSERT(ret_size == (unsigned)EVP_CIPHER_block_size(cipher)); + CONSUME_REDS(env, data); + goto done; + + bad_arg: + return enif_make_badarg(env); + + err: + ret = atom_notsup; + + done: + if (ctx) + CMAC_CTX_free(ctx); + return ret; + +#else + /* The CMAC functionality was introduced in OpenSSL 1.0.1 + * Although OTP requires at least version 0.9.8, the versions 0.9.8 and 1.0.0 are + * no longer maintained. */ + return atom_notsup; +#endif +} + diff --git a/lib/crypto/c_src/cmac.h b/lib/crypto/c_src/cmac.h new file mode 100644 index 0000000000..14488def58 --- /dev/null +++ b/lib/crypto/c_src/cmac.h @@ -0,0 +1,28 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_CMAC_H__ +#define E_CMAC_H__ 1 + +#include "common.h" + +ERL_NIF_TERM cmac_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_CMAC_H__ */ diff --git a/lib/crypto/c_src/common.h b/lib/crypto/c_src/common.h new file mode 100644 index 0000000000..a7e59d5d01 --- /dev/null +++ b/lib/crypto/c_src/common.h @@ -0,0 +1,52 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_COMMON_H__ +#define E_COMMON_H__ 1 + +#ifdef __WIN32__ +# include <windows.h> +#endif + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <limits.h> +#include <stdint.h> + +#include <erl_nif.h> +#include "openssl_config.h" +#include "atoms.h" + + +/* All nif functions return a valid value or throws an exception */ +#define EXCP(Env, Id, Str) enif_raise_exception((Env), \ + enif_make_tuple3((Env), \ + (Id), \ + enif_make_tuple2((Env), \ + enif_make_string((Env),__FILE__,(ERL_NIF_LATIN1)), \ + enif_make_int((Env), __LINE__)), \ + enif_make_string((Env),(Str),(ERL_NIF_LATIN1)) )) + +#define EXCP_NOTSUP(Env, Str) EXCP((Env), atom_notsup, (Str)) +#define EXCP_BADARG(Env, Str) EXCP((Env), atom_badarg, (Str)) +#define EXCP_ERROR(Env, Str) EXCP((Env), atom_error, (Str)) + +#endif /* E_COMMON_H__ */ diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index 194a3d30e9..d533cba140 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -23,983 +23,116 @@ * Based on OpenSSL. */ -#ifdef __WIN32__ - #include <windows.h> -#endif - -#include <stdlib.h> -#include <stdio.h> -#include <string.h> - -#include <erl_nif.h> - -#define OPENSSL_THREAD_DEFINES -#include <openssl/opensslconf.h> - -#include <openssl/crypto.h> -#ifndef OPENSSL_NO_DES -#include <openssl/des.h> -#endif /* #ifndef OPENSSL_NO_DES */ -/* #include <openssl/idea.h> This is not supported on the openssl OTP requires */ -#include <openssl/dsa.h> -#include <openssl/rsa.h> -#include <openssl/aes.h> -#include <openssl/md5.h> -#include <openssl/md4.h> -#include <openssl/sha.h> -#include <openssl/ripemd.h> -#include <openssl/bn.h> -#include <openssl/objects.h> -#ifndef OPENSSL_NO_RC4 - #include <openssl/rc4.h> -#endif /* OPENSSL_NO_RC4 */ -#ifndef OPENSSL_NO_RC2 - #include <openssl/rc2.h> -#endif -#include <openssl/blowfish.h> -#include <openssl/rand.h> -#include <openssl/evp.h> -#include <openssl/hmac.h> -#include <openssl/err.h> - -/* Helper macro to construct a OPENSSL_VERSION_NUMBER. - * See openssl/opensslv.h - */ -#define PACKED_OPENSSL_VERSION(MAJ, MIN, FIX, P) \ - ((((((((MAJ << 8) | MIN) << 8 ) | FIX) << 8) | (P-'a'+1)) << 4) | 0xf) - -#define PACKED_OPENSSL_VERSION_PLAIN(MAJ, MIN, FIX) \ - PACKED_OPENSSL_VERSION(MAJ,MIN,FIX,('a'-1)) - - -/* LibreSSL was cloned from OpenSSL 1.0.1g and claims to be API and BPI compatible - * with 1.0.1. - * - * LibreSSL has the same names on include files and symbols as OpenSSL, but defines - * the OPENSSL_VERSION_NUMBER to be >= 2.0.0 - * - * Therefor works tests like this as intendend: - * OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) - * (The test is for example "2.4.2" >= "1.0.0" although the test - * with the cloned OpenSSL test would be "1.0.1" >= "1.0.0") - * - * But tests like this gives wrong result: - * OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) - * (The test is false since "2.4.2" < "1.1.0". It should have been - * true because the LibreSSL API version is "1.0.1") - * - */ - -#ifdef LIBRESSL_VERSION_NUMBER -/* A macro to test on in this file */ -#define HAS_LIBRESSL -#endif - -#ifdef HAS_LIBRESSL -/* LibreSSL dislikes FIPS */ -# ifdef FIPS_SUPPORT -# undef FIPS_SUPPORT -# endif - -# if LIBRESSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(2,7,0) -/* LibreSSL wants the 1.0.1 API */ -# define NEED_EVP_COMPATIBILITY_FUNCTIONS -# endif -#endif - - -#if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) -# define NEED_EVP_COMPATIBILITY_FUNCTIONS -#endif - - -#ifndef HAS_LIBRESSL -# if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) -# define HAS_EVP_PKEY_CTX -# endif -#endif - - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) -#include <openssl/modes.h> -#endif - -#include "crypto_callback.h" - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(0,9,8) \ - && !defined(OPENSSL_NO_SHA224) && defined(NID_sha224) \ - && !defined(OPENSSL_NO_SHA256) /* disabled like this in my sha.h (?) */ -# define HAVE_SHA224 -#endif -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(0,9,8) \ - && !defined(OPENSSL_NO_SHA256) && defined(NID_sha256) -# define HAVE_SHA256 -#endif -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(0,9,8) \ - && !defined(OPENSSL_NO_SHA384) && defined(NID_sha384)\ - && !defined(OPENSSL_NO_SHA512) /* disabled like this in my sha.h (?) */ -# define HAVE_SHA384 -#endif -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(0,9,8) \ - && !defined(OPENSSL_NO_SHA512) && defined(NID_sha512) -# define HAVE_SHA512 -#endif -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(0,9,7,'e') -# define HAVE_DES_ede3_cfb_encrypt -#endif - -// SHA3: -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,1,1) -// An error in beta releases of 1.1.1 fixed in production release -# ifdef NID_sha3_224 -# define HAVE_SHA3_224 -# endif -# ifdef NID_sha3_256 -# define HAVE_SHA3_256 -# endif -#endif -# ifdef NID_sha3_384 -# define HAVE_SHA3_384 -# endif -# ifdef NID_sha3_512 -# define HAVE_SHA3_512 -# endif - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(0,9,8,'o') \ - && !defined(OPENSSL_NO_EC) \ - && !defined(OPENSSL_NO_ECDH) \ - && !defined(OPENSSL_NO_ECDSA) -# define HAVE_EC -#endif - -// (test for >= 1.1.1pre8) -#if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1) -7) \ - && !defined(HAS_LIBRESSL) \ - && defined(HAVE_EC) -# define HAVE_ED_CURVE_DH -# if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)) \ - && !defined(FIPS_SUPPORT) -# define HAVE_EDDSA -# endif -#endif - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(0,9,8,'c') -# define HAVE_AES_IGE -#endif - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,1) -# define HAVE_EVP_AES_CTR -# define HAVE_AEAD -# define HAVE_GCM -# define HAVE_CCM -# define HAVE_CMAC -# if defined(RSA_PKCS1_OAEP_PADDING) -# define HAVE_RSA_OAEP_PADDING -# endif -# define HAVE_RSA_MGF1_MD -# if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION(1,0,1,'d') -# define HAVE_GCM_EVP_DECRYPT_BUG -# endif -#endif - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,1,0) -# ifndef HAS_LIBRESSL -# define HAVE_CHACHA20 -# define HAVE_CHACHA20_POLY1305 -# define HAVE_RSA_OAEP_MD -# endif -#endif - -// OPENSSL_VERSION_NUMBER >= 1.1.1-pre8 -#if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)-7) -# ifndef HAS_LIBRESSL -# define HAVE_POLY1305 -# endif -#endif - -#if OPENSSL_VERSION_NUMBER <= PACKED_OPENSSL_VERSION(0,9,8,'l') -# define HAVE_ECB_IVEC_BUG -#endif - -#ifndef HAS_LIBRESSL -# ifdef RSA_SSLV23_PADDING -# define HAVE_RSA_SSLV23_PADDING -# endif -#endif - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) -# ifdef RSA_PKCS1_PSS_PADDING -# define HAVE_RSA_PKCS1_PSS_PADDING -# endif -#endif - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(0,9,8,'h') \ - && defined(HAVE_EC) -/* If OPENSSL_NO_EC is set, there will be an error in ec.h included from engine.h - So if EC is disabled, you can't use Engine either.... -*/ -# define HAS_ENGINE_SUPPORT -#endif - - -#if defined(HAS_ENGINE_SUPPORT) -# include <openssl/engine.h> -#endif - -#if defined(HAVE_CMAC) -#include <openssl/cmac.h> -#endif - -#if defined(HAVE_EC) -#include <openssl/ec.h> -#include <openssl/ecdh.h> -#include <openssl/ecdsa.h> -#endif - -#ifdef VALGRIND - # include <valgrind/memcheck.h> - -/* libcrypto mixes supplied buffer contents into its entropy pool, - which makes valgrind complain about the use of uninitialized data. - We use this valgrind "request" to make sure that no such seemingly - undefined data is returned. -*/ - # define ERL_VALGRIND_MAKE_MEM_DEFINED(ptr,size) \ - VALGRIND_MAKE_MEM_DEFINED(ptr,size) - - # define ERL_VALGRIND_ASSERT_MEM_DEFINED(Ptr,Size) \ - do { \ - int __erl_valgrind_mem_defined = VALGRIND_CHECK_MEM_IS_DEFINED((Ptr),(Size)); \ - if (__erl_valgrind_mem_defined != 0) { \ - fprintf(stderr,"\r\n####### VALGRIND_ASSSERT(%p,%ld) failed at %s:%d\r\n", \ - (Ptr),(long)(Size), __FILE__, __LINE__); \ - abort(); \ - } \ - } while (0) - -#else - # define ERL_VALGRIND_MAKE_MEM_DEFINED(ptr,size) - # define ERL_VALGRIND_ASSERT_MEM_DEFINED(ptr,size) -#endif - -#ifdef DEBUG - # define ASSERT(e) \ - ((void) ((e) ? 1 : (fprintf(stderr,"Assert '%s' failed at %s:%d\n",\ - #e, __FILE__, __LINE__), abort(), 0))) -#else - # define ASSERT(e) ((void) 1) -#endif - -#ifdef __GNUC__ - # define INLINE __inline__ -#elif defined(__WIN32__) - # define INLINE __forceinline -#else - # define INLINE -#endif - - -#define get_int32(s) ((((unsigned char*) (s))[0] << 24) | \ - (((unsigned char*) (s))[1] << 16) | \ - (((unsigned char*) (s))[2] << 8) | \ - (((unsigned char*) (s))[3])) - -#define put_int32(s,i) \ -{ (s)[0] = (char)(((i) >> 24) & 0xff);\ - (s)[1] = (char)(((i) >> 16) & 0xff);\ - (s)[2] = (char)(((i) >> 8) & 0xff);\ - (s)[3] = (char)((i) & 0xff);\ -} - -/* This shall correspond to the similar macro in crypto.erl */ -/* Current value is: erlang:system_info(context_reductions) * 10 */ -#define MAX_BYTES_TO_NIF 20000 - -#define CONSUME_REDS(NifEnv, Ibin) \ -do { \ - int _cost = ((Ibin).size * 100) / MAX_BYTES_TO_NIF;\ - if (_cost) { \ - (void) enif_consume_timeslice((NifEnv), \ - (_cost > 100) ? 100 : _cost); \ - } \ - } while (0) - - -#ifdef NEED_EVP_COMPATIBILITY_FUNCTIONS -/* - * In OpenSSL 1.1.0, most structs are opaque. That means that - * the structs cannot be allocated as automatic variables on the - * C stack (because the size is unknown) and that it is necessary - * to use access functions. - * - * For backward compatibility to previous versions of OpenSSL, define - * on our versions of the new functions defined in 1.1.0 here, so that - * we don't have to sprinkle ifdefs throughout the code. - */ - -static HMAC_CTX *HMAC_CTX_new(void); -static void HMAC_CTX_free(HMAC_CTX *ctx); - -static HMAC_CTX *HMAC_CTX_new() -{ - HMAC_CTX *ctx = CRYPTO_malloc(sizeof(HMAC_CTX), __FILE__, __LINE__); - HMAC_CTX_init(ctx); - return ctx; -} - -static void HMAC_CTX_free(HMAC_CTX *ctx) -{ - HMAC_CTX_cleanup(ctx); - CRYPTO_free(ctx); -} - -#define EVP_MD_CTX_new() EVP_MD_CTX_create() -#define EVP_MD_CTX_free(ctx) EVP_MD_CTX_destroy(ctx) - -static INLINE void *BN_GENCB_get_arg(BN_GENCB *cb); - -static INLINE void *BN_GENCB_get_arg(BN_GENCB *cb) -{ - return cb->arg; -} - -static INLINE int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d); -static INLINE void RSA_get0_key(const RSA *r, const BIGNUM **n, const BIGNUM **e, const BIGNUM **d); -static INLINE int RSA_set0_factors(RSA *r, BIGNUM *p, BIGNUM *q); -static INLINE void RSA_get0_factors(const RSA *r, const BIGNUM **p, const BIGNUM **q); -static INLINE int RSA_set0_crt_params(RSA *r, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp); -static INLINE void RSA_get0_crt_params(const RSA *r, const BIGNUM **dmp1, const BIGNUM **dmq1, const BIGNUM **iqmp); - -static INLINE int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d) -{ - r->n = n; - r->e = e; - r->d = d; - return 1; -} - -static INLINE void RSA_get0_key(const RSA *r, const BIGNUM **n, const BIGNUM **e, const BIGNUM **d) -{ - *n = r->n; - *e = r->e; - *d = r->d; -} - -static INLINE int RSA_set0_factors(RSA *r, BIGNUM *p, BIGNUM *q) -{ - r->p = p; - r->q = q; - return 1; -} - -static INLINE void RSA_get0_factors(const RSA *r, const BIGNUM **p, const BIGNUM **q) -{ - *p = r->p; - *q = r->q; -} - -static INLINE int RSA_set0_crt_params(RSA *r, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp) -{ - r->dmp1 = dmp1; - r->dmq1 = dmq1; - r->iqmp = iqmp; - return 1; -} - -static INLINE void RSA_get0_crt_params(const RSA *r, const BIGNUM **dmp1, const BIGNUM **dmq1, const BIGNUM **iqmp) -{ - *dmp1 = r->dmp1; - *dmq1 = r->dmq1; - *iqmp = r->iqmp; -} - -static INLINE int DSA_set0_key(DSA *d, BIGNUM *pub_key, BIGNUM *priv_key); -static INLINE int DSA_set0_pqg(DSA *d, BIGNUM *p, BIGNUM *q, BIGNUM *g); -static INLINE void DSA_get0_pqg(const DSA *dsa, - const BIGNUM **p, const BIGNUM **q, const BIGNUM **g); -static INLINE void DSA_get0_key(const DSA *dsa, - const BIGNUM **pub_key, const BIGNUM **priv_key); - -static INLINE int DSA_set0_key(DSA *d, BIGNUM *pub_key, BIGNUM *priv_key) -{ - d->pub_key = pub_key; - d->priv_key = priv_key; - return 1; -} - -static INLINE int DSA_set0_pqg(DSA *d, BIGNUM *p, BIGNUM *q, BIGNUM *g) -{ - d->p = p; - d->q = q; - d->g = g; - return 1; -} - -static INLINE void -DSA_get0_pqg(const DSA *dsa, const BIGNUM **p, const BIGNUM **q, const BIGNUM **g) -{ - *p = dsa->p; - *q = dsa->q; - *g = dsa->g; -} - -static INLINE void -DSA_get0_key(const DSA *dsa, const BIGNUM **pub_key, const BIGNUM **priv_key) -{ - if (pub_key) *pub_key = dsa->pub_key; - if (priv_key) *priv_key = dsa->priv_key; -} - - - -static INLINE int DH_set0_key(DH *dh, BIGNUM *pub_key, BIGNUM *priv_key); -static INLINE int DH_set0_pqg(DH *dh, BIGNUM *p, BIGNUM *q, BIGNUM *g); -static INLINE int DH_set_length(DH *dh, long length); -static INLINE void DH_get0_pqg(const DH *dh, - const BIGNUM **p, const BIGNUM **q, const BIGNUM **g); -static INLINE void DH_get0_key(const DH *dh, - const BIGNUM **pub_key, const BIGNUM **priv_key); - -static INLINE int DH_set0_key(DH *dh, BIGNUM *pub_key, BIGNUM *priv_key) -{ - dh->pub_key = pub_key; - dh->priv_key = priv_key; - return 1; -} - -static INLINE int DH_set0_pqg(DH *dh, BIGNUM *p, BIGNUM *q, BIGNUM *g) -{ - dh->p = p; - dh->q = q; - dh->g = g; - return 1; -} - -static INLINE int DH_set_length(DH *dh, long length) -{ - dh->length = length; - return 1; -} - - - -static INLINE void -DH_get0_pqg(const DH *dh, const BIGNUM **p, const BIGNUM **q, const BIGNUM **g) -{ - *p = dh->p; - *q = dh->q; - *g = dh->g; -} - -static INLINE void -DH_get0_key(const DH *dh, const BIGNUM **pub_key, const BIGNUM **priv_key) -{ - if (pub_key) *pub_key = dh->pub_key; - if (priv_key) *priv_key = dh->priv_key; -} - -#else /* End of compatibility definitions. */ - -#define HAVE_OPAQUE_BN_GENCB - -#endif +#include "common.h" + +#include "aead.h" +#include "aes.h" +#include "algorithms.h" +#include "api_ng.h" +#include "bn.h" +#include "cipher.h" +#include "cmac.h" +#include "dh.h" +#include "digest.h" +#include "dss.h" +#include "ec.h" +#include "ecdh.h" +#include "eddsa.h" +#include "engine.h" +#include "evp.h" +#include "fips.h" +#include "hash.h" +#include "hmac.h" +#include "info.h" +#include "math.h" +#include "pkey.h" +#include "poly1305.h" +#include "rand.h" +#include "rsa.h" +#include "srp.h" /* NIF interface declarations */ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info); static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); static void unload(ErlNifEnv* env, void* priv_data); -/* The NIFs: */ -static ERL_NIF_TERM info_lib(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM info_fips(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM enable_fips_mode(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM hash_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM hmac_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM hmac_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM hmac_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM hmac_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM cmac_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM aes_cfb_8_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM aes_cfb_128_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM aes_ctr_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM aes_ctr_stream_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM strong_rand_bytes_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM strong_rand_range_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM rand_uniform_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM do_exor(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM rc4_set_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM rc4_encrypt_with_state(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM pkey_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM pkey_verify_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM pkey_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM rsa_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM dh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM privkey_to_pubkey_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM srp_value_B_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM srp_user_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM srp_host_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -static ERL_NIF_TERM ec_key_generate(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM ecdh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -static ERL_NIF_TERM evp_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM evp_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -static ERL_NIF_TERM rand_seed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -static ERL_NIF_TERM aead_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM aead_decrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -#ifdef HAVE_GCM_EVP_DECRYPT_BUG -static ERL_NIF_TERM aes_gcm_decrypt_NO_EVP(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -#endif - -static ERL_NIF_TERM chacha20_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM chacha20_stream_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -static ERL_NIF_TERM poly1305_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -static ERL_NIF_TERM engine_by_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_finish_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_free_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_load_dynamic_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_ctrl_cmd_strings_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_register_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_unregister_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_add_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_remove_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_get_first_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_get_next_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_get_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_get_name_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM engine_get_all_methods_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -/* helpers */ -static void init_algorithms_types(ErlNifEnv*); -static void init_digest_types(ErlNifEnv* env); -static void init_cipher_types(ErlNifEnv* env); -#ifdef HAVE_EC -static EC_KEY* ec_key_new(ErlNifEnv* env, ERL_NIF_TERM curve_arg); -static int term2point(ErlNifEnv* env, ERL_NIF_TERM term, - EC_GROUP *group, EC_POINT **pptr); -#endif -static ERL_NIF_TERM bin_from_bn(ErlNifEnv* env, const BIGNUM *bn); - -#ifdef HAS_ENGINE_SUPPORT -static int get_engine_load_cmd_list(ErlNifEnv* env, const ERL_NIF_TERM term, char **cmds, int i); -static int zero_terminate(ErlNifBinary bin, char **buf); -#endif - static int library_refc = 0; /* number of users of this dynamic library */ static int library_initialized = 0; static ErlNifFunc nif_funcs[] = { - {"info_lib", 0, info_lib}, - {"info_fips", 0, info_fips}, - {"enable_fips_mode", 1, enable_fips_mode}, - {"algorithms", 0, algorithms}, - {"hash_nif", 2, hash_nif}, - {"hash_init_nif", 1, hash_init_nif}, - {"hash_update_nif", 2, hash_update_nif}, - {"hash_final_nif", 1, hash_final_nif}, - {"hmac_nif", 3, hmac_nif}, - {"hmac_nif", 4, hmac_nif}, - {"hmac_init_nif", 2, hmac_init_nif}, - {"hmac_update_nif", 2, hmac_update_nif}, - {"hmac_final_nif", 1, hmac_final_nif}, - {"hmac_final_nif", 2, hmac_final_nif}, - {"cmac_nif", 3, cmac_nif}, - {"block_crypt_nif", 5, block_crypt_nif}, - {"block_crypt_nif", 4, block_crypt_nif}, - {"aes_ige_crypt_nif", 4, aes_ige_crypt_nif}, - {"aes_ctr_stream_init", 2, aes_ctr_stream_init}, - {"aes_ctr_stream_encrypt", 2, aes_ctr_stream_encrypt}, - {"aes_ctr_stream_decrypt", 2, aes_ctr_stream_encrypt}, - {"strong_rand_bytes_nif", 1, strong_rand_bytes_nif}, - {"strong_rand_range_nif", 1, strong_rand_range_nif}, - {"rand_uniform_nif", 2, rand_uniform_nif}, - {"mod_exp_nif", 4, mod_exp_nif}, - {"do_exor", 2, do_exor}, - {"rc4_set_key", 1, rc4_set_key}, - {"rc4_encrypt_with_state", 2, rc4_encrypt_with_state}, - {"pkey_sign_nif", 5, pkey_sign_nif}, - {"pkey_verify_nif", 6, pkey_verify_nif}, - {"pkey_crypt_nif", 6, pkey_crypt_nif}, - {"rsa_generate_key_nif", 2, rsa_generate_key_nif}, - {"dh_generate_key_nif", 4, dh_generate_key_nif}, - {"dh_compute_key_nif", 3, dh_compute_key_nif}, - {"evp_compute_key_nif", 3, evp_compute_key_nif}, - {"evp_generate_key_nif", 1, evp_generate_key_nif}, - {"privkey_to_pubkey_nif", 2, privkey_to_pubkey_nif}, - {"srp_value_B_nif", 5, srp_value_B_nif}, - {"srp_user_secret_nif", 7, srp_user_secret_nif}, - {"srp_host_secret_nif", 5, srp_host_secret_nif}, - - {"ec_key_generate", 2, ec_key_generate}, - {"ecdh_compute_key_nif", 3, ecdh_compute_key_nif}, - - {"rand_seed_nif", 1, rand_seed_nif}, - - {"aead_encrypt", 6, aead_encrypt}, - {"aead_decrypt", 6, aead_decrypt}, - - {"chacha20_stream_init", 2, chacha20_stream_init}, - {"chacha20_stream_encrypt", 2, chacha20_stream_crypt}, - {"chacha20_stream_decrypt", 2, chacha20_stream_crypt}, - - {"poly1305_nif", 2, poly1305_nif}, - - {"engine_by_id_nif", 1, engine_by_id_nif}, - {"engine_init_nif", 1, engine_init_nif}, - {"engine_finish_nif", 1, engine_finish_nif}, - {"engine_free_nif", 1, engine_free_nif}, - {"engine_load_dynamic_nif", 0, engine_load_dynamic_nif}, - {"engine_ctrl_cmd_strings_nif", 3, engine_ctrl_cmd_strings_nif}, - {"engine_register_nif", 2, engine_register_nif}, - {"engine_unregister_nif", 2, engine_unregister_nif}, - {"engine_add_nif", 1, engine_add_nif}, - {"engine_remove_nif", 1, engine_remove_nif}, - {"engine_get_first_nif", 0, engine_get_first_nif}, - {"engine_get_next_nif", 1, engine_get_next_nif}, - {"engine_get_id_nif", 1, engine_get_id_nif}, - {"engine_get_name_nif", 1, engine_get_name_nif}, - {"engine_get_all_methods_nif", 0, engine_get_all_methods_nif} - + {"info_lib", 0, info_lib, 0}, + {"info_fips", 0, info_fips, 0}, + {"enable_fips_mode", 1, enable_fips_mode, 0}, + {"hash_algorithms", 0, hash_algorithms, 0}, + {"pubkey_algorithms", 0, pubkey_algorithms, 0}, + {"cipher_algorithms", 0, cipher_algorithms, 0}, + {"mac_algorithms", 0, mac_algorithms, 0}, + {"curve_algorithms", 0, curve_algorithms, 0}, + {"rsa_opts_algorithms", 0, rsa_opts_algorithms, 0}, + {"hash_info", 1, hash_info_nif, 0}, + {"hash_nif", 2, hash_nif, 0}, + {"hash_init_nif", 1, hash_init_nif, 0}, + {"hash_update_nif", 2, hash_update_nif, 0}, + {"hash_final_nif", 1, hash_final_nif, 0}, + {"hmac_nif", 3, hmac_nif, 0}, + {"hmac_nif", 4, hmac_nif, 0}, + {"hmac_init_nif", 2, hmac_init_nif, 0}, + {"hmac_update_nif", 2, hmac_update_nif, 0}, + {"hmac_final_nif", 1, hmac_final_nif, 0}, + {"hmac_final_nif", 2, hmac_final_nif, 0}, + {"cmac_nif", 3, cmac_nif, 0}, + {"cipher_info_nif", 1, cipher_info_nif, 0}, + {"aes_ige_crypt_nif", 4, aes_ige_crypt_nif, 0}, + {"ng_crypto_init_nif", 4, ng_crypto_init_nif, 0}, + {"ng_crypto_update_nif", 2, ng_crypto_update_nif, 0}, + {"ng_crypto_update_nif", 3, ng_crypto_update_nif, 0}, + {"ng_crypto_one_time_nif", 5, ng_crypto_one_time_nif, 0}, + {"strong_rand_bytes_nif", 1, strong_rand_bytes_nif, 0}, + {"strong_rand_range_nif", 1, strong_rand_range_nif, 0}, + {"rand_uniform_nif", 2, rand_uniform_nif, 0}, + {"mod_exp_nif", 4, mod_exp_nif, 0}, + {"do_exor", 2, do_exor, 0}, + {"pkey_sign_nif", 5, pkey_sign_nif, 0}, + {"pkey_verify_nif", 6, pkey_verify_nif, 0}, + {"pkey_crypt_nif", 6, pkey_crypt_nif, 0}, + {"rsa_generate_key_nif", 2, rsa_generate_key_nif, 0}, + {"dh_generate_key_nif", 4, dh_generate_key_nif, 0}, + {"dh_compute_key_nif", 3, dh_compute_key_nif, 0}, + {"evp_compute_key_nif", 3, evp_compute_key_nif, 0}, + {"evp_generate_key_nif", 1, evp_generate_key_nif, 0}, + {"privkey_to_pubkey_nif", 2, privkey_to_pubkey_nif, 0}, + {"srp_value_B_nif", 5, srp_value_B_nif, 0}, + {"srp_user_secret_nif", 7, srp_user_secret_nif, 0}, + {"srp_host_secret_nif", 5, srp_host_secret_nif, 0}, + + {"ec_key_generate", 2, ec_key_generate, 0}, + {"ecdh_compute_key_nif", 3, ecdh_compute_key_nif, 0}, + + {"rand_seed_nif", 1, rand_seed_nif, 0}, + + {"aead_cipher", 7, aead_cipher, 0}, + + {"poly1305_nif", 2, poly1305_nif, 0}, + + {"engine_by_id_nif", 1, engine_by_id_nif, 0}, + {"engine_init_nif", 1, engine_init_nif, 0}, + {"engine_finish_nif", 1, engine_finish_nif, 0}, + {"engine_free_nif", 1, engine_free_nif, 0}, + {"engine_load_dynamic_nif", 0, engine_load_dynamic_nif, 0}, + {"engine_ctrl_cmd_strings_nif", 3, engine_ctrl_cmd_strings_nif, 0}, + {"engine_register_nif", 2, engine_register_nif, 0}, + {"engine_unregister_nif", 2, engine_unregister_nif, 0}, + {"engine_add_nif", 1, engine_add_nif, 0}, + {"engine_remove_nif", 1, engine_remove_nif, 0}, + {"engine_get_first_nif", 0, engine_get_first_nif, 0}, + {"engine_get_next_nif", 1, engine_get_next_nif, 0}, + {"engine_get_id_nif", 1, engine_get_id_nif, 0}, + {"engine_get_name_nif", 1, engine_get_name_nif, 0}, + {"engine_get_all_methods_nif", 0, engine_get_all_methods_nif, 0} }; ERL_NIF_INIT(crypto,nif_funcs,load,NULL,upgrade,unload) -#define MD5_CTX_LEN (sizeof(MD5_CTX)) -#define MD4_CTX_LEN (sizeof(MD4_CTX)) -#define RIPEMD160_CTX_LEN (sizeof(RIPEMD160_CTX)) - - -static ERL_NIF_TERM atom_true; -static ERL_NIF_TERM atom_false; -static ERL_NIF_TERM atom_sha; -static ERL_NIF_TERM atom_error; -static ERL_NIF_TERM atom_rsa_pkcs1_padding; -static ERL_NIF_TERM atom_rsa_pkcs1_oaep_padding; -static ERL_NIF_TERM atom_rsa_no_padding; -static ERL_NIF_TERM atom_signature_md; -static ERL_NIF_TERM atom_undefined; - -static ERL_NIF_TERM atom_ok; -static ERL_NIF_TERM atom_not_prime; -static ERL_NIF_TERM atom_not_strong_prime; -static ERL_NIF_TERM atom_unable_to_check_generator; -static ERL_NIF_TERM atom_not_suitable_generator; -static ERL_NIF_TERM atom_check_failed; -static ERL_NIF_TERM atom_unknown; -static ERL_NIF_TERM atom_none; -static ERL_NIF_TERM atom_notsup; -static ERL_NIF_TERM atom_digest; -#ifdef FIPS_SUPPORT -static ERL_NIF_TERM atom_enabled; -static ERL_NIF_TERM atom_not_enabled; -#else -static ERL_NIF_TERM atom_not_supported; -#endif - -#if defined(HAVE_EC) -static ERL_NIF_TERM atom_ec; -static ERL_NIF_TERM atom_prime_field; -static ERL_NIF_TERM atom_characteristic_two_field; -static ERL_NIF_TERM atom_tpbasis; -static ERL_NIF_TERM atom_ppbasis; -static ERL_NIF_TERM atom_onbasis; -#endif - -static ERL_NIF_TERM atom_aes_cfb8; -static ERL_NIF_TERM atom_aes_cfb128; -#ifdef HAVE_GCM -static ERL_NIF_TERM atom_aes_gcm; -#endif -#ifdef HAVE_CCM -static ERL_NIF_TERM atom_aes_ccm; -#endif -#ifdef HAVE_CHACHA20_POLY1305 -static ERL_NIF_TERM atom_chacha20_poly1305; -#endif -#ifdef HAVE_ECB_IVEC_BUG -static ERL_NIF_TERM atom_aes_ecb; -static ERL_NIF_TERM atom_des_ecb; -static ERL_NIF_TERM atom_blowfish_ecb; -#endif - -static ERL_NIF_TERM atom_rsa; -static ERL_NIF_TERM atom_dss; -static ERL_NIF_TERM atom_ecdsa; - -#ifdef HAVE_ED_CURVE_DH -static ERL_NIF_TERM atom_x25519; -static ERL_NIF_TERM atom_x448; -#endif - -static ERL_NIF_TERM atom_eddsa; -#ifdef HAVE_EDDSA -static ERL_NIF_TERM atom_ed25519; -static ERL_NIF_TERM atom_ed448; -#endif - -static ERL_NIF_TERM atom_rsa_mgf1_md; -static ERL_NIF_TERM atom_rsa_oaep_label; -static ERL_NIF_TERM atom_rsa_oaep_md; -static ERL_NIF_TERM atom_rsa_pad; /* backwards compatibility */ -static ERL_NIF_TERM atom_rsa_padding; -static ERL_NIF_TERM atom_rsa_pkcs1_pss_padding; -#ifdef HAVE_RSA_SSLV23_PADDING -static ERL_NIF_TERM atom_rsa_sslv23_padding; -#endif -static ERL_NIF_TERM atom_rsa_x931_padding; -static ERL_NIF_TERM atom_rsa_pss_saltlen; -static ERL_NIF_TERM atom_sha224; -static ERL_NIF_TERM atom_sha256; -static ERL_NIF_TERM atom_sha384; -static ERL_NIF_TERM atom_sha512; -static ERL_NIF_TERM atom_sha3_224; -static ERL_NIF_TERM atom_sha3_256; -static ERL_NIF_TERM atom_sha3_384; -static ERL_NIF_TERM atom_sha3_512; -static ERL_NIF_TERM atom_md5; -static ERL_NIF_TERM atom_ripemd160; - -#ifdef HAS_ENGINE_SUPPORT -static ERL_NIF_TERM atom_bad_engine_method; -static ERL_NIF_TERM atom_bad_engine_id; -static ERL_NIF_TERM atom_ctrl_cmd_failed; -static ERL_NIF_TERM atom_engine_init_failed; -static ERL_NIF_TERM atom_register_engine_failed; -static ERL_NIF_TERM atom_add_engine_failed; -static ERL_NIF_TERM atom_remove_engine_failed; -static ERL_NIF_TERM atom_engine_method_not_supported; - -static ERL_NIF_TERM atom_engine_method_rsa; -static ERL_NIF_TERM atom_engine_method_dsa; -static ERL_NIF_TERM atom_engine_method_dh; -static ERL_NIF_TERM atom_engine_method_rand; -static ERL_NIF_TERM atom_engine_method_ecdh; -static ERL_NIF_TERM atom_engine_method_ecdsa; -static ERL_NIF_TERM atom_engine_method_ciphers; -static ERL_NIF_TERM atom_engine_method_digests; -static ERL_NIF_TERM atom_engine_method_store; -static ERL_NIF_TERM atom_engine_method_pkey_meths; -static ERL_NIF_TERM atom_engine_method_pkey_asn1_meths; -static ERL_NIF_TERM atom_engine_method_ec; - -static ERL_NIF_TERM atom_engine; -static ERL_NIF_TERM atom_key_id; -static ERL_NIF_TERM atom_password; -#endif - -static ErlNifResourceType* hmac_context_rtype; -struct hmac_context -{ - ErlNifMutex* mtx; - int alive; - HMAC_CTX* ctx; -}; -static void hmac_context_dtor(ErlNifEnv* env, struct hmac_context*); - -struct digest_type_t { - union { - const char* str; /* before init, NULL for end-of-table */ - ERL_NIF_TERM atom; /* after init, 'false' for end-of-table */ - }type; - union { - const EVP_MD* (*funcp)(void); /* before init, NULL if notsup */ - const EVP_MD* p; /* after init, NULL if notsup */ - }md; -}; - -static struct digest_type_t digest_types[] = -{ - {{"md4"}, {&EVP_md4}}, - {{"md5"}, {&EVP_md5}}, - {{"ripemd160"}, {&EVP_ripemd160}}, - {{"sha"}, {&EVP_sha1}}, - {{"sha224"}, -#ifdef HAVE_SHA224 - {&EVP_sha224} -#else - {NULL} -#endif - }, - {{"sha256"}, -#ifdef HAVE_SHA256 - {&EVP_sha256} -#else - {NULL} -#endif - }, - {{"sha384"}, -#ifdef HAVE_SHA384 - {&EVP_sha384} -#else - {NULL} -#endif - }, - {{"sha512"}, -#ifdef HAVE_SHA512 - {&EVP_sha512} -#else - {NULL} -#endif - }, - {{"sha3_224"}, -#ifdef HAVE_SHA3_224 - {&EVP_sha3_224} -#else - {NULL} -#endif - }, - {{"sha3_256"}, -#ifdef HAVE_SHA3_256 - {&EVP_sha3_256} -#else - {NULL} -#endif - }, - {{"sha3_384"}, -#ifdef HAVE_SHA3_384 - {&EVP_sha3_384} -#else - {NULL} -#endif - }, - {{"sha3_512"}, -#ifdef HAVE_SHA3_512 - {&EVP_sha3_512} -#else - {NULL} -#endif - }, - - {{NULL}} -}; - -static struct digest_type_t* get_digest_type(ERL_NIF_TERM type); - -struct cipher_type_t { - union { - const char* str; /* before init */ - ERL_NIF_TERM atom; /* after init */ - }type; - union { - const EVP_CIPHER* (*funcp)(void); /* before init, NULL if notsup */ - const EVP_CIPHER* p; /* after init, NULL if notsup */ - }cipher; - const size_t key_len; /* != 0 to also match on key_len */ -}; - -#ifdef OPENSSL_NO_DES -#define COND_NO_DES_PTR(Ptr) (NULL) -#else -#define COND_NO_DES_PTR(Ptr) (Ptr) -#endif - -static struct cipher_type_t cipher_types[] = -{ - {{"rc2_cbc"}, -#ifndef OPENSSL_NO_RC2 - {&EVP_rc2_cbc} -#else - {NULL} -#endif - }, - {{"des_cbc"}, {COND_NO_DES_PTR(&EVP_des_cbc)}}, - {{"des_cfb"}, {COND_NO_DES_PTR(&EVP_des_cfb8)}}, - {{"des_ecb"}, {COND_NO_DES_PTR(&EVP_des_ecb)}}, - {{"des_ede3_cbc"}, {COND_NO_DES_PTR(&EVP_des_ede3_cbc)}}, - {{"des_ede3_cbf"}, /* Misspelled, retained */ -#ifdef HAVE_DES_ede3_cfb_encrypt - {COND_NO_DES_PTR(&EVP_des_ede3_cfb8)} -#else - {NULL} -#endif - }, - {{"des_ede3_cfb"}, -#ifdef HAVE_DES_ede3_cfb_encrypt - {COND_NO_DES_PTR(&EVP_des_ede3_cfb8)} -#else - {NULL} -#endif - }, - {{"blowfish_cbc"}, {&EVP_bf_cbc}}, - {{"blowfish_cfb64"}, {&EVP_bf_cfb64}}, - {{"blowfish_ofb64"}, {&EVP_bf_ofb}}, - {{"blowfish_ecb"}, {&EVP_bf_ecb}}, - {{"aes_cbc"}, {&EVP_aes_128_cbc}, 16}, - {{"aes_cbc"}, {&EVP_aes_192_cbc}, 24}, - {{"aes_cbc"}, {&EVP_aes_256_cbc}, 32}, - {{"aes_cbc128"}, {&EVP_aes_128_cbc}}, - {{"aes_cbc256"}, {&EVP_aes_256_cbc}}, - {{"aes_cfb8"}, {&EVP_aes_128_cfb8}}, - {{"aes_cfb128"}, {&EVP_aes_128_cfb128}}, - {{"aes_ecb"}, {&EVP_aes_128_ecb}, 16}, - {{"aes_ecb"}, {&EVP_aes_192_ecb}, 24}, - {{"aes_ecb"}, {&EVP_aes_256_ecb}, 32}, - {{NULL}} -}; - -static struct cipher_type_t* get_cipher_type(ERL_NIF_TERM type, size_t key_len); - - -/* -#define PRINTF_ERR0(FMT) enif_fprintf(stderr, FMT "\n") -#define PRINTF_ERR1(FMT, A1) enif_fprintf(stderr, FMT "\n", A1) -#define PRINTF_ERR2(FMT, A1, A2) enif_fprintf(stderr, FMT "\n", A1, A2) -*/ - -#define PRINTF_ERR0(FMT) -#define PRINTF_ERR1(FMT,A1) -#define PRINTF_ERR2(FMT,A1,A2) - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) -/* Define resource types for OpenSSL context structures. */ -static ErlNifResourceType* evp_md_ctx_rtype; -struct evp_md_ctx { - EVP_MD_CTX* ctx; -}; -static void evp_md_ctx_dtor(ErlNifEnv* env, struct evp_md_ctx *ctx) { - EVP_MD_CTX_free(ctx->ctx); -} -#endif - -#ifdef HAVE_EVP_AES_CTR -static ErlNifResourceType* evp_cipher_ctx_rtype; -struct evp_cipher_ctx { - EVP_CIPHER_CTX* ctx; -}; -static void evp_cipher_ctx_dtor(ErlNifEnv* env, struct evp_cipher_ctx* ctx) { - EVP_CIPHER_CTX_free(ctx->ctx); -} -#endif - -// Engine -#ifdef HAS_ENGINE_SUPPORT -static ErlNifResourceType* engine_ctx_rtype; -struct engine_ctx { - ENGINE *engine; - char *id; -}; -static void engine_ctx_dtor(ErlNifEnv* env, struct engine_ctx* ctx) { - PRINTF_ERR0("engine_ctx_dtor"); - if(ctx->id) { - PRINTF_ERR1(" non empty ctx->id=%s", ctx->id); - enif_free(ctx->id); - } else - PRINTF_ERR0(" empty ctx->id=NULL"); -} -#endif static int verify_lib_version(void) { @@ -1016,46 +149,6 @@ static int verify_lib_version(void) return 1; } -#ifdef FIPS_SUPPORT -/* In FIPS mode non-FIPS algorithms are disabled and return badarg. */ -#define CHECK_NO_FIPS_MODE() { if (FIPS_mode()) return atom_notsup; } -#else -#define CHECK_NO_FIPS_MODE() -#endif - -#ifdef HAVE_DYNAMIC_CRYPTO_LIB - -# if defined(DEBUG) -static char crypto_callback_name[] = "crypto_callback.debug"; -# elif defined(VALGRIND) -static char crypto_callback_name[] = "crypto_callback.valgrind"; -# else -static char crypto_callback_name[] = "crypto_callback"; -# endif - -static int change_basename(ErlNifBinary* bin, char* buf, int bufsz, const char* newfile) -{ - int i; - - for (i = bin->size; i > 0; i--) { - if (bin->data[i-1] == '/') - break; - } - if (i + strlen(newfile) >= bufsz) { - PRINTF_ERR0("CRYPTO: lib name too long"); - return 0; - } - memcpy(buf, bin->data, i); - strcpy(buf+i, newfile); - return 1; -} - -static void error_handler(void* null, const char* errstr) -{ - PRINTF_ERR1("CRYPTO LOADING ERROR: '%s'", errstr); -} -#endif /* HAVE_DYNAMIC_CRYPTO_LIB */ - static int initialize(ErlNifEnv* env, ERL_NIF_TERM load_info) { #ifdef OPENSSL_THREADS @@ -1069,59 +162,37 @@ static int initialize(ErlNifEnv* env, ERL_NIF_TERM load_info) int vernum; ErlNifBinary lib_bin; char lib_buf[1000]; +#ifdef HAVE_DYNAMIC_CRYPTO_LIB + void *handle; +#endif if (!verify_lib_version()) return __LINE__; /* load_info: {302, <<"/full/path/of/this/library">>,true|false} */ - if (!enif_get_tuple(env, load_info, &tpl_arity, &tpl_array) - || tpl_arity != 3 - || !enif_get_int(env, tpl_array[0], &vernum) - || vernum != 302 - || !enif_inspect_binary(env, tpl_array[1], &lib_bin)) { - - PRINTF_ERR1("CRYPTO: Invalid load_info '%T'", load_info); - return __LINE__; - } + if (!enif_get_tuple(env, load_info, &tpl_arity, &tpl_array)) + return __LINE__; + if (tpl_arity != 3) + return __LINE__; + if (!enif_get_int(env, tpl_array[0], &vernum)) + return __LINE__; + if (vernum != 302) + return __LINE__; + if (!enif_inspect_binary(env, tpl_array[1], &lib_bin)) + return __LINE__; - hmac_context_rtype = enif_open_resource_type(env, NULL, "hmac_context", - (ErlNifResourceDtor*) hmac_context_dtor, - ERL_NIF_RT_CREATE|ERL_NIF_RT_TAKEOVER, - NULL); - if (!hmac_context_rtype) { - PRINTF_ERR0("CRYPTO: Could not open resource type 'hmac_context'"); + if (!init_hmac_ctx(env)) { return __LINE__; } -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) - evp_md_ctx_rtype = enif_open_resource_type(env, NULL, "EVP_MD_CTX", - (ErlNifResourceDtor*) evp_md_ctx_dtor, - ERL_NIF_RT_CREATE|ERL_NIF_RT_TAKEOVER, - NULL); - if (!evp_md_ctx_rtype) { - PRINTF_ERR0("CRYPTO: Could not open resource type 'EVP_MD_CTX'"); + if (!init_hash_ctx(env)) { return __LINE__; } -#endif -#ifdef HAVE_EVP_AES_CTR - evp_cipher_ctx_rtype = enif_open_resource_type(env, NULL, "EVP_CIPHER_CTX", - (ErlNifResourceDtor*) evp_cipher_ctx_dtor, - ERL_NIF_RT_CREATE|ERL_NIF_RT_TAKEOVER, - NULL); - if (!evp_cipher_ctx_rtype) { - PRINTF_ERR0("CRYPTO: Could not open resource type 'EVP_CIPHER_CTX'"); + if (!init_cipher_ctx(env)) { return __LINE__; } -#endif -#ifdef HAS_ENGINE_SUPPORT - engine_ctx_rtype = enif_open_resource_type(env, NULL, "ENGINE_CTX", - (ErlNifResourceDtor*) engine_ctx_dtor, - ERL_NIF_RT_CREATE|ERL_NIF_RT_TAKEOVER, - NULL); - if (!engine_ctx_rtype) { - PRINTF_ERR0("CRYPTO: Could not open resource type 'ENGINE_CTX'"); + if (!init_engine_ctx(env)) { return __LINE__; } -#endif if (library_initialized) { /* Repeated loading of this library (module upgrade). @@ -1130,149 +201,18 @@ static int initialize(ErlNifEnv* env, ERL_NIF_TERM load_info) return 0; } - atom_true = enif_make_atom(env,"true"); - atom_false = enif_make_atom(env,"false"); - /* Enter FIPS mode */ - if (tpl_array[2] == atom_true) { -#ifdef FIPS_SUPPORT - if (!FIPS_mode_set(1)) { -#else - { -#endif - PRINTF_ERR0("CRYPTO: Could not setup FIPS mode"); - return 0; - } - } else if (tpl_array[2] != atom_false) { - PRINTF_ERR1("CRYPTO: Invalid load_info '%T'", load_info); - return 0; + if (!init_atoms(env, tpl_array[2], load_info)) { + return __LINE__; } - atom_sha = enif_make_atom(env,"sha"); - atom_error = enif_make_atom(env,"error"); - atom_rsa_pkcs1_padding = enif_make_atom(env,"rsa_pkcs1_padding"); - atom_rsa_pkcs1_oaep_padding = enif_make_atom(env,"rsa_pkcs1_oaep_padding"); - atom_rsa_no_padding = enif_make_atom(env,"rsa_no_padding"); - atom_signature_md = enif_make_atom(env,"signature_md"); - atom_undefined = enif_make_atom(env,"undefined"); - atom_ok = enif_make_atom(env,"ok"); - atom_not_prime = enif_make_atom(env,"not_prime"); - atom_not_strong_prime = enif_make_atom(env,"not_strong_prime"); - atom_unable_to_check_generator = enif_make_atom(env,"unable_to_check_generator"); - atom_not_suitable_generator = enif_make_atom(env,"not_suitable_generator"); - atom_check_failed = enif_make_atom(env,"check_failed"); - atom_unknown = enif_make_atom(env,"unknown"); - atom_none = enif_make_atom(env,"none"); - atom_notsup = enif_make_atom(env,"notsup"); - atom_digest = enif_make_atom(env,"digest"); - -#if defined(HAVE_EC) - atom_ec = enif_make_atom(env,"ec"); - atom_prime_field = enif_make_atom(env,"prime_field"); - atom_characteristic_two_field = enif_make_atom(env,"characteristic_two_field"); - atom_tpbasis = enif_make_atom(env,"tpbasis"); - atom_ppbasis = enif_make_atom(env,"ppbasis"); - atom_onbasis = enif_make_atom(env,"onbasis"); -#endif - - atom_aes_cfb8 = enif_make_atom(env, "aes_cfb8"); - atom_aes_cfb128 = enif_make_atom(env, "aes_cfb128"); -#ifdef HAVE_GCM - atom_aes_gcm = enif_make_atom(env, "aes_gcm"); -#endif -#ifdef HAVE_CCM - atom_aes_ccm = enif_make_atom(env, "aes_ccm"); -#endif -#ifdef HAVE_CHACHA20_POLY1305 - atom_chacha20_poly1305 = enif_make_atom(env,"chacha20_poly1305"); -#endif -#ifdef HAVE_ECB_IVEC_BUG - atom_aes_ecb = enif_make_atom(env, "aes_ecb"); - atom_des_ecb = enif_make_atom(env, "des_ecb"); - atom_blowfish_ecb = enif_make_atom(env, "blowfish_ecb"); -#endif - -#ifdef FIPS_SUPPORT - atom_enabled = enif_make_atom(env,"enabled"); - atom_not_enabled = enif_make_atom(env,"not_enabled"); -#else - atom_not_supported = enif_make_atom(env,"not_supported"); -#endif - atom_rsa = enif_make_atom(env,"rsa"); - atom_dss = enif_make_atom(env,"dss"); - atom_ecdsa = enif_make_atom(env,"ecdsa"); -#ifdef HAVE_ED_CURVE_DH - atom_x25519 = enif_make_atom(env,"x25519"); - atom_x448 = enif_make_atom(env,"x448"); -#endif - atom_eddsa = enif_make_atom(env,"eddsa"); -#ifdef HAVE_EDDSA - atom_ed25519 = enif_make_atom(env,"ed25519"); - atom_ed448 = enif_make_atom(env,"ed448"); -#endif - atom_rsa_mgf1_md = enif_make_atom(env,"rsa_mgf1_md"); - atom_rsa_oaep_label = enif_make_atom(env,"rsa_oaep_label"); - atom_rsa_oaep_md = enif_make_atom(env,"rsa_oaep_md"); - atom_rsa_pad = enif_make_atom(env,"rsa_pad"); /* backwards compatibility */ - atom_rsa_padding = enif_make_atom(env,"rsa_padding"); - atom_rsa_pkcs1_pss_padding = enif_make_atom(env,"rsa_pkcs1_pss_padding"); -#ifdef HAVE_RSA_SSLV23_PADDING - atom_rsa_sslv23_padding = enif_make_atom(env,"rsa_sslv23_padding"); -#endif - atom_rsa_x931_padding = enif_make_atom(env,"rsa_x931_padding"); - atom_rsa_pss_saltlen = enif_make_atom(env,"rsa_pss_saltlen"); - atom_sha224 = enif_make_atom(env,"sha224"); - atom_sha256 = enif_make_atom(env,"sha256"); - atom_sha384 = enif_make_atom(env,"sha384"); - atom_sha512 = enif_make_atom(env,"sha512"); - atom_sha3_224 = enif_make_atom(env,"sha3_224"); - atom_sha3_256 = enif_make_atom(env,"sha3_256"); - atom_sha3_384 = enif_make_atom(env,"sha3_384"); - atom_sha3_512 = enif_make_atom(env,"sha3_512"); - atom_md5 = enif_make_atom(env,"md5"); - atom_ripemd160 = enif_make_atom(env,"ripemd160"); - -#ifdef HAS_ENGINE_SUPPORT - atom_bad_engine_method = enif_make_atom(env,"bad_engine_method"); - atom_bad_engine_id = enif_make_atom(env,"bad_engine_id"); - atom_ctrl_cmd_failed = enif_make_atom(env,"ctrl_cmd_failed"); - atom_engine_init_failed = enif_make_atom(env,"engine_init_failed"); - atom_engine_method_not_supported = enif_make_atom(env,"engine_method_not_supported"); - atom_add_engine_failed = enif_make_atom(env,"add_engine_failed"); - atom_remove_engine_failed = enif_make_atom(env,"remove_engine_failed"); - - atom_engine_method_rsa = enif_make_atom(env,"engine_method_rsa"); - atom_engine_method_dsa = enif_make_atom(env,"engine_method_dsa"); - atom_engine_method_dh = enif_make_atom(env,"engine_method_dh"); - atom_engine_method_rand = enif_make_atom(env,"engine_method_rand"); - atom_engine_method_ecdh = enif_make_atom(env,"engine_method_ecdh"); - atom_engine_method_ecdsa = enif_make_atom(env,"engine_method_ecdsa"); - atom_engine_method_store = enif_make_atom(env,"engine_method_store"); - atom_engine_method_ciphers = enif_make_atom(env,"engine_method_ciphers"); - atom_engine_method_digests = enif_make_atom(env,"engine_method_digests"); - atom_engine_method_pkey_meths = enif_make_atom(env,"engine_method_pkey_meths"); - atom_engine_method_pkey_asn1_meths = enif_make_atom(env,"engine_method_pkey_asn1_meths"); - atom_engine_method_ec = enif_make_atom(env,"engine_method_ec"); - - atom_engine = enif_make_atom(env,"engine"); - atom_key_id = enif_make_atom(env,"key_id"); - atom_password = enif_make_atom(env,"password"); -#endif - - #ifdef HAVE_DYNAMIC_CRYPTO_LIB - { - void* handle; - if (!change_basename(&lib_bin, lib_buf, sizeof(lib_buf), crypto_callback_name)) { - return __LINE__; - } - if (!(handle = enif_dlopen(lib_buf, &error_handler, NULL))) { - return __LINE__; - } - if (!(funcp = (get_crypto_callbacks_t*) enif_dlsym(handle, "get_crypto_callbacks", - &error_handler, NULL))) { - return __LINE__; - } - } + if (!change_basename(&lib_bin, lib_buf, sizeof(lib_buf), crypto_callback_name)) + return __LINE__; + if ((handle = enif_dlopen(lib_buf, &error_handler, NULL)) == NULL) + return __LINE__; + if ((funcp = (get_crypto_callbacks_t*) enif_dlsym(handle, "get_crypto_callbacks", + &error_handler, NULL)) == NULL) + return __LINE__; #else /* !HAVE_DYNAMIC_CRYPTO_LIB */ funcp = &get_crypto_callbacks; #endif @@ -1292,7 +232,10 @@ static int initialize(ErlNifEnv* env, ERL_NIF_TERM load_info) return __LINE__; } - CRYPTO_set_mem_functions(ccb->crypto_alloc, ccb->crypto_realloc, ccb->crypto_free); +#ifdef HAS_CRYPTO_MEM_FUNCTIONS + if (!CRYPTO_set_mem_functions(ccb->crypto_alloc, ccb->crypto_realloc, ccb->crypto_free)) + return __LINE__; +#endif #ifdef OPENSSL_THREADS if (nlocks > 0) { @@ -1346,4812 +289,3 @@ static void unload(ErlNifEnv* env, void* priv_data) { --library_refc; } - -static int algo_hash_cnt, algo_hash_fips_cnt; -static ERL_NIF_TERM algo_hash[12]; /* increase when extending the list */ -static int algo_pubkey_cnt, algo_pubkey_fips_cnt; -static ERL_NIF_TERM algo_pubkey[12]; /* increase when extending the list */ -static int algo_cipher_cnt, algo_cipher_fips_cnt; -static ERL_NIF_TERM algo_cipher[25]; /* increase when extending the list */ -static int algo_mac_cnt, algo_mac_fips_cnt; -static ERL_NIF_TERM algo_mac[3]; /* increase when extending the list */ -static int algo_curve_cnt, algo_curve_fips_cnt; -static ERL_NIF_TERM algo_curve[89]; /* increase when extending the list */ -static int algo_rsa_opts_cnt, algo_rsa_opts_fips_cnt; -static ERL_NIF_TERM algo_rsa_opts[11]; /* increase when extending the list */ - -static void init_algorithms_types(ErlNifEnv* env) -{ - // Validated algorithms first - algo_hash_cnt = 0; - algo_hash[algo_hash_cnt++] = atom_sha; -#ifdef HAVE_SHA224 - algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha224"); -#endif -#ifdef HAVE_SHA256 - algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha256"); -#endif -#ifdef HAVE_SHA384 - algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha384"); -#endif -#ifdef HAVE_SHA512 - algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha512"); -#endif -#ifdef HAVE_SHA3_224 - algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha3_224"); -#endif -#ifdef HAVE_SHA3_256 - algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha3_256"); -#endif -#ifdef HAVE_SHA3_384 - algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha3_384"); -#endif -#ifdef HAVE_SHA3_512 - algo_hash[algo_hash_cnt++] = enif_make_atom(env, "sha3_512"); -#endif - // Non-validated algorithms follow - algo_hash_fips_cnt = algo_hash_cnt; - algo_hash[algo_hash_cnt++] = enif_make_atom(env, "md4"); - algo_hash[algo_hash_cnt++] = enif_make_atom(env, "md5"); - algo_hash[algo_hash_cnt++] = enif_make_atom(env, "ripemd160"); - - algo_pubkey_cnt = 0; - algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "rsa"); - algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "dss"); - algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "dh"); -#if defined(HAVE_EC) -#if !defined(OPENSSL_NO_EC2M) - algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "ec_gf2m"); -#endif - algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "ecdsa"); - algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "ecdh"); -#endif - // Non-validated algorithms follow - algo_pubkey_fips_cnt = algo_pubkey_cnt; - // Don't know if Edward curves are fips validated -#if defined(HAVE_EDDSA) - algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "eddsa"); -#endif - algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "srp"); - - // Validated algorithms first - algo_cipher_cnt = 0; -#ifndef OPENSSL_NO_DES - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "des3_cbc"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "des_ede3"); -#ifdef HAVE_DES_ede3_cfb_encrypt - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "des3_cbf"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "des3_cfb"); -#endif -#endif - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cbc"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cbc128"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cbc256"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_ctr"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_ecb"); -#if defined(HAVE_GCM) - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"aes_gcm"); -#endif -#if defined(HAVE_CCM) - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"aes_ccm"); -#endif - // Non-validated algorithms follow - algo_cipher_fips_cnt = algo_cipher_cnt; -#ifdef HAVE_AES_IGE - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"aes_ige256"); -#endif - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cfb8"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cfb128"); -#ifndef OPENSSL_NO_DES - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"des_cbc"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"des_cfb"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"des_ecb"); -#endif - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"blowfish_cbc"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"blowfish_cfb64"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"blowfish_ofb64"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"blowfish_ecb"); -#ifndef OPENSSL_NO_RC2 - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"rc2_cbc"); -#endif -#ifndef OPENSSL_NO_RC4 - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"rc4"); -#endif -#if defined(HAVE_CHACHA20_POLY1305) - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"chacha20_poly1305"); -#endif -#if defined(HAVE_CHACHA20) - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"chacha20"); -#endif - - // Validated algorithms first - algo_mac_cnt = 0; - algo_mac[algo_mac_cnt++] = enif_make_atom(env,"hmac"); -#ifdef HAVE_CMAC - algo_mac[algo_mac_cnt++] = enif_make_atom(env,"cmac"); -#endif -#ifdef HAVE_POLY1305 - algo_mac[algo_mac_cnt++] = enif_make_atom(env,"poly1305"); -#endif - // Non-validated algorithms follow - algo_mac_fips_cnt = algo_mac_cnt; - - // Validated algorithms first - algo_curve_cnt = 0; -#if defined(HAVE_EC) - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp160k1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp160r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp160r2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp192r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp192k1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp224k1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp224r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp256k1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp256r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp384r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp521r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime192v1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime192v2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime192v3"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime239v1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime239v2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime239v3"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"prime256v1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls7"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls9"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls12"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP160r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP160t1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP192r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP192t1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP224r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP224t1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP256r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP256t1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP320r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP320t1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP384r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP384t1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP512r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"brainpoolP512t1"); -#if !defined(OPENSSL_NO_EC2M) - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect163k1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect163r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect163r2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect193r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect193r2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect233k1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect233r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect239k1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect283k1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect283r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect409k1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect409r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect571k1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect571r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb163v1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb163v2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb163v3"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb176v1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb191v1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb191v2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb191v3"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb208w1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb239v1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb239v2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb239v3"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb272w1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb304w1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb359v1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2pnb368w1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"c2tnb431r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls3"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls5"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls10"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls11"); -#endif -#endif - // Non-validated algorithms follow - algo_curve_fips_cnt = algo_curve_cnt; -#if defined(HAVE_EC) - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp112r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp112r2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp128r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"secp128r2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls6"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls8"); -#if !defined(OPENSSL_NO_EC2M) - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect113r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect113r2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect131r1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"sect131r2"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls1"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"wtls4"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"ipsec3"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"ipsec4"); -#endif -#endif - //-- -#ifdef HAVE_EDDSA - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"ed25519"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"ed448"); -#endif -#ifdef HAVE_ED_CURVE_DH - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"x25519"); - algo_curve[algo_curve_cnt++] = enif_make_atom(env,"x448"); -#endif - - // Validated algorithms first - algo_rsa_opts_cnt = 0; -#ifdef HAS_EVP_PKEY_CTX -# ifdef HAVE_RSA_PKCS1_PSS_PADDING - algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_pkcs1_pss_padding"); - algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_pss_saltlen"); -# endif -# ifdef HAVE_RSA_MGF1_MD - algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_mgf1_md"); -# endif -# ifdef HAVE_RSA_OAEP_PADDING - algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_pkcs1_oaep_padding"); -# endif -# ifdef HAVE_RSA_OAEP_MD - algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_oaep_label"); - algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_oaep_md"); -# endif - algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"signature_md"); -#endif - algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_pkcs1_padding"); - algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_x931_padding"); -#ifdef HAVE_RSA_SSLV23_PADDING - algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_sslv23_padding"); -#endif - algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_no_padding"); - algo_rsa_opts_fips_cnt = algo_rsa_opts_cnt; - - - // Check that the max number of algos is updated - ASSERT(algo_hash_cnt <= sizeof(algo_hash)/sizeof(ERL_NIF_TERM)); - ASSERT(algo_pubkey_cnt <= sizeof(algo_pubkey)/sizeof(ERL_NIF_TERM)); - ASSERT(algo_cipher_cnt <= sizeof(algo_cipher)/sizeof(ERL_NIF_TERM)); - ASSERT(algo_mac_cnt <= sizeof(algo_mac)/sizeof(ERL_NIF_TERM)); - ASSERT(algo_curve_cnt <= sizeof(algo_curve)/sizeof(ERL_NIF_TERM)); - ASSERT(algo_rsa_opts_cnt <= sizeof(algo_rsa_opts)/sizeof(ERL_NIF_TERM)); -} - -static ERL_NIF_TERM algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ -#ifdef FIPS_SUPPORT - int fips_mode = FIPS_mode(); - int hash_cnt = fips_mode ? algo_hash_fips_cnt : algo_hash_cnt; - int pubkey_cnt = fips_mode ? algo_pubkey_fips_cnt : algo_pubkey_cnt; - int cipher_cnt = fips_mode ? algo_cipher_fips_cnt : algo_cipher_cnt; - int mac_cnt = fips_mode ? algo_mac_fips_cnt : algo_mac_cnt; - int curve_cnt = fips_mode ? algo_curve_fips_cnt : algo_curve_cnt; - int rsa_opts_cnt = fips_mode ? algo_rsa_opts_fips_cnt : algo_rsa_opts_cnt; -#else - int hash_cnt = algo_hash_cnt; - int pubkey_cnt = algo_pubkey_cnt; - int cipher_cnt = algo_cipher_cnt; - int mac_cnt = algo_mac_cnt; - int curve_cnt = algo_curve_cnt; - int rsa_opts_cnt = algo_rsa_opts_cnt; -#endif - return enif_make_tuple6(env, - enif_make_list_from_array(env, algo_hash, hash_cnt), - enif_make_list_from_array(env, algo_pubkey, pubkey_cnt), - enif_make_list_from_array(env, algo_cipher, cipher_cnt), - enif_make_list_from_array(env, algo_mac, mac_cnt), - enif_make_list_from_array(env, algo_curve, curve_cnt), - enif_make_list_from_array(env, algo_rsa_opts, rsa_opts_cnt) - ); -} - -static ERL_NIF_TERM info_lib(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - /* [{<<"OpenSSL">>,9470143,<<"OpenSSL 0.9.8k 25 Mar 2009">>}] */ - - static const char libname[] = "OpenSSL"; - unsigned name_sz = strlen(libname); - const char* ver = SSLeay_version(SSLEAY_VERSION); - unsigned ver_sz = strlen(ver); - ERL_NIF_TERM name_term, ver_term; - int ver_num = OPENSSL_VERSION_NUMBER; - /* R16: - * Ignore library version number from SSLeay() and instead show header - * version. Otherwise user might try to call a function that is implemented - * by a newer library but not supported by the headers used at compile time. - * Example: DES_ede3_cfb_encrypt in 0.9.7i but not in 0.9.7d. - * - * Version string is still from library though. - */ - - memcpy(enif_make_new_binary(env, name_sz, &name_term), libname, name_sz); - memcpy(enif_make_new_binary(env, ver_sz, &ver_term), ver, ver_sz); - - return enif_make_list1(env, enif_make_tuple3(env, name_term, - enif_make_int(env, ver_num), - ver_term)); -} - -static ERL_NIF_TERM info_fips(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ -#ifdef FIPS_SUPPORT - return FIPS_mode() ? atom_enabled : atom_not_enabled; -#else - return atom_not_supported; -#endif -} - -static ERL_NIF_TERM enable_fips_mode(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Boolean) */ - if (argv[0] == atom_true) { -#ifdef FIPS_SUPPORT - if (FIPS_mode_set(1)) { - return atom_true; - } -#endif - PRINTF_ERR0("CRYPTO: Could not setup FIPS mode"); - return atom_false; - } else if (argv[0] == atom_false) { -#ifdef FIPS_SUPPORT - if (!FIPS_mode_set(0)) { - return atom_false; - } -#endif - return atom_true; - } else { - return enif_make_badarg(env); - } -} - - -#if defined(HAVE_EC) -static ERL_NIF_TERM make_badarg_maybe(ErlNifEnv* env) -{ - ERL_NIF_TERM reason; - if (enif_has_pending_exception(env, &reason)) - return reason; /* dummy return value ignored */ - else - return enif_make_badarg(env); -} -#endif - -static ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type, Data) */ - struct digest_type_t *digp = NULL; - const EVP_MD *md; - ErlNifBinary data; - ERL_NIF_TERM ret; - unsigned ret_size; - - digp = get_digest_type(argv[0]); - if (!digp || - !enif_inspect_iolist_as_binary(env, argv[1], &data)) { - return enif_make_badarg(env); - } - md = digp->md.p; - if (!md) { - return atom_notsup; - } - - ret_size = (unsigned)EVP_MD_size(md); - ASSERT(0 < ret_size && ret_size <= EVP_MAX_MD_SIZE); - if (!EVP_Digest(data.data, data.size, - enif_make_new_binary(env, ret_size, &ret), &ret_size, - md, NULL)) { - return atom_notsup; - } - ASSERT(ret_size == (unsigned)EVP_MD_size(md)); - - CONSUME_REDS(env, data); - return ret; -} - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) - -static ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type) */ - struct digest_type_t *digp = NULL; - struct evp_md_ctx *ctx; - ERL_NIF_TERM ret; - - digp = get_digest_type(argv[0]); - if (!digp) { - return enif_make_badarg(env); - } - if (!digp->md.p) { - return atom_notsup; - } - - ctx = enif_alloc_resource(evp_md_ctx_rtype, sizeof(struct evp_md_ctx)); - ctx->ctx = EVP_MD_CTX_new(); - if (!EVP_DigestInit(ctx->ctx, digp->md.p)) { - enif_release_resource(ctx); - return atom_notsup; - } - ret = enif_make_resource(env, ctx); - enif_release_resource(ctx); - return ret; -} -static ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Context, Data) */ - struct evp_md_ctx *ctx, *new_ctx; - ErlNifBinary data; - ERL_NIF_TERM ret; - - if (!enif_get_resource(env, argv[0], evp_md_ctx_rtype, (void**)&ctx) || - !enif_inspect_iolist_as_binary(env, argv[1], &data)) { - return enif_make_badarg(env); - } - - new_ctx = enif_alloc_resource(evp_md_ctx_rtype, sizeof(struct evp_md_ctx)); - new_ctx->ctx = EVP_MD_CTX_new(); - if (!EVP_MD_CTX_copy(new_ctx->ctx, ctx->ctx) || - !EVP_DigestUpdate(new_ctx->ctx, data.data, data.size)) { - enif_release_resource(new_ctx); - return atom_notsup; - } - - ret = enif_make_resource(env, new_ctx); - enif_release_resource(new_ctx); - CONSUME_REDS(env, data); - return ret; -} -static ERL_NIF_TERM hash_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Context) */ - struct evp_md_ctx *ctx; - EVP_MD_CTX *new_ctx; - ERL_NIF_TERM ret; - unsigned ret_size; - - if (!enif_get_resource(env, argv[0], evp_md_ctx_rtype, (void**)&ctx)) { - return enif_make_badarg(env); - } - - ret_size = (unsigned)EVP_MD_CTX_size(ctx->ctx); - ASSERT(0 < ret_size && ret_size <= EVP_MAX_MD_SIZE); - - new_ctx = EVP_MD_CTX_new(); - if (!EVP_MD_CTX_copy(new_ctx, ctx->ctx) || - !EVP_DigestFinal(new_ctx, - enif_make_new_binary(env, ret_size, &ret), - &ret_size)) { - EVP_MD_CTX_free(new_ctx); - return atom_notsup; - } - EVP_MD_CTX_free(new_ctx); - ASSERT(ret_size == (unsigned)EVP_MD_CTX_size(ctx->ctx)); - - return ret; -} - -#else /* if OPENSSL_VERSION_NUMBER < 1.0 */ - -static ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type) */ - typedef int (*init_fun)(unsigned char*); - struct digest_type_t *digp = NULL; - ERL_NIF_TERM ctx; - size_t ctx_size = 0; - init_fun ctx_init = 0; - - digp = get_digest_type(argv[0]); - if (!digp) { - return enif_make_badarg(env); - } - if (!digp->md.p) { - return atom_notsup; - } - - switch (EVP_MD_type(digp->md.p)) - { - case NID_md4: - ctx_size = MD4_CTX_LEN; - ctx_init = (init_fun)(&MD4_Init); - break; - case NID_md5: - ctx_size = MD5_CTX_LEN; - ctx_init = (init_fun)(&MD5_Init); - break; - case NID_ripemd160: - ctx_size = RIPEMD160_CTX_LEN; - ctx_init = (init_fun)(&RIPEMD160_Init); - break; - case NID_sha1: - ctx_size = sizeof(SHA_CTX); - ctx_init = (init_fun)(&SHA1_Init); - break; -#ifdef HAVE_SHA224 - case NID_sha224: - ctx_size = sizeof(SHA256_CTX); - ctx_init = (init_fun)(&SHA224_Init); - break; -#endif -#ifdef HAVE_SHA256 - case NID_sha256: - ctx_size = sizeof(SHA256_CTX); - ctx_init = (init_fun)(&SHA256_Init); - break; -#endif -#ifdef HAVE_SHA384 - case NID_sha384: - ctx_size = sizeof(SHA512_CTX); - ctx_init = (init_fun)(&SHA384_Init); - break; -#endif -#ifdef HAVE_SHA512 - case NID_sha512: - ctx_size = sizeof(SHA512_CTX); - ctx_init = (init_fun)(&SHA512_Init); - break; -#endif - default: - return atom_notsup; - } - ASSERT(ctx_size); - ASSERT(ctx_init); - - ctx_init(enif_make_new_binary(env, ctx_size, &ctx)); - return enif_make_tuple2(env, argv[0], ctx); -} -static ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* ({Type, Context}, Data) */ - typedef int (*update_fun)(unsigned char*, const unsigned char*, size_t); - ERL_NIF_TERM new_ctx; - ErlNifBinary ctx, data; - const ERL_NIF_TERM *tuple; - int arity; - struct digest_type_t *digp = NULL; - unsigned char *ctx_buff; - size_t ctx_size = 0; - update_fun ctx_update = 0; - - if (!enif_get_tuple(env, argv[0], &arity, &tuple) || - arity != 2 || - !(digp = get_digest_type(tuple[0])) || - !enif_inspect_binary(env, tuple[1], &ctx) || - !enif_inspect_iolist_as_binary(env, argv[1], &data)) { - return enif_make_badarg(env); - } - if (!digp->md.p) { - return atom_notsup; - } - - switch (EVP_MD_type(digp->md.p)) - { - case NID_md4: - ctx_size = MD4_CTX_LEN; - ctx_update = (update_fun)(&MD4_Update); - break; - case NID_md5: - ctx_size = MD5_CTX_LEN; - ctx_update = (update_fun)(&MD5_Update); - break; - case NID_ripemd160: - ctx_size = RIPEMD160_CTX_LEN; - ctx_update = (update_fun)(&RIPEMD160_Update); - break; - case NID_sha1: - ctx_size = sizeof(SHA_CTX); - ctx_update = (update_fun)(&SHA1_Update); - break; -#ifdef HAVE_SHA224 - case NID_sha224: - ctx_size = sizeof(SHA256_CTX); - ctx_update = (update_fun)(&SHA224_Update); - break; -#endif -#ifdef HAVE_SHA256 - case NID_sha256: - ctx_size = sizeof(SHA256_CTX); - ctx_update = (update_fun)(&SHA256_Update); - break; -#endif -#ifdef HAVE_SHA384 - case NID_sha384: - ctx_size = sizeof(SHA512_CTX); - ctx_update = (update_fun)(&SHA384_Update); - break; -#endif -#ifdef HAVE_SHA512 - case NID_sha512: - ctx_size = sizeof(SHA512_CTX); - ctx_update = (update_fun)(&SHA512_Update); - break; -#endif - default: - return atom_notsup; - } - ASSERT(ctx_size); - ASSERT(ctx_update); - - if (ctx.size != ctx_size) { - return enif_make_badarg(env); - } - - ctx_buff = enif_make_new_binary(env, ctx_size, &new_ctx); - memcpy(ctx_buff, ctx.data, ctx_size); - ctx_update(ctx_buff, data.data, data.size); - - CONSUME_REDS(env, data); - return enif_make_tuple2(env, tuple[0], new_ctx); -} -static ERL_NIF_TERM hash_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* ({Type, Context}) */ - typedef int (*final_fun)(unsigned char*, void*); - ERL_NIF_TERM ret; - ErlNifBinary ctx; - const ERL_NIF_TERM *tuple; - int arity; - struct digest_type_t *digp = NULL; - const EVP_MD *md; - void *new_ctx; - size_t ctx_size = 0; - final_fun ctx_final = 0; - - if (!enif_get_tuple(env, argv[0], &arity, &tuple) || - arity != 2 || - !(digp = get_digest_type(tuple[0])) || - !enif_inspect_binary(env, tuple[1], &ctx)) { - return enif_make_badarg(env); - } - md = digp->md.p; - if (!md) { - return atom_notsup; - } - - - switch (EVP_MD_type(md)) - { - case NID_md4: - ctx_size = MD4_CTX_LEN; - ctx_final = (final_fun)(&MD4_Final); - break; - case NID_md5: - ctx_size = MD5_CTX_LEN; - ctx_final = (final_fun)(&MD5_Final); - break; - case NID_ripemd160: - ctx_size = RIPEMD160_CTX_LEN; - ctx_final = (final_fun)(&RIPEMD160_Final); - break; - case NID_sha1: - ctx_size = sizeof(SHA_CTX); - ctx_final = (final_fun)(&SHA1_Final); - break; -#ifdef HAVE_SHA224 - case NID_sha224: - ctx_size = sizeof(SHA256_CTX); - ctx_final = (final_fun)(&SHA224_Final); - break; -#endif -#ifdef HAVE_SHA256 - case NID_sha256: - ctx_size = sizeof(SHA256_CTX); - ctx_final = (final_fun)(&SHA256_Final); - break; -#endif -#ifdef HAVE_SHA384 - case NID_sha384: - ctx_size = sizeof(SHA512_CTX); - ctx_final = (final_fun)(&SHA384_Final); - break; -#endif -#ifdef HAVE_SHA512 - case NID_sha512: - ctx_size = sizeof(SHA512_CTX); - ctx_final = (final_fun)(&SHA512_Final); - break; -#endif - default: - return atom_notsup; - } - ASSERT(ctx_size); - ASSERT(ctx_final); - - if (ctx.size != ctx_size) { - return enif_make_badarg(env); - } - - new_ctx = enif_alloc(ctx_size); - memcpy(new_ctx, ctx.data, ctx_size); - ctx_final(enif_make_new_binary(env, (size_t)EVP_MD_size(md), &ret), - new_ctx); - enif_free(new_ctx); - - return ret; -} -#endif /* OPENSSL_VERSION_NUMBER < 1.0 */ - - -static ERL_NIF_TERM hmac_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type, Key, Data) or (Type, Key, Data, MacSize) */ - struct digest_type_t *digp = NULL; - ErlNifBinary key, data; - unsigned char buff[EVP_MAX_MD_SIZE]; - unsigned size = 0, req_size = 0; - ERL_NIF_TERM ret; - - digp = get_digest_type(argv[0]); - if (!digp || - !enif_inspect_iolist_as_binary(env, argv[1], &key) || - !enif_inspect_iolist_as_binary(env, argv[2], &data) || - (argc == 4 && !enif_get_uint(env, argv[3], &req_size))) { - return enif_make_badarg(env); - } - - if (!digp->md.p || - !HMAC(digp->md.p, - key.data, key.size, - data.data, data.size, - buff, &size)) { - return atom_notsup; - } - ASSERT(0 < size && size <= EVP_MAX_MD_SIZE); - CONSUME_REDS(env, data); - - if (argc == 4) { - if (req_size <= size) { - size = req_size; - } - else { - return enif_make_badarg(env); - } - } - memcpy(enif_make_new_binary(env, size, &ret), buff, size); - return ret; -} - -static void hmac_context_dtor(ErlNifEnv* env, struct hmac_context *obj) -{ - if (obj->alive) { - HMAC_CTX_free(obj->ctx); - obj->alive = 0; - } - enif_mutex_destroy(obj->mtx); -} - -static ERL_NIF_TERM hmac_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type, Key) */ - struct digest_type_t *digp = NULL; - ErlNifBinary key; - ERL_NIF_TERM ret; - struct hmac_context *obj; - - digp = get_digest_type(argv[0]); - if (!digp || - !enif_inspect_iolist_as_binary(env, argv[1], &key)) { - return enif_make_badarg(env); - } - if (!digp->md.p) { - return atom_notsup; - } - - obj = enif_alloc_resource(hmac_context_rtype, sizeof(struct hmac_context)); - obj->mtx = enif_mutex_create("crypto.hmac"); - obj->alive = 1; - obj->ctx = HMAC_CTX_new(); -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) - // Check the return value of HMAC_Init: it may fail in FIPS mode - // for disabled algorithms - if (!HMAC_Init_ex(obj->ctx, key.data, key.size, digp->md.p, NULL)) { - enif_release_resource(obj); - return atom_notsup; - } -#else - HMAC_Init_ex(obj->ctx, key.data, key.size, digp->md.p, NULL); -#endif - - ret = enif_make_resource(env, obj); - enif_release_resource(obj); - return ret; -} - -static ERL_NIF_TERM hmac_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Context, Data) */ - ErlNifBinary data; - struct hmac_context* obj; - - if (!enif_get_resource(env, argv[0], hmac_context_rtype, (void**)&obj) - || !enif_inspect_iolist_as_binary(env, argv[1], &data)) { - return enif_make_badarg(env); - } - enif_mutex_lock(obj->mtx); - if (!obj->alive) { - enif_mutex_unlock(obj->mtx); - return enif_make_badarg(env); - } - HMAC_Update(obj->ctx, data.data, data.size); - enif_mutex_unlock(obj->mtx); - - CONSUME_REDS(env,data); - return argv[0]; -} - -static ERL_NIF_TERM hmac_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Context) or (Context, HashLen) */ - ERL_NIF_TERM ret; - struct hmac_context* obj; - unsigned char mac_buf[EVP_MAX_MD_SIZE]; - unsigned char * mac_bin; - unsigned int req_len = 0; - unsigned int mac_len; - - if (!enif_get_resource(env,argv[0],hmac_context_rtype, (void**)&obj) - || (argc == 2 && !enif_get_uint(env, argv[1], &req_len))) { - return enif_make_badarg(env); - } - - enif_mutex_lock(obj->mtx); - if (!obj->alive) { - enif_mutex_unlock(obj->mtx); - return enif_make_badarg(env); - } - - HMAC_Final(obj->ctx, mac_buf, &mac_len); - HMAC_CTX_free(obj->ctx); - obj->alive = 0; - enif_mutex_unlock(obj->mtx); - - if (argc == 2 && req_len < mac_len) { - /* Only truncate to req_len bytes if asked. */ - mac_len = req_len; - } - mac_bin = enif_make_new_binary(env, mac_len, &ret); - memcpy(mac_bin, mac_buf, mac_len); - - return ret; -} - -static ERL_NIF_TERM cmac_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type, Key, Data) */ -#if defined(HAVE_CMAC) - struct cipher_type_t *cipherp = NULL; - const EVP_CIPHER *cipher; - CMAC_CTX *ctx; - ErlNifBinary key; - ErlNifBinary data; - ERL_NIF_TERM ret; - size_t ret_size; - - if (!enif_inspect_iolist_as_binary(env, argv[1], &key) - || !(cipherp = get_cipher_type(argv[0], key.size)) - || !enif_inspect_iolist_as_binary(env, argv[2], &data)) { - return enif_make_badarg(env); - } - cipher = cipherp->cipher.p; - if (!cipher) { - return enif_raise_exception(env, atom_notsup); - } - - ctx = CMAC_CTX_new(); - if (!CMAC_Init(ctx, key.data, key.size, cipher, NULL)) { - CMAC_CTX_free(ctx); - return atom_notsup; - } - - if (!CMAC_Update(ctx, data.data, data.size) || - !CMAC_Final(ctx, - enif_make_new_binary(env, EVP_CIPHER_block_size(cipher), &ret), - &ret_size)) { - CMAC_CTX_free(ctx); - return atom_notsup; - } - ASSERT(ret_size == (unsigned)EVP_CIPHER_block_size(cipher)); - - CMAC_CTX_free(ctx); - CONSUME_REDS(env, data); - return ret; -#else - /* The CMAC functionality was introduced in OpenSSL 1.0.1 - * Although OTP requires at least version 0.9.8, the versions 0.9.8 and 1.0.0 are - * no longer maintained. */ - return atom_notsup; -#endif -} - -/* For OpenSSL >= 1.1.1 the hmac_nif and cmac_nif could be integrated into poly1305 (with 'type' as parameter) */ -static ERL_NIF_TERM poly1305_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, Text) */ -#ifdef HAVE_POLY1305 - ErlNifBinary key_bin, text, ret_bin; - ERL_NIF_TERM ret = atom_error; - EVP_PKEY *key = NULL; - EVP_MD_CTX *mctx = NULL; - EVP_PKEY_CTX *pctx = NULL; - const EVP_MD *md = NULL; - size_t size; - int type; - - type = EVP_PKEY_POLY1305; - - if (!enif_inspect_binary(env, argv[0], &key_bin) || - !(key_bin.size == 32) ) { - return enif_make_badarg(env); - } - - if (!enif_inspect_binary(env, argv[1], &text) ) { - return enif_make_badarg(env); - } - - key = EVP_PKEY_new_raw_private_key(type, /*engine*/ NULL, key_bin.data, key_bin.size); - - if (!key || - !(mctx = EVP_MD_CTX_new()) || - !EVP_DigestSignInit(mctx, &pctx, md, /*engine*/ NULL, key) || - !EVP_DigestSignUpdate(mctx, text.data, text.size)) { - goto err; - } - - if (!EVP_DigestSignFinal(mctx, NULL, &size) || - !enif_alloc_binary(size, &ret_bin) || - !EVP_DigestSignFinal(mctx, ret_bin.data, &size)) { - goto err; - } - - if ((size != ret_bin.size) && - !enif_realloc_binary(&ret_bin, size)) { - goto err; - } - - ret = enif_make_binary(env, &ret_bin); - - err: - EVP_MD_CTX_free(mctx); - EVP_PKEY_free(key); - return ret; - -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type, Key, Ivec, Text, IsEncrypt) or (Type, Key, Text, IsEncrypt) */ - struct cipher_type_t *cipherp = NULL; - const EVP_CIPHER *cipher; - ErlNifBinary key, ivec, text; - EVP_CIPHER_CTX* ctx; - ERL_NIF_TERM ret; - unsigned char *out; - int ivec_size, out_size = 0; - - if (!enif_inspect_iolist_as_binary(env, argv[1], &key) - || !(cipherp = get_cipher_type(argv[0], key.size)) - || !enif_inspect_iolist_as_binary(env, argv[argc - 2], &text)) { - return enif_make_badarg(env); - } - cipher = cipherp->cipher.p; - if (!cipher) { - return enif_raise_exception(env, atom_notsup); - } - - if (argv[0] == atom_aes_cfb8) { - CHECK_NO_FIPS_MODE(); - if ((key.size == 24 || key.size == 32)) { - /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? - * Fall back on low level API - */ - return aes_cfb_8_crypt(env, argc-1, argv+1); - } - } - else if (argv[0] == atom_aes_cfb128) { - CHECK_NO_FIPS_MODE(); - if ((key.size == 24 || key.size == 32)) { - /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? - * Fall back on low level API - */ - return aes_cfb_128_crypt_nif(env, argc-1, argv+1); - } - } - ivec_size = EVP_CIPHER_iv_length(cipher); - -#ifdef HAVE_ECB_IVEC_BUG - if (argv[0] == atom_aes_ecb || argv[0] == atom_blowfish_ecb || - argv[0] == atom_des_ecb) - ivec_size = 0; /* 0.9.8l returns faulty ivec_size */ -#endif - - if (text.size % EVP_CIPHER_block_size(cipher) != 0 || - (ivec_size == 0 ? argc != 4 - : (argc != 5 || - !enif_inspect_iolist_as_binary(env, argv[2], &ivec) || - ivec.size != ivec_size))) { - return enif_make_badarg(env); - } - - out = enif_make_new_binary(env, text.size, &ret); - - ctx = EVP_CIPHER_CTX_new(); - if (!EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, - (argv[argc - 1] == atom_true)) || - !EVP_CIPHER_CTX_set_key_length(ctx, key.size) || - !(EVP_CIPHER_type(cipher) != NID_rc2_cbc || - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_SET_RC2_KEY_BITS, key.size * 8, NULL)) || - !EVP_CipherInit_ex(ctx, NULL, NULL, - key.data, ivec_size ? ivec.data : NULL, -1) || - !EVP_CIPHER_CTX_set_padding(ctx, 0)) { - - EVP_CIPHER_CTX_free(ctx); - return enif_raise_exception(env, atom_notsup); - } - - if (text.size > 0 && /* OpenSSL 0.9.8h asserts text.size > 0 */ - (!EVP_CipherUpdate(ctx, out, &out_size, text.data, text.size) - || (ASSERT(out_size == text.size), 0) - || !EVP_CipherFinal_ex(ctx, out + out_size, &out_size))) { - - EVP_CIPHER_CTX_free(ctx); - return enif_raise_exception(env, atom_notsup); - } - ASSERT(out_size == 0); - EVP_CIPHER_CTX_free(ctx); - CONSUME_REDS(env, text); - - return ret; -} - -static ERL_NIF_TERM aes_cfb_8_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IVec, Data, IsEncrypt) */ - ErlNifBinary key, ivec, text; - AES_KEY aes_key; - unsigned char ivec_clone[16]; /* writable copy */ - int new_ivlen = 0; - ERL_NIF_TERM ret; - - CHECK_NO_FIPS_MODE(); - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key) - || !(key.size == 16 || key.size == 24 || key.size == 32) - || !enif_inspect_binary(env, argv[1], &ivec) || ivec.size != 16 - || !enif_inspect_iolist_as_binary(env, argv[2], &text)) { - return enif_make_badarg(env); - } - - memcpy(ivec_clone, ivec.data, 16); - AES_set_encrypt_key(key.data, key.size * 8, &aes_key); - AES_cfb8_encrypt((unsigned char *) text.data, - enif_make_new_binary(env, text.size, &ret), - text.size, &aes_key, ivec_clone, &new_ivlen, - (argv[3] == atom_true)); - CONSUME_REDS(env,text); - return ret; -} - -static ERL_NIF_TERM aes_cfb_128_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IVec, Data, IsEncrypt) */ - ErlNifBinary key, ivec, text; - AES_KEY aes_key; - unsigned char ivec_clone[16]; /* writable copy */ - int new_ivlen = 0; - ERL_NIF_TERM ret; - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key) - || !(key.size == 16 || key.size == 24 || key.size == 32) - || !enif_inspect_binary(env, argv[1], &ivec) || ivec.size != 16 - || !enif_inspect_iolist_as_binary(env, argv[2], &text)) { - return enif_make_badarg(env); - } - - memcpy(ivec_clone, ivec.data, 16); - AES_set_encrypt_key(key.data, key.size * 8, &aes_key); - AES_cfb128_encrypt((unsigned char *) text.data, - enif_make_new_binary(env, text.size, &ret), - text.size, &aes_key, ivec_clone, &new_ivlen, - (argv[3] == atom_true)); - CONSUME_REDS(env,text); - return ret; -} - -static ERL_NIF_TERM aes_ige_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IVec, Data, IsEncrypt) */ -#ifdef HAVE_AES_IGE - ErlNifBinary key_bin, ivec_bin, data_bin; - AES_KEY aes_key; - unsigned char ivec[32]; - int i; - unsigned char* ret_ptr; - ERL_NIF_TERM ret; - - CHECK_NO_FIPS_MODE(); - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key_bin) - || (key_bin.size != 16 && key_bin.size != 32) - || !enif_inspect_binary(env, argv[1], &ivec_bin) - || ivec_bin.size != 32 - || !enif_inspect_iolist_as_binary(env, argv[2], &data_bin) - || data_bin.size % 16 != 0) { - - return enif_make_badarg(env); - } - - if (argv[3] == atom_true) { - i = AES_ENCRYPT; - AES_set_encrypt_key(key_bin.data, key_bin.size*8, &aes_key); - } - else { - i = AES_DECRYPT; - AES_set_decrypt_key(key_bin.data, key_bin.size*8, &aes_key); - } - - ret_ptr = enif_make_new_binary(env, data_bin.size, &ret); - memcpy(ivec, ivec_bin.data, 32); /* writable copy */ - AES_ige_encrypt(data_bin.data, ret_ptr, data_bin.size, &aes_key, ivec, i); - CONSUME_REDS(env,data_bin); - return ret; -#else - return atom_notsup; -#endif -} - - -/* Initializes state for ctr streaming (de)encryption -*/ -#ifdef HAVE_EVP_AES_CTR -static ERL_NIF_TERM aes_ctr_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IVec) */ - ErlNifBinary key_bin, ivec_bin; - struct evp_cipher_ctx *ctx; - const EVP_CIPHER *cipher; - ERL_NIF_TERM ret; - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key_bin) - || !enif_inspect_binary(env, argv[1], &ivec_bin) - || ivec_bin.size != 16) { - return enif_make_badarg(env); - } - - switch (key_bin.size) - { - case 16: cipher = EVP_aes_128_ctr(); break; - case 24: cipher = EVP_aes_192_ctr(); break; - case 32: cipher = EVP_aes_256_ctr(); break; - default: return enif_make_badarg(env); - } - - ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx)); - ctx->ctx = EVP_CIPHER_CTX_new(); - EVP_CipherInit_ex(ctx->ctx, cipher, NULL, - key_bin.data, ivec_bin.data, 1); - EVP_CIPHER_CTX_set_padding(ctx->ctx, 0); - ret = enif_make_resource(env, ctx); - enif_release_resource(ctx); - return ret; -} -static ERL_NIF_TERM aes_ctr_stream_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Context, Data) */ - struct evp_cipher_ctx *ctx, *new_ctx; - ErlNifBinary data_bin; - ERL_NIF_TERM ret, cipher_term; - unsigned char *out; - int outl = 0; - - if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx) - || !enif_inspect_iolist_as_binary(env, argv[1], &data_bin)) { - return enif_make_badarg(env); - } - new_ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx)); - new_ctx->ctx = EVP_CIPHER_CTX_new(); - EVP_CIPHER_CTX_copy(new_ctx->ctx, ctx->ctx); - out = enif_make_new_binary(env, data_bin.size, &cipher_term); - EVP_CipherUpdate(new_ctx->ctx, out, &outl, data_bin.data, data_bin.size); - ASSERT(outl == data_bin.size); - - ret = enif_make_tuple2(env, enif_make_resource(env, new_ctx), cipher_term); - enif_release_resource(new_ctx); - CONSUME_REDS(env,data_bin); - return ret; -} - -#else /* if not HAVE_EVP_AES_CTR */ - -static ERL_NIF_TERM aes_ctr_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IVec) */ - ErlNifBinary key_bin, ivec_bin; - ERL_NIF_TERM ecount_bin; - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key_bin) - || !enif_inspect_binary(env, argv[1], &ivec_bin) - || !(key_bin.size == 16 || key_bin.size == 24 || key_bin.size ==32) - || ivec_bin.size != 16) { - return enif_make_badarg(env); - } - - memset(enif_make_new_binary(env, AES_BLOCK_SIZE, &ecount_bin), - 0, AES_BLOCK_SIZE); - return enif_make_tuple4(env, argv[0], argv[1], ecount_bin, enif_make_int(env, 0)); -} - -static ERL_NIF_TERM aes_ctr_stream_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* ({Key, IVec, ECount, Num}, Data) */ - ErlNifBinary key_bin, ivec_bin, text_bin, ecount_bin; - AES_KEY aes_key; - unsigned int num; - ERL_NIF_TERM ret, num2_term, cipher_term, ivec2_term, ecount2_term, new_state_term; - int state_arity; - const ERL_NIF_TERM *state_term; - unsigned char * ivec2_buf; - unsigned char * ecount2_buf; - - if (!enif_get_tuple(env, argv[0], &state_arity, &state_term) - || state_arity != 4 - || !enif_inspect_iolist_as_binary(env, state_term[0], &key_bin) - || AES_set_encrypt_key(key_bin.data, key_bin.size*8, &aes_key) != 0 - || !enif_inspect_binary(env, state_term[1], &ivec_bin) || ivec_bin.size != 16 - || !enif_inspect_binary(env, state_term[2], &ecount_bin) || ecount_bin.size != AES_BLOCK_SIZE - || !enif_get_uint(env, state_term[3], &num) - || !enif_inspect_iolist_as_binary(env, argv[1], &text_bin)) { - return enif_make_badarg(env); - } - - ivec2_buf = enif_make_new_binary(env, ivec_bin.size, &ivec2_term); - ecount2_buf = enif_make_new_binary(env, ecount_bin.size, &ecount2_term); - - memcpy(ivec2_buf, ivec_bin.data, 16); - memcpy(ecount2_buf, ecount_bin.data, ecount_bin.size); - - AES_ctr128_encrypt((unsigned char *) text_bin.data, - enif_make_new_binary(env, text_bin.size, &cipher_term), - text_bin.size, &aes_key, ivec2_buf, ecount2_buf, &num); - - num2_term = enif_make_uint(env, num); - new_state_term = enif_make_tuple4(env, state_term[0], ivec2_term, ecount2_term, num2_term); - ret = enif_make_tuple2(env, new_state_term, cipher_term); - CONSUME_REDS(env,text_bin); - return ret; -} -#endif /* !HAVE_EVP_AES_CTR */ - -static ERL_NIF_TERM aead_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type,Key,Iv,AAD,In) */ -#if defined(HAVE_AEAD) - EVP_CIPHER_CTX *ctx; - const EVP_CIPHER *cipher = NULL; - ErlNifBinary key, iv, aad, in; - unsigned int tag_len; - unsigned char *outp, *tagp; - ERL_NIF_TERM type, out, out_tag; - int len, ctx_ctrl_set_ivlen, ctx_ctrl_get_tag; - - type = argv[0]; - - if (!enif_is_atom(env, type) - || !enif_inspect_iolist_as_binary(env, argv[1], &key) - || !enif_inspect_binary(env, argv[2], &iv) - || !enif_inspect_iolist_as_binary(env, argv[3], &aad) - || !enif_inspect_iolist_as_binary(env, argv[4], &in) - || !enif_get_uint(env, argv[5], &tag_len)) { - return enif_make_badarg(env); - } - - /* Use cipher_type some day. Must check block_encrypt|decrypt first */ -#if defined(HAVE_GCM) - if (type == atom_aes_gcm) { - if ((iv.size > 0) - && (1 <= tag_len && tag_len <= 16)) { - ctx_ctrl_set_ivlen = EVP_CTRL_GCM_SET_IVLEN; - ctx_ctrl_get_tag = EVP_CTRL_GCM_GET_TAG; - if (key.size == 16) cipher = EVP_aes_128_gcm(); - else if (key.size == 24) cipher = EVP_aes_192_gcm(); - else if (key.size == 32) cipher = EVP_aes_256_gcm(); - else enif_make_badarg(env); - } else - enif_make_badarg(env); - } else -#endif -#if defined(HAVE_CCM) - if (type == atom_aes_ccm) { - if ((7 <= iv.size && iv.size <= 13) - && (4 <= tag_len && tag_len <= 16) - && ((tag_len & 1) == 0) - ) { - ctx_ctrl_set_ivlen = EVP_CTRL_CCM_SET_IVLEN; - ctx_ctrl_get_tag = EVP_CTRL_CCM_GET_TAG; - if (key.size == 16) cipher = EVP_aes_128_ccm(); - else if (key.size == 24) cipher = EVP_aes_192_ccm(); - else if (key.size == 32) cipher = EVP_aes_256_ccm(); - else enif_make_badarg(env); - } else - enif_make_badarg(env); - } else -#endif -#if defined(HAVE_CHACHA20_POLY1305) - if (type == atom_chacha20_poly1305) { - if ((key.size == 32) - && (1 <= iv.size && iv.size <= 16) - && (tag_len == 16) - ) { - ctx_ctrl_set_ivlen = EVP_CTRL_AEAD_SET_IVLEN; - ctx_ctrl_get_tag = EVP_CTRL_AEAD_GET_TAG, - cipher = EVP_chacha20_poly1305(); - } else enif_make_badarg(env); - } else -#endif - return enif_raise_exception(env, atom_notsup); - - ctx = EVP_CIPHER_CTX_new(); - if (EVP_EncryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1) goto out_err; - if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_ivlen, iv.size, NULL) != 1) goto out_err; - -#if defined(HAVE_CCM) - if (type == atom_aes_ccm) { - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, NULL) != 1) goto out_err; - if (EVP_EncryptInit_ex(ctx, NULL, NULL, key.data, iv.data) != 1) goto out_err; - if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, in.size) != 1) goto out_err; - } else -#endif - if (EVP_EncryptInit_ex(ctx, NULL, NULL, key.data, iv.data) != 1) goto out_err; - - if (EVP_EncryptUpdate(ctx, NULL, &len, aad.data, aad.size) != 1) goto out_err; - - outp = enif_make_new_binary(env, in.size, &out); - - if (EVP_EncryptUpdate(ctx, outp, &len, in.data, in.size) != 1) goto out_err; - if (EVP_EncryptFinal_ex(ctx, outp/*+len*/, &len) != 1) goto out_err; - - tagp = enif_make_new_binary(env, tag_len, &out_tag); - - if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_get_tag, tag_len, tagp) != 1) goto out_err; - - EVP_CIPHER_CTX_free(ctx); - CONSUME_REDS(env, in); - return enif_make_tuple2(env, out, out_tag); - -out_err: - EVP_CIPHER_CTX_free(ctx); - return atom_error; - -#else - return enif_raise_exception(env, atom_notsup); -#endif -} - -static ERL_NIF_TERM aead_decrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type,Key,Iv,AAD,In,Tag) */ -#if defined(HAVE_AEAD) - EVP_CIPHER_CTX *ctx; - const EVP_CIPHER *cipher = NULL; - ErlNifBinary key, iv, aad, in, tag; - unsigned char *outp; - ERL_NIF_TERM type, out; - int len, ctx_ctrl_set_ivlen, ctx_ctrl_set_tag; - - type = argv[0]; -#if defined(HAVE_GCM_EVP_DECRYPT_BUG) - if (type == atom_aes_gcm) - return aes_gcm_decrypt_NO_EVP(env, argc, argv); -#endif - - if (!enif_is_atom(env, type) - || !enif_inspect_iolist_as_binary(env, argv[1], &key) - || !enif_inspect_binary(env, argv[2], &iv) - || !enif_inspect_iolist_as_binary(env, argv[3], &aad) - || !enif_inspect_iolist_as_binary(env, argv[4], &in) - || !enif_inspect_iolist_as_binary(env, argv[5], &tag)) { - return enif_make_badarg(env); - } - - /* Use cipher_type some day. Must check block_encrypt|decrypt first */ -#if defined(HAVE_GCM) - if (type == atom_aes_gcm) { - if (iv.size > 0) { - ctx_ctrl_set_ivlen = EVP_CTRL_GCM_SET_IVLEN; - ctx_ctrl_set_tag = EVP_CTRL_GCM_SET_TAG; - if (key.size == 16) cipher = EVP_aes_128_gcm(); - else if (key.size == 24) cipher = EVP_aes_192_gcm(); - else if (key.size == 32) cipher = EVP_aes_256_gcm(); - else enif_make_badarg(env); - } else - enif_make_badarg(env); - } else -#endif -#if defined(HAVE_CCM) - if (type == atom_aes_ccm) { - if (iv.size > 0) { - ctx_ctrl_set_ivlen = EVP_CTRL_CCM_SET_IVLEN; - if (key.size == 16) cipher = EVP_aes_128_ccm(); - else if (key.size == 24) cipher = EVP_aes_192_ccm(); - else if (key.size == 32) cipher = EVP_aes_256_ccm(); - else enif_make_badarg(env); - } else - enif_make_badarg(env); - } else -#endif -#if defined(HAVE_CHACHA20_POLY1305) - if (type == atom_chacha20_poly1305) { - if ((key.size == 32) - && (1 <= iv.size && iv.size <= 16) - && tag.size == 16 - ) { - ctx_ctrl_set_ivlen = EVP_CTRL_AEAD_SET_IVLEN; - ctx_ctrl_set_tag = EVP_CTRL_AEAD_SET_TAG; - cipher = EVP_chacha20_poly1305(); - } else enif_make_badarg(env); - } else -#endif - return enif_raise_exception(env, atom_notsup); - - outp = enif_make_new_binary(env, in.size, &out); - - ctx = EVP_CIPHER_CTX_new(); - if (EVP_DecryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1) goto out_err; - if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_ivlen, iv.size, NULL) != 1) goto out_err; - -#if defined(HAVE_CCM) - if (type == atom_aes_ccm) { - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag.size, tag.data) != 1) goto out_err; - } -#endif - - if (EVP_DecryptInit_ex(ctx, NULL, NULL, key.data, iv.data) != 1) goto out_err; - -#if defined(HAVE_CCM) - if (type == atom_aes_ccm) { - if (1 != EVP_DecryptUpdate(ctx, NULL, &len, NULL, in.size)) goto out_err; - } -#endif - - if (EVP_DecryptUpdate(ctx, NULL, &len, aad.data, aad.size) != 1) goto out_err; - if (EVP_DecryptUpdate(ctx, outp, &len, in.data, in.size) != 1) goto out_err; - -#if defined(HAVE_GCM) || defined(HAVE_CHACHA20_POLY1305) - if (type == atom_aes_gcm) { - if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_tag, tag.size, tag.data) != 1) goto out_err; - if (EVP_DecryptFinal_ex(ctx, outp+len, &len) != 1) goto out_err; - } -#endif - EVP_CIPHER_CTX_free(ctx); - - CONSUME_REDS(env, in); - return out; - -out_err: - EVP_CIPHER_CTX_free(ctx); - return atom_error; -#else - return enif_raise_exception(env, atom_notsup); -#endif -} - -#ifdef HAVE_GCM_EVP_DECRYPT_BUG -static ERL_NIF_TERM aes_gcm_decrypt_NO_EVP(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type,Key,Iv,AAD,In,Tag) */ - GCM128_CONTEXT *ctx; - ErlNifBinary key, iv, aad, in, tag; - AES_KEY aes_key; - unsigned char *outp; - ERL_NIF_TERM out; - - if (!enif_inspect_iolist_as_binary(env, argv[1], &key) - || AES_set_encrypt_key(key.data, key.size*8, &aes_key) != 0 - || !enif_inspect_binary(env, argv[2], &iv) || iv.size == 0 - || !enif_inspect_iolist_as_binary(env, argv[3], &aad) - || !enif_inspect_iolist_as_binary(env, argv[4], &in) - || !enif_inspect_iolist_as_binary(env, argv[5], &tag)) { - return enif_make_badarg(env); - } - - if (!(ctx = CRYPTO_gcm128_new(&aes_key, (block128_f)AES_encrypt))) - return atom_error; - - CRYPTO_gcm128_setiv(ctx, iv.data, iv.size); - - if (CRYPTO_gcm128_aad(ctx, aad.data, aad.size)) - goto out_err; - - outp = enif_make_new_binary(env, in.size, &out); - - /* decrypt */ - if (CRYPTO_gcm128_decrypt(ctx, in.data, outp, in.size)) - goto out_err; - - /* calculate and check the tag */ - if (CRYPTO_gcm128_finish(ctx, tag.data, tag.size)) - goto out_err; - - CRYPTO_gcm128_release(ctx); - CONSUME_REDS(env, in); - - return out; - -out_err: - CRYPTO_gcm128_release(ctx); - return atom_error; -} -#endif /* HAVE_GCM_EVP_DECRYPT_BUG */ - - -static ERL_NIF_TERM chacha20_stream_init(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key, IV) */ -#if defined(HAVE_CHACHA20) - ErlNifBinary key_bin, ivec_bin; - struct evp_cipher_ctx *ctx; - const EVP_CIPHER *cipher; - ERL_NIF_TERM ret; - - if (!enif_inspect_iolist_as_binary(env, argv[0], &key_bin) - || !enif_inspect_binary(env, argv[1], &ivec_bin) - || key_bin.size != 32 - || ivec_bin.size != 16) { - return enif_make_badarg(env); - } - - cipher = EVP_chacha20(); - - ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx)); - ctx->ctx = EVP_CIPHER_CTX_new(); - - - EVP_CipherInit_ex(ctx->ctx, cipher, NULL, - key_bin.data, ivec_bin.data, 1); - EVP_CIPHER_CTX_set_padding(ctx->ctx, 0); - ret = enif_make_resource(env, ctx); - enif_release_resource(ctx); - return ret; -#else - return enif_raise_exception(env, atom_notsup); -#endif -}; - -static ERL_NIF_TERM chacha20_stream_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (State, Data) */ -#if defined(HAVE_CHACHA20) - struct evp_cipher_ctx *ctx, *new_ctx; - ErlNifBinary data_bin; - ERL_NIF_TERM ret, cipher_term; - unsigned char *out; - int outl = 0; - - if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx) - || !enif_inspect_iolist_as_binary(env, argv[1], &data_bin)) { - return enif_make_badarg(env); - } - new_ctx = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx)); - new_ctx->ctx = EVP_CIPHER_CTX_new(); - EVP_CIPHER_CTX_copy(new_ctx->ctx, ctx->ctx); - out = enif_make_new_binary(env, data_bin.size, &cipher_term); - EVP_CipherUpdate(new_ctx->ctx, out, &outl, data_bin.data, data_bin.size); - ASSERT(outl == data_bin.size); - - ret = enif_make_tuple2(env, enif_make_resource(env, new_ctx), cipher_term); - enif_release_resource(new_ctx); - CONSUME_REDS(env,data_bin); - return ret; -#else - return enif_raise_exception(env, atom_notsup); -#endif -}; - - -static ERL_NIF_TERM strong_rand_bytes_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Bytes) */ - unsigned bytes; - unsigned char* data; - ERL_NIF_TERM ret; - - if (!enif_get_uint(env, argv[0], &bytes)) { - return enif_make_badarg(env); - } - data = enif_make_new_binary(env, bytes, &ret); - if ( RAND_bytes(data, bytes) != 1) { - return atom_false; - } - ERL_VALGRIND_MAKE_MEM_DEFINED(data, bytes); - return ret; -} - - -static int get_bn_from_mpint(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp) -{ - ErlNifBinary bin; - int sz; - if (!enif_inspect_binary(env,term,&bin)) { - return 0; - } - ERL_VALGRIND_ASSERT_MEM_DEFINED(bin.data, bin.size); - sz = bin.size - 4; - if (sz < 0 || get_int32(bin.data) != sz) { - return 0; - } - *bnp = BN_bin2bn(bin.data+4, sz, NULL); - return 1; -} - -static int get_bn_from_bin(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp) -{ - ErlNifBinary bin; - if (!enif_inspect_binary(env,term,&bin)) { - return 0; - } - ERL_VALGRIND_ASSERT_MEM_DEFINED(bin.data, bin.size); - *bnp = BN_bin2bn(bin.data, bin.size, NULL); - return 1; -} - -static ERL_NIF_TERM bin_from_bn(ErlNifEnv* env, const BIGNUM *bn) -{ - int bn_len; - unsigned char *bin_ptr; - ERL_NIF_TERM term; - - /* Copy the bignum into an erlang binary. */ - bn_len = BN_num_bytes(bn); - bin_ptr = enif_make_new_binary(env, bn_len, &term); - BN_bn2bin(bn, bin_ptr); - - return term; -} - -static ERL_NIF_TERM strong_rand_range_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Range) */ - BIGNUM *bn_range, *bn_rand; - ERL_NIF_TERM ret; - - if(!get_bn_from_bin(env, argv[0], &bn_range)) { - return enif_make_badarg(env); - } - - bn_rand = BN_new(); - if (BN_rand_range(bn_rand, bn_range) != 1) { - ret = atom_false; - } - else { - ret = bin_from_bn(env, bn_rand); - } - BN_free(bn_rand); - BN_free(bn_range); - return ret; -} - -static ERL_NIF_TERM rand_uniform_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Lo,Hi) */ - BIGNUM *bn_from = NULL, *bn_to, *bn_rand; - unsigned char* data; - unsigned dlen; - ERL_NIF_TERM ret; - - if (!get_bn_from_mpint(env, argv[0], &bn_from) - || !get_bn_from_mpint(env, argv[1], &bn_rand)) { - if (bn_from) BN_free(bn_from); - return enif_make_badarg(env); - } - - bn_to = BN_new(); - BN_sub(bn_to, bn_rand, bn_from); - BN_pseudo_rand_range(bn_rand, bn_to); - BN_add(bn_rand, bn_rand, bn_from); - dlen = BN_num_bytes(bn_rand); - data = enif_make_new_binary(env, dlen+4, &ret); - put_int32(data, dlen); - BN_bn2bin(bn_rand, data+4); - ERL_VALGRIND_MAKE_MEM_DEFINED(data+4, dlen); - BN_free(bn_rand); - BN_free(bn_from); - BN_free(bn_to); - return ret; -} - -static ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Base,Exponent,Modulo,bin_hdr) */ - BIGNUM *bn_base=NULL, *bn_exponent=NULL, *bn_modulo=NULL, *bn_result; - BN_CTX *bn_ctx; - unsigned char* ptr; - unsigned dlen; - unsigned bin_hdr; /* return type: 0=plain binary, 4: mpint */ - unsigned extra_byte; - ERL_NIF_TERM ret; - - if (!get_bn_from_bin(env, argv[0], &bn_base) - || !get_bn_from_bin(env, argv[1], &bn_exponent) - || !get_bn_from_bin(env, argv[2], &bn_modulo) - || !enif_get_uint(env,argv[3],&bin_hdr) || (bin_hdr & ~4)) { - - if (bn_base) BN_free(bn_base); - if (bn_exponent) BN_free(bn_exponent); - if (bn_modulo) BN_free(bn_modulo); - return enif_make_badarg(env); - } - bn_result = BN_new(); - bn_ctx = BN_CTX_new(); - BN_mod_exp(bn_result, bn_base, bn_exponent, bn_modulo, bn_ctx); - dlen = BN_num_bytes(bn_result); - extra_byte = bin_hdr && BN_is_bit_set(bn_result, dlen*8-1); - ptr = enif_make_new_binary(env, bin_hdr+extra_byte+dlen, &ret); - if (bin_hdr) { - put_int32(ptr, extra_byte+dlen); - ptr[4] = 0; /* extra zeroed byte to ensure a positive mpint */ - ptr += bin_hdr + extra_byte; - } - BN_bn2bin(bn_result, ptr); - BN_free(bn_result); - BN_CTX_free(bn_ctx); - BN_free(bn_modulo); - BN_free(bn_exponent); - BN_free(bn_base); - return ret; -} - -static void init_digest_types(ErlNifEnv* env) -{ - struct digest_type_t* p = digest_types; - - for (p = digest_types; p->type.str; p++) { - p->type.atom = enif_make_atom(env, p->type.str); - if (p->md.funcp) - p->md.p = p->md.funcp(); - } - p->type.atom = atom_false; /* end marker */ -} - -static void init_cipher_types(ErlNifEnv* env) -{ - struct cipher_type_t* p = cipher_types; - - for (p = cipher_types; p->type.str; p++) { - p->type.atom = enif_make_atom(env, p->type.str); - if (p->cipher.funcp) - p->cipher.p = p->cipher.funcp(); - } - p->type.atom = atom_false; /* end marker */ -} - -static struct digest_type_t* get_digest_type(ERL_NIF_TERM type) -{ - struct digest_type_t* p = NULL; - for (p = digest_types; p->type.atom != atom_false; p++) { - if (type == p->type.atom) { - return p; - } - } - return NULL; -} - -static struct cipher_type_t* get_cipher_type(ERL_NIF_TERM type, size_t key_len) -{ - struct cipher_type_t* p = NULL; - for (p = cipher_types; p->type.atom != atom_false; p++) { - if (type == p->type.atom && (!p->key_len || key_len == p->key_len)) { - return p; - } - } - return NULL; -} - - -static ERL_NIF_TERM do_exor(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Data1, Data2) */ - ErlNifBinary d1, d2; - unsigned char* ret_ptr; - int i; - ERL_NIF_TERM ret; - - if (!enif_inspect_iolist_as_binary(env,argv[0], &d1) - || !enif_inspect_iolist_as_binary(env,argv[1], &d2) - || d1.size != d2.size) { - return enif_make_badarg(env); - } - ret_ptr = enif_make_new_binary(env, d1.size, &ret); - - for (i=0; i<d1.size; i++) { - ret_ptr[i] = d1.data[i] ^ d2.data[i]; - } - CONSUME_REDS(env,d1); - return ret; -} - -static ERL_NIF_TERM rc4_set_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Key) */ -#ifndef OPENSSL_NO_RC4 - ErlNifBinary key; - ERL_NIF_TERM ret; - - CHECK_NO_FIPS_MODE(); - - if (!enif_inspect_iolist_as_binary(env,argv[0], &key)) { - return enif_make_badarg(env); - } - RC4_set_key((RC4_KEY*)enif_make_new_binary(env, sizeof(RC4_KEY), &ret), - key.size, key.data); - return ret; -#else - return enif_raise_exception(env, atom_notsup); -#endif -} - -static ERL_NIF_TERM rc4_encrypt_with_state(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (State, Data) */ -#ifndef OPENSSL_NO_RC4 - ErlNifBinary state, data; - RC4_KEY* rc4_key; - ERL_NIF_TERM new_state, new_data; - - CHECK_NO_FIPS_MODE(); - - if (!enif_inspect_iolist_as_binary(env,argv[0], &state) - || state.size != sizeof(RC4_KEY) - || !enif_inspect_iolist_as_binary(env,argv[1], &data)) { - return enif_make_badarg(env); - } - rc4_key = (RC4_KEY*)enif_make_new_binary(env, sizeof(RC4_KEY), &new_state); - memcpy(rc4_key, state.data, sizeof(RC4_KEY)); - RC4(rc4_key, data.size, data.data, - enif_make_new_binary(env, data.size, &new_data)); - CONSUME_REDS(env,data); - return enif_make_tuple2(env,new_state,new_data); -#else - return enif_raise_exception(env, atom_notsup); -#endif -} - -static int get_rsa_private_key(ErlNifEnv* env, ERL_NIF_TERM key, RSA *rsa) -{ - /* key=[E,N,D]|[E,N,D,P1,P2,E1,E2,C] */ - ERL_NIF_TERM head, tail; - BIGNUM *e, *n, *d; - BIGNUM *p, *q; - BIGNUM *dmp1, *dmq1, *iqmp; - - if (!enif_get_list_cell(env, key, &head, &tail) - || !get_bn_from_bin(env, head, &e) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &n) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &d)) { - return 0; - } - (void) RSA_set0_key(rsa, n, e, d); - if (enif_is_empty_list(env, tail)) { - return 1; - } - if (!enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &p) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &q) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dmp1) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dmq1) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &iqmp) - || !enif_is_empty_list(env, tail)) { - return 0; - } - (void) RSA_set0_factors(rsa, p, q); - (void) RSA_set0_crt_params(rsa, dmp1, dmq1, iqmp); - return 1; -} - - -static int get_rsa_public_key(ErlNifEnv* env, ERL_NIF_TERM key, RSA *rsa) -{ - /* key=[E,N] */ - ERL_NIF_TERM head, tail; - BIGNUM *e, *n; - - if (!enif_get_list_cell(env, key, &head, &tail) - || !get_bn_from_bin(env, head, &e) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &n) - || !enif_is_empty_list(env, tail)) { - return 0; - } - - (void) RSA_set0_key(rsa, n, e, NULL); - return 1; -} - -#ifdef HAVE_EDDSA - static int get_eddsa_key(ErlNifEnv* env, int public, ERL_NIF_TERM key, EVP_PKEY **pkey) -{ - /* key=[K] */ - ERL_NIF_TERM head, tail, tail2, algo; - ErlNifBinary bin; - int type; - - if (!enif_get_list_cell(env, key, &head, &tail) - || !enif_inspect_binary(env, head, &bin) - || !enif_get_list_cell(env, tail, &algo, &tail2) - || !enif_is_empty_list(env, tail2)) { - return 0; - } - if (algo == atom_ed25519) type = EVP_PKEY_ED25519; - else if (algo == atom_ed448) type = EVP_PKEY_ED448; - else - return 0; - - if (public) - *pkey = EVP_PKEY_new_raw_public_key(type, NULL, bin.data, bin.size); - else - *pkey = EVP_PKEY_new_raw_private_key(type, NULL, bin.data, bin.size); - - if (!pkey) - return 0; - return 1; -} -#endif - -static int get_dss_private_key(ErlNifEnv* env, ERL_NIF_TERM key, DSA *dsa) -{ - /* key=[P,Q,G,KEY] */ - ERL_NIF_TERM head, tail; - BIGNUM *dsa_p = NULL, *dsa_q = NULL, *dsa_g = NULL; - BIGNUM *dummy_pub_key, *priv_key = NULL; - - if (!enif_get_list_cell(env, key, &head, &tail) - || !get_bn_from_bin(env, head, &dsa_p) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dsa_q) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dsa_g) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &priv_key) - || !enif_is_empty_list(env,tail)) { - if (dsa_p) BN_free(dsa_p); - if (dsa_q) BN_free(dsa_q); - if (dsa_g) BN_free(dsa_g); - if (priv_key) BN_free(priv_key); - return 0; - } - - /* Note: DSA_set0_key() does not allow setting only the - * private key, although DSA_sign() does not use the - * public key. Work around this limitation by setting - * the public key to a copy of the private key. - */ - dummy_pub_key = BN_dup(priv_key); - - DSA_set0_pqg(dsa, dsa_p, dsa_q, dsa_g); - DSA_set0_key(dsa, dummy_pub_key, priv_key); - return 1; -} - - -static int get_dss_public_key(ErlNifEnv* env, ERL_NIF_TERM key, DSA *dsa) -{ - /* key=[P, Q, G, Y] */ - ERL_NIF_TERM head, tail; - BIGNUM *dsa_p = NULL, *dsa_q = NULL, *dsa_g = NULL, *dsa_y = NULL; - - if (!enif_get_list_cell(env, key, &head, &tail) - || !get_bn_from_bin(env, head, &dsa_p) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dsa_q) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dsa_g) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dsa_y) - || !enif_is_empty_list(env,tail)) { - if (dsa_p) BN_free(dsa_p); - if (dsa_q) BN_free(dsa_q); - if (dsa_g) BN_free(dsa_g); - if (dsa_y) BN_free(dsa_y); - return 0; - } - - DSA_set0_pqg(dsa, dsa_p, dsa_q, dsa_g); - DSA_set0_key(dsa, dsa_y, NULL); - return 1; -} - -/* Creates a term which can be parsed by get_rsa_private_key(). This is a list of plain integer binaries (not mpints). */ -static ERL_NIF_TERM put_rsa_private_key(ErlNifEnv* env, const RSA *rsa) -{ - ERL_NIF_TERM result[8]; - const BIGNUM *n, *e, *d, *p, *q, *dmp1, *dmq1, *iqmp; - - /* Return at least [E,N,D] */ - n = NULL; e = NULL; d = NULL; - RSA_get0_key(rsa, &n, &e, &d); - - result[0] = bin_from_bn(env, e); // Exponent E - result[1] = bin_from_bn(env, n); // Modulus N = p*q - result[2] = bin_from_bn(env, d); // Exponent D - - /* Check whether the optional additional parameters are available */ - p = NULL; q = NULL; - RSA_get0_factors(rsa, &p, &q); - dmp1 = NULL; dmq1 = NULL; iqmp = NULL; - RSA_get0_crt_params(rsa, &dmp1, &dmq1, &iqmp); - - if (p && q && dmp1 && dmq1 && iqmp) { - result[3] = bin_from_bn(env, p); // Factor p - result[4] = bin_from_bn(env, q); // Factor q - result[5] = bin_from_bn(env, dmp1); // D mod (p-1) - result[6] = bin_from_bn(env, dmq1); // D mod (q-1) - result[7] = bin_from_bn(env, iqmp); // (1/q) mod p - - return enif_make_list_from_array(env, result, 8); - } else { - return enif_make_list_from_array(env, result, 3); - } -} - -static int check_erlang_interrupt(int maj, int min, BN_GENCB *ctxt) -{ - ErlNifEnv *env = BN_GENCB_get_arg(ctxt); - - if (!enif_is_current_process_alive(env)) { - return 0; - } else { - return 1; - } -} - -static ERL_NIF_TERM rsa_generate_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (ModulusSize, PublicExponent) */ - int modulus_bits; - BIGNUM *pub_exp, *three; - RSA *rsa; - int success; - ERL_NIF_TERM result; - BN_GENCB *intr_cb; -#ifndef HAVE_OPAQUE_BN_GENCB - BN_GENCB intr_cb_buf; -#endif - - if (!enif_get_int(env, argv[0], &modulus_bits) || modulus_bits < 256) { - return enif_make_badarg(env); - } - - if (!get_bn_from_bin(env, argv[1], &pub_exp)) { - return enif_make_badarg(env); - } - - /* Make sure the public exponent is large enough (at least 3). - * Without this, RSA_generate_key_ex() can run forever. */ - three = BN_new(); - BN_set_word(three, 3); - success = BN_cmp(pub_exp, three); - BN_free(three); - if (success < 0) { - BN_free(pub_exp); - return enif_make_badarg(env); - } - - /* For large keys, prime generation can take many seconds. Set up - * the callback which we use to test whether the process has been - * interrupted. */ -#ifdef HAVE_OPAQUE_BN_GENCB - intr_cb = BN_GENCB_new(); -#else - intr_cb = &intr_cb_buf; -#endif - BN_GENCB_set(intr_cb, check_erlang_interrupt, env); - - rsa = RSA_new(); - success = RSA_generate_key_ex(rsa, modulus_bits, pub_exp, intr_cb); - BN_free(pub_exp); - -#ifdef HAVE_OPAQUE_BN_GENCB - BN_GENCB_free(intr_cb); -#endif - - if (!success) { - RSA_free(rsa); - return atom_error; - } - - result = put_rsa_private_key(env, rsa); - RSA_free(rsa); - - return result; -} - -static ERL_NIF_TERM rsa_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - /* RSA key generation can take a long time (>1 sec for a large - * modulus), so schedule it as a CPU-bound operation. */ - return enif_schedule_nif(env, "rsa_generate_key", - ERL_NIF_DIRTY_JOB_CPU_BOUND, - rsa_generate_key, argc, argv); -} - -static ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (PrivKey|undefined, DHParams=[P,G], Mpint, Len|0) */ - DH *dh_params = NULL; - int mpint; /* 0 or 4 */ - - { - ERL_NIF_TERM head, tail; - BIGNUM - *dh_p = NULL, - *dh_g = NULL, - *priv_key_in = NULL; - unsigned long - len = 0; - - if (!(get_bn_from_bin(env, argv[0], &priv_key_in) - || argv[0] == atom_undefined) - || !enif_get_list_cell(env, argv[1], &head, &tail) - || !get_bn_from_bin(env, head, &dh_p) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dh_g) - || !enif_is_empty_list(env, tail) - || !enif_get_int(env, argv[2], &mpint) || (mpint & ~4) - || !enif_get_ulong(env, argv[3], &len) - - /* Load dh_params with values to use by the generator. - Mem mgmnt transfered from dh_p etc to dh_params */ - || !(dh_params = DH_new()) - || (priv_key_in && !DH_set0_key(dh_params, NULL, priv_key_in)) - || !DH_set0_pqg(dh_params, dh_p, NULL, dh_g) - ) { - if (priv_key_in) BN_free(priv_key_in); - if (dh_p) BN_free(dh_p); - if (dh_g) BN_free(dh_g); - if (dh_params) DH_free(dh_params); - return enif_make_badarg(env); - } - - if (len) { - if (len < BN_num_bits(dh_p)) - DH_set_length(dh_params, len); - else { - if (priv_key_in) BN_free(priv_key_in); - if (dh_p) BN_free(dh_p); - if (dh_g) BN_free(dh_g); - if (dh_params) DH_free(dh_params); - return enif_make_badarg(env); - } - } - } - -#ifdef HAS_EVP_PKEY_CTX - { - EVP_PKEY_CTX *ctx; - EVP_PKEY *dhkey, *params; - int success; - - params = EVP_PKEY_new(); - success = EVP_PKEY_set1_DH(params, dh_params); /* set the key referenced by params to dh_params... */ - DH_free(dh_params); /* ...dh_params (and params) must be freed */ - if (!success) return atom_error; - - ctx = EVP_PKEY_CTX_new(params, NULL); - EVP_PKEY_free(params); - if (!ctx) { - return atom_error; - } - - if (!EVP_PKEY_keygen_init(ctx)) { - /* EVP_PKEY_CTX_free(ctx); */ - return atom_error; - } - - dhkey = EVP_PKEY_new(); - if (!EVP_PKEY_keygen(ctx, &dhkey)) { /* "performs a key generation operation, the ... */ - /*... generated key is written to ppkey." (=last arg) */ - /* EVP_PKEY_CTX_free(ctx); */ - /* EVP_PKEY_free(dhkey); */ - return atom_error; - } - - dh_params = EVP_PKEY_get1_DH(dhkey); /* return the referenced key. dh_params and dhkey must be freed */ - EVP_PKEY_free(dhkey); - if (!dh_params) { - /* EVP_PKEY_CTX_free(ctx); */ - return atom_error; - } - EVP_PKEY_CTX_free(ctx); - } -#else - if (!DH_generate_key(dh_params)) return atom_error; -#endif - { - unsigned char *pub_ptr, *prv_ptr; - int pub_len, prv_len; - ERL_NIF_TERM ret_pub, ret_prv; - const BIGNUM *pub_key_gen, *priv_key_gen; - - DH_get0_key(dh_params, - &pub_key_gen, &priv_key_gen); /* Get pub_key_gen and priv_key_gen. - "The values point to the internal representation of - the public key and private key values. This memory - should not be freed directly." says man */ - pub_len = BN_num_bytes(pub_key_gen); - prv_len = BN_num_bytes(priv_key_gen); - pub_ptr = enif_make_new_binary(env, pub_len+mpint, &ret_pub); - prv_ptr = enif_make_new_binary(env, prv_len+mpint, &ret_prv); - if (mpint) { - put_int32(pub_ptr, pub_len); pub_ptr += 4; - put_int32(prv_ptr, prv_len); prv_ptr += 4; - } - BN_bn2bin(pub_key_gen, pub_ptr); - BN_bn2bin(priv_key_gen, prv_ptr); - ERL_VALGRIND_MAKE_MEM_DEFINED(pub_ptr, pub_len); - ERL_VALGRIND_MAKE_MEM_DEFINED(prv_ptr, prv_len); - - DH_free(dh_params); - - return enif_make_tuple2(env, ret_pub, ret_prv); - } -} - -static ERL_NIF_TERM dh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (OthersPublicKey, MyPrivateKey, DHParams=[P,G]) */ - BIGNUM *other_pub_key = NULL, - *dh_p = NULL, - *dh_g = NULL; - DH *dh_priv = DH_new(); - - /* Check the arguments and get - my private key (dh_priv), - the peer's public key (other_pub_key), - the parameters p & q - */ - - { - BIGNUM *dummy_pub_key = NULL, - *priv_key = NULL; - ERL_NIF_TERM head, tail; - - if (!get_bn_from_bin(env, argv[0], &other_pub_key) - || !get_bn_from_bin(env, argv[1], &priv_key) - || !enif_get_list_cell(env, argv[2], &head, &tail) - || !get_bn_from_bin(env, head, &dh_p) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dh_g) - || !enif_is_empty_list(env, tail) - - /* Note: DH_set0_key() does not allow setting only the - * private key, although DH_compute_key() does not use the - * public key. Work around this limitation by setting - * the public key to a copy of the private key. - */ - || !(dummy_pub_key = BN_dup(priv_key)) - || !DH_set0_key(dh_priv, dummy_pub_key, priv_key) - || !DH_set0_pqg(dh_priv, dh_p, NULL, dh_g) - ) { - if (dh_p) BN_free(dh_p); - if (dh_g) BN_free(dh_g); - if (other_pub_key) BN_free(other_pub_key); - if (dummy_pub_key) BN_free(dummy_pub_key); - if (priv_key) BN_free(priv_key); - return enif_make_badarg(env); - } - } - { - ErlNifBinary ret_bin; - int size; - - enif_alloc_binary(DH_size(dh_priv), &ret_bin); - size = DH_compute_key(ret_bin.data, other_pub_key, dh_priv); - BN_free(other_pub_key); - DH_free(dh_priv); - if (size<=0) { - enif_release_binary(&ret_bin); - return atom_error; - } - - if (size != ret_bin.size) enif_realloc_binary(&ret_bin, size); - return enif_make_binary(env, &ret_bin); - } -} - - -static ERL_NIF_TERM srp_value_B_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Multiplier, Verifier, Generator, Exponent, Prime) */ - BIGNUM *bn_verifier = NULL; - BIGNUM *bn_exponent = NULL, *bn_generator = NULL, *bn_prime = NULL, *bn_multiplier = NULL, *bn_result; - BN_CTX *bn_ctx; - unsigned char* ptr; - unsigned dlen; - ERL_NIF_TERM ret; - - CHECK_NO_FIPS_MODE(); - - if (!get_bn_from_bin(env, argv[0], &bn_multiplier) - || !get_bn_from_bin(env, argv[1], &bn_verifier) - || !get_bn_from_bin(env, argv[2], &bn_generator) - || !get_bn_from_bin(env, argv[3], &bn_exponent) - || !get_bn_from_bin(env, argv[4], &bn_prime)) { - if (bn_multiplier) BN_free(bn_multiplier); - if (bn_verifier) BN_free(bn_verifier); - if (bn_generator) BN_free(bn_generator); - if (bn_exponent) BN_free(bn_exponent); - if (bn_prime) BN_free(bn_prime); - return enif_make_badarg(env); - } - - bn_result = BN_new(); - bn_ctx = BN_CTX_new(); - - /* B = k*v + g^b % N */ - - /* k * v */ - BN_mod_mul(bn_multiplier, bn_multiplier, bn_verifier, bn_prime, bn_ctx); - - /* g^b % N */ - BN_mod_exp(bn_result, bn_generator, bn_exponent, bn_prime, bn_ctx); - - /* k*v + g^b % N */ - BN_mod_add(bn_result, bn_result, bn_multiplier, bn_prime, bn_ctx); - - /* check that B % N != 0, reuse bn_multiplier */ - BN_nnmod(bn_multiplier, bn_result, bn_prime, bn_ctx); - if (BN_is_zero(bn_multiplier)) { - ret = atom_error; - } else { - dlen = BN_num_bytes(bn_result); - ptr = enif_make_new_binary(env, dlen, &ret); - BN_bn2bin(bn_result, ptr); - } - BN_free(bn_result); - BN_CTX_free(bn_ctx); - BN_free(bn_prime); - BN_free(bn_generator); - BN_free(bn_multiplier); - BN_free(bn_exponent); - BN_free(bn_verifier); - return ret; -} - -static ERL_NIF_TERM srp_user_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (a, u, B, Multiplier, Prime, Exponent, Generator) */ -/* - <premaster secret> = (B - (k * g^x)) ^ (a + (u * x)) % N -*/ - BIGNUM *bn_exponent = NULL, *bn_a = NULL; - BIGNUM *bn_u = NULL, *bn_multiplier = NULL, *bn_exp2, - *bn_base, *bn_prime = NULL, *bn_generator = NULL, - *bn_B = NULL, *bn_result; - BN_CTX *bn_ctx; - unsigned char* ptr; - unsigned dlen; - ERL_NIF_TERM ret; - - CHECK_NO_FIPS_MODE(); - - if (!get_bn_from_bin(env, argv[0], &bn_a) - || !get_bn_from_bin(env, argv[1], &bn_u) - || !get_bn_from_bin(env, argv[2], &bn_B) - || !get_bn_from_bin(env, argv[3], &bn_multiplier) - || !get_bn_from_bin(env, argv[4], &bn_generator) - || !get_bn_from_bin(env, argv[5], &bn_exponent) - || !get_bn_from_bin(env, argv[6], &bn_prime)) - { - if (bn_exponent) BN_free(bn_exponent); - if (bn_a) BN_free(bn_a); - if (bn_u) BN_free(bn_u); - if (bn_B) BN_free(bn_B); - if (bn_multiplier) BN_free(bn_multiplier); - if (bn_generator) BN_free(bn_generator); - if (bn_prime) BN_free(bn_prime); - return enif_make_badarg(env); - } - - bn_ctx = BN_CTX_new(); - bn_result = BN_new(); - - /* check that B % N != 0 */ - BN_nnmod(bn_result, bn_B, bn_prime, bn_ctx); - if (BN_is_zero(bn_result)) { - BN_free(bn_exponent); - BN_free(bn_a); - BN_free(bn_generator); - BN_free(bn_prime); - BN_free(bn_u); - BN_free(bn_B); - BN_CTX_free(bn_ctx); - - return atom_error; - } - - /* (B - (k * g^x)) */ - bn_base = BN_new(); - BN_mod_exp(bn_result, bn_generator, bn_exponent, bn_prime, bn_ctx); - BN_mod_mul(bn_result, bn_multiplier, bn_result, bn_prime, bn_ctx); - BN_mod_sub(bn_base, bn_B, bn_result, bn_prime, bn_ctx); - - /* a + (u * x) */ - bn_exp2 = BN_new(); - BN_mul(bn_result, bn_u, bn_exponent, bn_ctx); - BN_add(bn_exp2, bn_a, bn_result); - - /* (B - (k * g^x)) ^ (a + (u * x)) % N */ - BN_mod_exp(bn_result, bn_base, bn_exp2, bn_prime, bn_ctx); - - dlen = BN_num_bytes(bn_result); - ptr = enif_make_new_binary(env, dlen, &ret); - BN_bn2bin(bn_result, ptr); - BN_free(bn_result); - BN_CTX_free(bn_ctx); - - BN_free(bn_multiplier); - BN_free(bn_exp2); - BN_free(bn_u); - BN_free(bn_exponent); - BN_free(bn_a); - BN_free(bn_B); - BN_free(bn_base); - BN_free(bn_generator); - BN_free(bn_prime); - return ret; -} - -static ERL_NIF_TERM srp_host_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Verifier, b, u, A, Prime) */ -/* - <premaster secret> = (A * v^u) ^ b % N -*/ - BIGNUM *bn_b = NULL, *bn_verifier = NULL; - BIGNUM *bn_prime = NULL, *bn_A = NULL, *bn_u = NULL, *bn_base, *bn_result; - BN_CTX *bn_ctx; - unsigned char* ptr; - unsigned dlen; - ERL_NIF_TERM ret; - - CHECK_NO_FIPS_MODE(); - - if (!get_bn_from_bin(env, argv[0], &bn_verifier) - || !get_bn_from_bin(env, argv[1], &bn_b) - || !get_bn_from_bin(env, argv[2], &bn_u) - || !get_bn_from_bin(env, argv[3], &bn_A) - || !get_bn_from_bin(env, argv[4], &bn_prime)) - { - if (bn_verifier) BN_free(bn_verifier); - if (bn_b) BN_free(bn_b); - if (bn_u) BN_free(bn_u); - if (bn_A) BN_free(bn_A); - if (bn_prime) BN_free(bn_prime); - return enif_make_badarg(env); - } - - bn_ctx = BN_CTX_new(); - bn_result = BN_new(); - - /* check that A % N != 0 */ - BN_nnmod(bn_result, bn_A, bn_prime, bn_ctx); - if (BN_is_zero(bn_result)) { - BN_free(bn_b); - BN_free(bn_verifier); - BN_free(bn_prime); - BN_free(bn_A); - BN_CTX_free(bn_ctx); - - return atom_error; - } - - /* (A * v^u) */ - bn_base = BN_new(); - BN_mod_exp(bn_base, bn_verifier, bn_u, bn_prime, bn_ctx); - BN_mod_mul(bn_base, bn_A, bn_base, bn_prime, bn_ctx); - - /* (A * v^u) ^ b % N */ - BN_mod_exp(bn_result, bn_base, bn_b, bn_prime, bn_ctx); - - dlen = BN_num_bytes(bn_result); - ptr = enif_make_new_binary(env, dlen, &ret); - BN_bn2bin(bn_result, ptr); - BN_free(bn_result); - BN_CTX_free(bn_ctx); - - BN_free(bn_u); - BN_free(bn_base); - BN_free(bn_verifier); - BN_free(bn_prime); - BN_free(bn_A); - BN_free(bn_b); - return ret; -} - -#if defined(HAVE_EC) -static EC_KEY* ec_key_new(ErlNifEnv* env, ERL_NIF_TERM curve_arg) -{ - EC_KEY *key = NULL; - int c_arity = -1; - const ERL_NIF_TERM* curve; - ErlNifBinary seed; - BIGNUM *p = NULL; - BIGNUM *a = NULL; - BIGNUM *b = NULL; - BIGNUM *bn_order = NULL; - BIGNUM *cofactor = NULL; - EC_GROUP *group = NULL; - EC_POINT *point = NULL; - - /* {Field, Prime, Point, Order, CoFactor} = Curve */ - if (enif_get_tuple(env,curve_arg,&c_arity,&curve) - && c_arity == 5 - && get_bn_from_bin(env, curve[3], &bn_order) - && (curve[4] != atom_none && get_bn_from_bin(env, curve[4], &cofactor))) { - - int f_arity = -1; - const ERL_NIF_TERM* field; - int p_arity = -1; - const ERL_NIF_TERM* prime; - - long field_bits; - - /* {A, B, Seed} = Prime */ - if (!enif_get_tuple(env,curve[1],&p_arity,&prime) - || !get_bn_from_bin(env, prime[0], &a) - || !get_bn_from_bin(env, prime[1], &b)) - goto out_err; - - if (!enif_get_tuple(env,curve[0],&f_arity,&field)) - goto out_err; - - if (f_arity == 2 && field[0] == atom_prime_field) { - /* {prime_field, Prime} */ - - if (!get_bn_from_bin(env, field[1], &p)) - goto out_err; - - if (BN_is_negative(p) || BN_is_zero(p)) - goto out_err; - - field_bits = BN_num_bits(p); - if (field_bits > OPENSSL_ECC_MAX_FIELD_BITS) - goto out_err; - - /* create the EC_GROUP structure */ - group = EC_GROUP_new_curve_GFp(p, a, b, NULL); - - } else if (f_arity == 3 && field[0] == atom_characteristic_two_field) { -#if defined(OPENSSL_NO_EC2M) - enif_raise_exception(env, atom_notsup); - goto out_err; -#else - /* {characteristic_two_field, M, Basis} */ - - int b_arity = -1; - const ERL_NIF_TERM* basis; - unsigned int k1, k2, k3; - - if ((p = BN_new()) == NULL) - goto out_err; - - if (!enif_get_long(env, field[1], &field_bits) - || field_bits > OPENSSL_ECC_MAX_FIELD_BITS) - goto out_err; - - if (enif_get_tuple(env,field[2],&b_arity,&basis)) { - if (b_arity == 2 - && basis[0] == atom_tpbasis - && enif_get_uint(env, basis[1], &k1)) { - /* {tpbasis, k} = Basis */ - - if (!(field_bits > k1 && k1 > 0)) - goto out_err; - - /* create the polynomial */ - if (!BN_set_bit(p, (int)field_bits) - || !BN_set_bit(p, (int)k1) - || !BN_set_bit(p, 0)) - goto out_err; - - } else if (b_arity == 4 - && basis[0] == atom_ppbasis - && enif_get_uint(env, basis[1], &k1) - && enif_get_uint(env, basis[2], &k2) - && enif_get_uint(env, basis[3], &k3)) { - /* {ppbasis, k1, k2, k3} = Basis */ - - if (!(field_bits > k3 && k3 > k2 && k2 > k1 && k1 > 0)) - goto out_err; - - /* create the polynomial */ - if (!BN_set_bit(p, (int)field_bits) - || !BN_set_bit(p, (int)k1) - || !BN_set_bit(p, (int)k2) - || !BN_set_bit(p, (int)k3) - || !BN_set_bit(p, 0)) - goto out_err; - - } else - goto out_err; - } else if (field[2] == atom_onbasis) { - /* onbasis = Basis */ - /* no parameters */ - goto out_err; - - } else - goto out_err; - - group = EC_GROUP_new_curve_GF2m(p, a, b, NULL); -#endif - } else - goto out_err; - - if (!group) - goto out_err; - - if (enif_inspect_binary(env, prime[2], &seed)) { - EC_GROUP_set_seed(group, seed.data, seed.size); - } - - if (!term2point(env, curve[2], group, &point)) - goto out_err; - - if (BN_is_negative(bn_order) - || BN_is_zero(bn_order) - || BN_num_bits(bn_order) > (int)field_bits + 1) - goto out_err; - - if (!EC_GROUP_set_generator(group, point, bn_order, cofactor)) - goto out_err; - - EC_GROUP_set_asn1_flag(group, 0x0); - - key = EC_KEY_new(); - if (!key) - goto out_err; - EC_KEY_set_group(key, group); - } - else { - goto out_err; - } - - - goto out; - -out_err: - if (key) EC_KEY_free(key); - key = NULL; - -out: - /* some OpenSSL structures are mem-dup'ed into the key, - so we have to free our copies here */ - if (p) BN_free(p); - if (a) BN_free(a); - if (b) BN_free(b); - if (bn_order) BN_free(bn_order); - if (cofactor) BN_free(cofactor); - if (group) EC_GROUP_free(group); - if (point) EC_POINT_free(point); - - return key; -} - - -static ERL_NIF_TERM bn2term(ErlNifEnv* env, const BIGNUM *bn) -{ - unsigned dlen; - unsigned char* ptr; - ERL_NIF_TERM ret; - - if (!bn) - return atom_undefined; - - dlen = BN_num_bytes(bn); - ptr = enif_make_new_binary(env, dlen, &ret); - BN_bn2bin(bn, ptr); - ERL_VALGRIND_MAKE_MEM_DEFINED(ptr, dlen); - return ret; -} - -static ERL_NIF_TERM point2term(ErlNifEnv* env, - const EC_GROUP *group, - const EC_POINT *point, - point_conversion_form_t form) -{ - unsigned dlen; - ErlNifBinary bin; - - dlen = EC_POINT_point2oct(group, point, form, NULL, 0, NULL); - if (dlen == 0) - return atom_undefined; - - if (!enif_alloc_binary(dlen, &bin)) - return enif_make_badarg(env); - - if (!EC_POINT_point2oct(group, point, form, bin.data, bin.size, NULL)) { - enif_release_binary(&bin); - return enif_make_badarg(env); - } - ERL_VALGRIND_MAKE_MEM_DEFINED(bin.data, bin.size); - return enif_make_binary(env, &bin); -} - -static int term2point(ErlNifEnv* env, ERL_NIF_TERM term, - EC_GROUP *group, EC_POINT **pptr) -{ - int ret = 0; - ErlNifBinary bin; - EC_POINT *point; - - if (!enif_inspect_binary(env,term,&bin)) { - return 0; - } - - if ((*pptr = point = EC_POINT_new(group)) == NULL) { - return 0; - } - - /* set the point conversion form */ - EC_GROUP_set_point_conversion_form(group, (point_conversion_form_t)(bin.data[0] & ~0x01)); - - /* extract the ec point */ - if (!EC_POINT_oct2point(group, point, bin.data, bin.size, NULL)) { - EC_POINT_free(point); - *pptr = NULL; - } else - ret = 1; - - return ret; -} - -static int get_ec_key(ErlNifEnv* env, - ERL_NIF_TERM curve, ERL_NIF_TERM priv, ERL_NIF_TERM pub, - EC_KEY** res) -{ - EC_KEY *key = NULL; - BIGNUM *priv_key = NULL; - EC_POINT *pub_key = NULL; - EC_GROUP *group = NULL; - - if (!(priv == atom_undefined || get_bn_from_bin(env, priv, &priv_key)) - || !(pub == atom_undefined || enif_is_binary(env, pub))) { - goto out_err; - } - - key = ec_key_new(env, curve); - - if (!key) { - goto out_err; - } - - if (!group) - group = EC_GROUP_dup(EC_KEY_get0_group(key)); - - if (term2point(env, pub, group, &pub_key)) { - if (!EC_KEY_set_public_key(key, pub_key)) { - goto out_err; - } - } - if (priv != atom_undefined - && !BN_is_zero(priv_key)) { - if (!EC_KEY_set_private_key(key, priv_key)) - goto out_err; - - /* calculate public key (if necessary) */ - if (EC_KEY_get0_public_key(key) == NULL) - { - /* the public key was not included in the SEC1 private - * key => calculate the public key */ - pub_key = EC_POINT_new(group); - if (pub_key == NULL - || !EC_POINT_copy(pub_key, EC_GROUP_get0_generator(group)) - || !EC_POINT_mul(group, pub_key, priv_key, NULL, NULL, NULL) - || !EC_KEY_set_public_key(key, pub_key)) - goto out_err; - } - } - - goto out; - -out_err: - if (key) EC_KEY_free(key); - key = NULL; - -out: - /* some OpenSSL structures are mem-dup'ed into the key, - so we have to free our copies here */ - if (priv_key) BN_clear_free(priv_key); - if (pub_key) EC_POINT_free(pub_key); - if (group) EC_GROUP_free(group); - if (!key) - return 0; - *res = key; - return 1; -} -#endif /* HAVE_EC */ - -static ERL_NIF_TERM ec_key_generate(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ -#if defined(HAVE_EC) - EC_KEY *key = NULL; - const EC_GROUP *group; - const EC_POINT *public_key; - ERL_NIF_TERM priv_key; - ERL_NIF_TERM pub_key = atom_undefined; - - if (!get_ec_key(env, argv[0], argv[1], atom_undefined, &key)) - goto badarg; - - if (argv[1] == atom_undefined) { - if (!EC_KEY_generate_key(key)) - goto badarg; - } - - group = EC_KEY_get0_group(key); - public_key = EC_KEY_get0_public_key(key); - - if (group && public_key) { - pub_key = point2term(env, group, public_key, - EC_KEY_get_conv_form(key)); - } - priv_key = bn2term(env, EC_KEY_get0_private_key(key)); - EC_KEY_free(key); - return enif_make_tuple2(env, pub_key, priv_key); - -badarg: - if (key) - EC_KEY_free(key); - return make_badarg_maybe(env); -#else - return atom_notsup; -#endif -} - -/* - (_OthersPublicKey, _MyPrivateKey) - (_OthersPublicKey, _MyEC_Point) -*/ -static ERL_NIF_TERM ecdh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -/* (OtherPublicKey, Curve, My) */ -{ -#if defined(HAVE_EC) - ERL_NIF_TERM ret; - unsigned char *p; - EC_KEY* key = NULL; - int field_size = 0; - int i; - EC_GROUP *group; - const BIGNUM *priv_key; - EC_POINT *my_ecpoint = NULL; - EC_KEY *other_ecdh = NULL; - - if (!get_ec_key(env, argv[1], argv[2], atom_undefined, &key)) - return make_badarg_maybe(env); - - group = EC_GROUP_dup(EC_KEY_get0_group(key)); - priv_key = EC_KEY_get0_private_key(key); - - if (!term2point(env, argv[0], group, &my_ecpoint)) { - goto out_err; - } - - if ((other_ecdh = EC_KEY_new()) == NULL - || !EC_KEY_set_group(other_ecdh, group) - || !EC_KEY_set_private_key(other_ecdh, priv_key)) - goto out_err; - - field_size = EC_GROUP_get_degree(group); - if (field_size <= 0) - goto out_err; - - p = enif_make_new_binary(env, (field_size+7)/8, &ret); - i = ECDH_compute_key(p, (field_size+7)/8, my_ecpoint, other_ecdh, NULL); - - if (i < 0) - goto out_err; -out: - if (group) EC_GROUP_free(group); - if (my_ecpoint) EC_POINT_free(my_ecpoint); - if (other_ecdh) EC_KEY_free(other_ecdh); - if (key) EC_KEY_free(key); - - return ret; - -out_err: - ret = enif_make_badarg(env); - goto out; -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM evp_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) - /* (Curve, PeerBin, MyBin) */ -{ -#ifdef HAVE_ED_CURVE_DH - int type; - EVP_PKEY_CTX *ctx = NULL; - ErlNifBinary peer_bin, my_bin, key_bin; - EVP_PKEY *peer_key = NULL, *my_key = NULL; - size_t max_size; - - if (argv[0] == atom_x25519) type = EVP_PKEY_X25519; - else if (argv[0] == atom_x448) type = EVP_PKEY_X448; - else return enif_make_badarg(env); - - if (!enif_inspect_binary(env, argv[1], &peer_bin) || - !enif_inspect_binary(env, argv[2], &my_bin)) - goto return_badarg; - - if (!(my_key = EVP_PKEY_new_raw_private_key(type, NULL, my_bin.data, my_bin.size)) || - !(ctx = EVP_PKEY_CTX_new(my_key, NULL))) - goto return_badarg; - - if (!EVP_PKEY_derive_init(ctx)) - goto return_badarg; - - if (!(peer_key = EVP_PKEY_new_raw_public_key(type, NULL, peer_bin.data, peer_bin.size)) || - !EVP_PKEY_derive_set_peer(ctx, peer_key)) - goto return_badarg; - - if (!EVP_PKEY_derive(ctx, NULL, &max_size)) - goto return_badarg; - - if (!enif_alloc_binary(max_size, &key_bin) || - !EVP_PKEY_derive(ctx, key_bin.data, &key_bin.size)) - goto return_badarg; - - if (key_bin.size < max_size) { - size_t actual_size = key_bin.size; - if (!enif_realloc_binary(&key_bin, actual_size)) - goto return_badarg; - } - - EVP_PKEY_free(my_key); - EVP_PKEY_free(peer_key); - EVP_PKEY_CTX_free(ctx); - return enif_make_binary(env, &key_bin); - -return_badarg: - if (my_key) EVP_PKEY_free(my_key); - if (peer_key) EVP_PKEY_free(peer_key); - if (ctx) EVP_PKEY_CTX_free(ctx); - return enif_make_badarg(env); -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM evp_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -/* (Curve) */ -{ -#ifdef HAVE_ED_CURVE_DH - int type; - EVP_PKEY_CTX *ctx = NULL; - EVP_PKEY *pkey = NULL; - ERL_NIF_TERM ret_pub, ret_prv; - size_t key_len; - - if (argv[0] == atom_x25519) type = EVP_PKEY_X25519; - else if (argv[0] == atom_x448) type = EVP_PKEY_X448; - else return enif_make_badarg(env); - - if (!(ctx = EVP_PKEY_CTX_new_id(type, NULL))) return enif_make_badarg(env); - - if (!EVP_PKEY_keygen_init(ctx)) goto return_error; - if (!EVP_PKEY_keygen(ctx, &pkey)) goto return_error; - - if (!EVP_PKEY_get_raw_public_key(pkey, NULL, &key_len)) goto return_error; - if (!EVP_PKEY_get_raw_public_key(pkey, - enif_make_new_binary(env, key_len, &ret_pub), - &key_len)) - goto return_error; - - if (!EVP_PKEY_get_raw_private_key(pkey, NULL, &key_len)) goto return_error; - if (!EVP_PKEY_get_raw_private_key(pkey, - enif_make_new_binary(env, key_len, &ret_prv), - &key_len)) - goto return_error; - - EVP_PKEY_free(pkey); - EVP_PKEY_CTX_free(ctx); - return enif_make_tuple2(env, ret_pub, ret_prv); - -return_error: - if (pkey) EVP_PKEY_free(pkey); - if (ctx) EVP_PKEY_CTX_free(ctx); - return atom_error; - -#else - return atom_notsup; -#endif -} - -/*================================================================*/ -#define PKEY_BADARG -1 -#define PKEY_NOTSUP 0 -#define PKEY_OK 1 - -typedef struct PKeyCryptOptions { - const EVP_MD *rsa_mgf1_md; - ErlNifBinary rsa_oaep_label; - const EVP_MD *rsa_oaep_md; - int rsa_padding; - const EVP_MD *signature_md; -} PKeyCryptOptions; - -typedef struct PKeySignOptions { - const EVP_MD *rsa_mgf1_md; - int rsa_padding; - int rsa_pss_saltlen; -} PKeySignOptions; - -static int get_pkey_digest_type(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM type, - const EVP_MD **md) -{ - struct digest_type_t *digp = NULL; - *md = NULL; - - if (type == atom_none && algorithm == atom_rsa) return PKEY_OK; - if (algorithm == atom_eddsa) -#ifdef HAVE_EDDSA - return PKEY_OK; -#else - return PKEY_NOTSUP; -#endif - digp = get_digest_type(type); - if (!digp) return PKEY_BADARG; - if (!digp->md.p) return PKEY_NOTSUP; - - *md = digp->md.p; - return PKEY_OK; -} - - -static int get_pkey_sign_digest(ErlNifEnv *env, ERL_NIF_TERM algorithm, - ERL_NIF_TERM type, ERL_NIF_TERM data, - unsigned char *md_value, const EVP_MD **mdp, - unsigned char **tbsp, size_t *tbslenp) -{ - int i; - const ERL_NIF_TERM *tpl_terms; - int tpl_arity; - ErlNifBinary tbs_bin; - EVP_MD_CTX *mdctx; - const EVP_MD *md = *mdp; - unsigned char *tbs = *tbsp; - size_t tbslen = *tbslenp; - unsigned int tbsleni; - - if ((i = get_pkey_digest_type(env, algorithm, type, &md)) != PKEY_OK) { - return i; - } - if (enif_get_tuple(env, data, &tpl_arity, &tpl_terms)) { - if (tpl_arity != 2 || tpl_terms[0] != atom_digest - || !enif_inspect_binary(env, tpl_terms[1], &tbs_bin) - || (md != NULL && tbs_bin.size != EVP_MD_size(md))) { - return PKEY_BADARG; - } - /* We have a digest (= hashed text) in tbs_bin */ - tbs = tbs_bin.data; - tbslen = tbs_bin.size; - } else if (md == NULL) { - if (!enif_inspect_binary(env, data, &tbs_bin)) { - return PKEY_BADARG; - } - /* md == NULL, that is no hashing because DigestType argument was atom_none */ - tbs = tbs_bin.data; - tbslen = tbs_bin.size; - } else { - if (!enif_inspect_binary(env, data, &tbs_bin)) { - return PKEY_BADARG; - } - /* We have the cleartext in tbs_bin and the hash algo info in md */ - tbs = md_value; - mdctx = EVP_MD_CTX_create(); - if (!mdctx) { - return PKEY_BADARG; - } - /* Looks well, now hash the plain text into a digest according to md */ - if (EVP_DigestInit_ex(mdctx, md, NULL) <= 0) { - EVP_MD_CTX_destroy(mdctx); - return PKEY_BADARG; - } - if (EVP_DigestUpdate(mdctx, tbs_bin.data, tbs_bin.size) <= 0) { - EVP_MD_CTX_destroy(mdctx); - return PKEY_BADARG; - } - if (EVP_DigestFinal_ex(mdctx, tbs, &tbsleni) <= 0) { - EVP_MD_CTX_destroy(mdctx); - return PKEY_BADARG; - } - tbslen = (size_t)(tbsleni); - EVP_MD_CTX_destroy(mdctx); - } - - *mdp = md; - *tbsp = tbs; - *tbslenp = tbslen; - - return PKEY_OK; -} - - -static int get_pkey_sign_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM options, - const EVP_MD *md, PKeySignOptions *opt) -{ - ERL_NIF_TERM head, tail; - const ERL_NIF_TERM *tpl_terms; - int tpl_arity; - const EVP_MD *opt_md; - int i; - - if (!enif_is_list(env, options)) { - return PKEY_BADARG; - } - - /* defaults */ - if (algorithm == atom_rsa) { - opt->rsa_mgf1_md = NULL; - opt->rsa_padding = RSA_PKCS1_PADDING; - opt->rsa_pss_saltlen = -2; - } - - if (enif_is_empty_list(env, options)) { - return PKEY_OK; - } - - if (algorithm == atom_rsa) { - tail = options; - while (enif_get_list_cell(env, tail, &head, &tail)) { - if (enif_get_tuple(env, head, &tpl_arity, &tpl_terms) && tpl_arity == 2) { - if (tpl_terms[0] == atom_rsa_mgf1_md && enif_is_atom(env, tpl_terms[1])) { - i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); - if (i != PKEY_OK) { - return i; - } - opt->rsa_mgf1_md = opt_md; - } else if (tpl_terms[0] == atom_rsa_padding) { - if (tpl_terms[1] == atom_rsa_pkcs1_padding) { - opt->rsa_padding = RSA_PKCS1_PADDING; - } else if (tpl_terms[1] == atom_rsa_pkcs1_pss_padding) { -#ifdef HAVE_RSA_PKCS1_PSS_PADDING - opt->rsa_padding = RSA_PKCS1_PSS_PADDING; - if (opt->rsa_mgf1_md == NULL) { - opt->rsa_mgf1_md = md; - } -#else - return PKEY_NOTSUP; -#endif - } else if (tpl_terms[1] == atom_rsa_x931_padding) { - opt->rsa_padding = RSA_X931_PADDING; - } else if (tpl_terms[1] == atom_rsa_no_padding) { - opt->rsa_padding = RSA_NO_PADDING; - } else { - return PKEY_BADARG; - } - } else if (tpl_terms[0] == atom_rsa_pss_saltlen) { - if (!enif_get_int(env, tpl_terms[1], &(opt->rsa_pss_saltlen)) - || opt->rsa_pss_saltlen < -2) { - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } - } - } else { - return PKEY_BADARG; - } - - return PKEY_OK; -} - - -#ifdef HAS_ENGINE_SUPPORT -static int get_engine_and_key_id(ErlNifEnv *env, ERL_NIF_TERM key, char ** id, ENGINE **e) -{ - ERL_NIF_TERM engine_res, key_id_term; - struct engine_ctx *ctx; - ErlNifBinary key_id_bin; - - if (!enif_get_map_value(env, key, atom_engine, &engine_res) || - !enif_get_resource(env, engine_res, engine_ctx_rtype, (void**)&ctx) || - !enif_get_map_value(env, key, atom_key_id, &key_id_term) || - !enif_inspect_binary(env, key_id_term, &key_id_bin)) { - return 0; - } - else { - *e = ctx->engine; - return zero_terminate(key_id_bin, id); - } -} - - -static char *get_key_password(ErlNifEnv *env, ERL_NIF_TERM key) { - ERL_NIF_TERM tmp_term; - ErlNifBinary pwd_bin; - char *pwd = NULL; - if (enif_get_map_value(env, key, atom_password, &tmp_term) && - enif_inspect_binary(env, tmp_term, &pwd_bin) && - zero_terminate(pwd_bin, &pwd) - ) return pwd; - - return NULL; -} - -static int zero_terminate(ErlNifBinary bin, char **buf) { - *buf = enif_alloc(bin.size+1); - if (!*buf) - return 0; - memcpy(*buf, bin.data, bin.size); - *(*buf+bin.size) = 0; - return 1; -} -#endif - -static int get_pkey_private_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM key, EVP_PKEY **pkey) -{ - if (enif_is_map(env, key)) { -#ifdef HAS_ENGINE_SUPPORT - /* Use key stored in engine */ - ENGINE *e; - char *id = NULL; - char *password; - - if (!get_engine_and_key_id(env, key, &id, &e)) - return PKEY_BADARG; - password = get_key_password(env, key); - *pkey = ENGINE_load_private_key(e, id, NULL, password); - if (password) enif_free(password); - enif_free(id); - if (!*pkey) - return PKEY_BADARG; -#else - return PKEY_BADARG; -#endif - } - else if (algorithm == atom_rsa) { - RSA *rsa = RSA_new(); - - if (!get_rsa_private_key(env, key, rsa)) { - RSA_free(rsa); - return PKEY_BADARG; - } - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_RSA(*pkey, rsa)) { - EVP_PKEY_free(*pkey); - RSA_free(rsa); - return PKEY_BADARG; - } - } else if (algorithm == atom_ecdsa) { -#if defined(HAVE_EC) - EC_KEY *ec = NULL; - const ERL_NIF_TERM *tpl_terms; - int tpl_arity; - - if (enif_get_tuple(env, key, &tpl_arity, &tpl_terms) && tpl_arity == 2 - && enif_is_tuple(env, tpl_terms[0]) && enif_is_binary(env, tpl_terms[1]) - && get_ec_key(env, tpl_terms[0], tpl_terms[1], atom_undefined, &ec)) { - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_EC_KEY(*pkey, ec)) { - EVP_PKEY_free(*pkey); - EC_KEY_free(ec); - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } -#else - return PKEY_NOTSUP; -#endif - } else if (algorithm == atom_eddsa) { -#if defined(HAVE_EDDSA) - if (!get_eddsa_key(env, 0, key, pkey)) { - return PKEY_BADARG; - } -#else - return PKEY_NOTSUP; -#endif - } else if (algorithm == atom_dss) { - DSA *dsa = DSA_new(); - - if (!get_dss_private_key(env, key, dsa)) { - DSA_free(dsa); - return PKEY_BADARG; - } - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_DSA(*pkey, dsa)) { - EVP_PKEY_free(*pkey); - DSA_free(dsa); - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } - - return PKEY_OK; -} - - -static int get_pkey_public_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM key, - EVP_PKEY **pkey) -{ - if (enif_is_map(env, key)) { -#ifdef HAS_ENGINE_SUPPORT - /* Use key stored in engine */ - ENGINE *e; - char *id = NULL; - char *password; - - if (!get_engine_and_key_id(env, key, &id, &e)) - return PKEY_BADARG; - password = get_key_password(env, key); - *pkey = ENGINE_load_public_key(e, id, NULL, password); - if (password) enif_free(password); - enif_free(id); - if (!pkey) - return PKEY_BADARG; -#else - return PKEY_BADARG; -#endif - } else if (algorithm == atom_rsa) { - RSA *rsa = RSA_new(); - - if (!get_rsa_public_key(env, key, rsa)) { - RSA_free(rsa); - return PKEY_BADARG; - } - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_RSA(*pkey, rsa)) { - EVP_PKEY_free(*pkey); - RSA_free(rsa); - return PKEY_BADARG; - } - } else if (algorithm == atom_ecdsa) { -#if defined(HAVE_EC) - EC_KEY *ec = NULL; - const ERL_NIF_TERM *tpl_terms; - int tpl_arity; - - if (enif_get_tuple(env, key, &tpl_arity, &tpl_terms) && tpl_arity == 2 - && enif_is_tuple(env, tpl_terms[0]) && enif_is_binary(env, tpl_terms[1]) - && get_ec_key(env, tpl_terms[0], atom_undefined, tpl_terms[1], &ec)) { - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_EC_KEY(*pkey, ec)) { - EVP_PKEY_free(*pkey); - EC_KEY_free(ec); - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } -#else - return PKEY_NOTSUP; -#endif - } else if (algorithm == atom_eddsa) { -#if defined(HAVE_EDDSA) - if (!get_eddsa_key(env, 1, key, pkey)) { - return PKEY_BADARG; - } -#else - return PKEY_NOTSUP; -#endif - } else if (algorithm == atom_dss) { - DSA *dsa = DSA_new(); - - if (!get_dss_public_key(env, key, dsa)) { - DSA_free(dsa); - return PKEY_BADARG; - } - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_DSA(*pkey, dsa)) { - EVP_PKEY_free(*pkey); - DSA_free(dsa); - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } - - return PKEY_OK; -} - -static ERL_NIF_TERM pkey_sign_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{/* (Algorithm, Type, Data|{digest,Digest}, Key|#{}, Options) */ - int i; - const EVP_MD *md = NULL; - unsigned char md_value[EVP_MAX_MD_SIZE]; - EVP_PKEY *pkey; -#ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX *ctx; - size_t siglen; -#else - unsigned len, siglen; -#endif - PKeySignOptions sig_opt; - ErlNifBinary sig_bin; /* signature */ - unsigned char *tbs; /* data to be signed */ - size_t tbslen; -/*char buf[1024]; -enif_get_atom(env,argv[0],buf,1024,ERL_NIF_LATIN1); printf("algo=%s ",buf); -enif_get_atom(env,argv[1],buf,1024,ERL_NIF_LATIN1); printf("hash=%s ",buf); -printf("\r\n"); -*/ - -#ifndef HAS_ENGINE_SUPPORT - if (enif_is_map(env, argv[3])) { - return atom_notsup; - } -#endif - - i = get_pkey_sign_digest(env, argv[0], argv[1], argv[2], md_value, &md, &tbs, &tbslen); - if (i != PKEY_OK) { - if (i == PKEY_NOTSUP) - return atom_notsup; - else - return enif_make_badarg(env); - } - - i = get_pkey_sign_options(env, argv[0], argv[4], md, &sig_opt); - if (i != PKEY_OK) { - if (i == PKEY_NOTSUP) - return atom_notsup; - else - return enif_make_badarg(env); - } - - if (get_pkey_private_key(env, argv[0], argv[3], &pkey) != PKEY_OK) { - return enif_make_badarg(env); - } - -#ifdef HAS_EVP_PKEY_CTX - ctx = EVP_PKEY_CTX_new(pkey, NULL); - if (!ctx) goto badarg; - - if (argv[0] != atom_eddsa) { - if (EVP_PKEY_sign_init(ctx) <= 0) goto badarg; - if (md != NULL && EVP_PKEY_CTX_set_signature_md(ctx, md) <= 0) goto badarg; - } - - if (argv[0] == atom_rsa) { - if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) <= 0) goto badarg; -# ifdef HAVE_RSA_PKCS1_PSS_PADDING - if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) { - if (sig_opt.rsa_mgf1_md != NULL) { -# ifdef HAVE_RSA_MGF1_MD - if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) <= 0) goto badarg; -# else - EVP_PKEY_CTX_free(ctx); - EVP_PKEY_free(pkey); - return atom_notsup; -# endif - } - if (sig_opt.rsa_pss_saltlen > -2 - && EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) <= 0) - goto badarg; - } -#endif - } - - if (argv[0] == atom_eddsa) { -#ifdef HAVE_EDDSA - EVP_MD_CTX* mdctx = EVP_MD_CTX_new(); - if (!EVP_DigestSignInit(mdctx, NULL, NULL, NULL, pkey)) { - if (mdctx) EVP_MD_CTX_free(mdctx); - goto badarg; - } - - if (!EVP_DigestSign(mdctx, NULL, &siglen, tbs, tbslen)) { - EVP_MD_CTX_free(mdctx); - goto badarg; - } - enif_alloc_binary(siglen, &sig_bin); - - if (!EVP_DigestSign(mdctx, sig_bin.data, &siglen, tbs, tbslen)) { - EVP_MD_CTX_free(mdctx); - goto badarg; - } - EVP_MD_CTX_free(mdctx); -#else - goto badarg; -#endif - } - else - { - if (EVP_PKEY_sign(ctx, NULL, &siglen, tbs, tbslen) <= 0) goto badarg; - enif_alloc_binary(siglen, &sig_bin); - - if (md != NULL) { - ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); - } - i = EVP_PKEY_sign(ctx, sig_bin.data, &siglen, tbs, tbslen); - } - - EVP_PKEY_CTX_free(ctx); -#else -/*printf("Old interface\r\n"); - */ - if (argv[0] == atom_rsa) { - RSA *rsa = EVP_PKEY_get1_RSA(pkey); - enif_alloc_binary(RSA_size(rsa), &sig_bin); - len = EVP_MD_size(md); - ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); - i = RSA_sign(md->type, tbs, len, sig_bin.data, &siglen, rsa); - RSA_free(rsa); - } else if (argv[0] == atom_dss) { - DSA *dsa = EVP_PKEY_get1_DSA(pkey); - enif_alloc_binary(DSA_size(dsa), &sig_bin); - len = EVP_MD_size(md); - ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); - i = DSA_sign(md->type, tbs, len, sig_bin.data, &siglen, dsa); - DSA_free(dsa); - } else if (argv[0] == atom_ecdsa) { -#if defined(HAVE_EC) - EC_KEY *ec = EVP_PKEY_get1_EC_KEY(pkey); - enif_alloc_binary(ECDSA_size(ec), &sig_bin); - len = EVP_MD_size(md); - ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); - i = ECDSA_sign(md->type, tbs, len, sig_bin.data, &siglen, ec); - EC_KEY_free(ec); -#else - EVP_PKEY_free(pkey); - return atom_notsup; -#endif - } else { - goto badarg; - } -#endif - - EVP_PKEY_free(pkey); - if (i == 1) { - ERL_VALGRIND_MAKE_MEM_DEFINED(sig_bin.data, siglen); - if (siglen != sig_bin.size) { - enif_realloc_binary(&sig_bin, siglen); - ERL_VALGRIND_ASSERT_MEM_DEFINED(sig_bin.data, siglen); - } - return enif_make_binary(env, &sig_bin); - } else { - enif_release_binary(&sig_bin); - return atom_error; - } - - badarg: -#ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX_free(ctx); -#endif - EVP_PKEY_free(pkey); - return enif_make_badarg(env); -} - - -static ERL_NIF_TERM pkey_verify_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{/* (Algorithm, Type, Data|{digest,Digest}, Signature, Key, Options) */ - int i; - const EVP_MD *md = NULL; - unsigned char md_value[EVP_MAX_MD_SIZE]; - EVP_PKEY *pkey; -#ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX *ctx; -#else -#endif - PKeySignOptions sig_opt; - ErlNifBinary sig_bin; /* signature */ - unsigned char *tbs; /* data to be signed */ - size_t tbslen; - -#ifndef HAS_ENGINE_SUPPORT - if (enif_is_map(env, argv[4])) { - return atom_notsup; - } -#endif - - if (!enif_inspect_binary(env, argv[3], &sig_bin)) { - return enif_make_badarg(env); - } - - i = get_pkey_sign_digest(env, argv[0], argv[1], argv[2], md_value, &md, &tbs, &tbslen); - if (i != PKEY_OK) { - if (i == PKEY_NOTSUP) - return atom_notsup; - else - return enif_make_badarg(env); - } - - i = get_pkey_sign_options(env, argv[0], argv[5], md, &sig_opt); - if (i != PKEY_OK) { - if (i == PKEY_NOTSUP) - return atom_notsup; - else - return enif_make_badarg(env); - } - - if (get_pkey_public_key(env, argv[0], argv[4], &pkey) != PKEY_OK) { - return enif_make_badarg(env); - } - -#ifdef HAS_EVP_PKEY_CTX -/* printf("EVP interface\r\n"); - */ - ctx = EVP_PKEY_CTX_new(pkey, NULL); - if (!ctx) goto badarg; - - if (argv[0] != atom_eddsa) { - if (EVP_PKEY_verify_init(ctx) <= 0) goto badarg; - if (md != NULL && EVP_PKEY_CTX_set_signature_md(ctx, md) <= 0) goto badarg; - } - - if (argv[0] == atom_rsa) { - if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) <= 0) goto badarg; - if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) { - if (sig_opt.rsa_mgf1_md != NULL) { -# ifdef HAVE_RSA_MGF1_MD - if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) <= 0) goto badarg; -# else - EVP_PKEY_CTX_free(ctx); - EVP_PKEY_free(pkey); - return atom_notsup; -# endif - } - if (sig_opt.rsa_pss_saltlen > -2 - && EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) <= 0) - goto badarg; - } - } - - if (argv[0] == atom_eddsa) { -#ifdef HAVE_EDDSA - EVP_MD_CTX* mdctx = EVP_MD_CTX_create(); - - if (!EVP_DigestVerifyInit(mdctx, NULL, NULL, NULL, pkey)) { - if (mdctx) EVP_MD_CTX_destroy(mdctx); - goto badarg; - } - - i = EVP_DigestVerify(mdctx, sig_bin.data, sig_bin.size, tbs, tbslen); - EVP_MD_CTX_destroy(mdctx); -#else - goto badarg; -#endif - } - else - { - if (md != NULL) { - ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); - } - i = EVP_PKEY_verify(ctx, sig_bin.data, sig_bin.size, tbs, tbslen); - } - - EVP_PKEY_CTX_free(ctx); -#else -/*printf("Old interface\r\n"); -*/ - if (argv[0] == atom_rsa) { - RSA *rsa = EVP_PKEY_get1_RSA(pkey); - i = RSA_verify(md->type, tbs, tbslen, sig_bin.data, sig_bin.size, rsa); - RSA_free(rsa); - } else if (argv[0] == atom_dss) { - DSA *dsa = EVP_PKEY_get1_DSA(pkey); - i = DSA_verify(0, tbs, tbslen, sig_bin.data, sig_bin.size, dsa); - DSA_free(dsa); - } else if (argv[0] == atom_ecdsa) { -#if defined(HAVE_EC) - EC_KEY *ec = EVP_PKEY_get1_EC_KEY(pkey); - i = ECDSA_verify(EVP_MD_type(md), tbs, tbslen, sig_bin.data, sig_bin.size, ec); - EC_KEY_free(ec); -#else - EVP_PKEY_free(pkey); - return atom_notsup; -#endif - } else { - goto badarg; - } -#endif - - EVP_PKEY_free(pkey); - if (i == 1) { - return atom_true; - } else { - return atom_false; - } - - badarg: -#ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX_free(ctx); -#endif - EVP_PKEY_free(pkey); - return enif_make_badarg(env); -} - - -/*--------------------------------*/ - -static int get_pkey_crypt_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM options, - PKeyCryptOptions *opt) -{ - ERL_NIF_TERM head, tail; - const ERL_NIF_TERM *tpl_terms; - int tpl_arity; - const EVP_MD *opt_md; - int i; - - if (!enif_is_list(env, options)) { - return PKEY_BADARG; - } - - /* defaults */ - if (algorithm == atom_rsa) { - opt->rsa_mgf1_md = NULL; - opt->rsa_oaep_label.data = NULL; - opt->rsa_oaep_label.size = 0; - opt->rsa_oaep_md = NULL; - opt->rsa_padding = RSA_PKCS1_PADDING; - opt->signature_md = NULL; - } - - if (enif_is_empty_list(env, options)) { - return PKEY_OK; - } - - if (algorithm == atom_rsa) { - tail = options; - while (enif_get_list_cell(env, tail, &head, &tail)) { - if (enif_get_tuple(env, head, &tpl_arity, &tpl_terms) && tpl_arity == 2) { - if (tpl_terms[0] == atom_rsa_padding - || tpl_terms[0] == atom_rsa_pad /* Compatibility */ - ) { - if (tpl_terms[1] == atom_rsa_pkcs1_padding) { - opt->rsa_padding = RSA_PKCS1_PADDING; -#ifdef HAVE_RSA_OAEP_PADDING - } else if (tpl_terms[1] == atom_rsa_pkcs1_oaep_padding) { - opt->rsa_padding = RSA_PKCS1_OAEP_PADDING; -#endif -#ifdef HAVE_RSA_SSLV23_PADDING - } else if (tpl_terms[1] == atom_rsa_sslv23_padding) { - opt->rsa_padding = RSA_SSLV23_PADDING; -#endif - } else if (tpl_terms[1] == atom_rsa_x931_padding) { - opt->rsa_padding = RSA_X931_PADDING; - } else if (tpl_terms[1] == atom_rsa_no_padding) { - opt->rsa_padding = RSA_NO_PADDING; - } else { - return PKEY_BADARG; - } - } else if (tpl_terms[0] == atom_signature_md && enif_is_atom(env, tpl_terms[1])) { - i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); - if (i != PKEY_OK) { - return i; - } - opt->signature_md = opt_md; - } else if (tpl_terms[0] == atom_rsa_mgf1_md && enif_is_atom(env, tpl_terms[1])) { -#ifndef HAVE_RSA_MGF1_MD - if (tpl_terms[1] != atom_sha) - return PKEY_NOTSUP; -#endif - i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); - if (i != PKEY_OK) { - return i; - } - opt->rsa_mgf1_md = opt_md; - } else if (tpl_terms[0] == atom_rsa_oaep_label - && enif_inspect_binary(env, tpl_terms[1], &(opt->rsa_oaep_label))) { -#ifdef HAVE_RSA_OAEP_MD - continue; -#else - return PKEY_NOTSUP; -#endif - } else if (tpl_terms[0] == atom_rsa_oaep_md && enif_is_atom(env, tpl_terms[1])) { -#ifndef HAVE_RSA_OAEP_MD - if (tpl_terms[1] != atom_sha) - return PKEY_NOTSUP; -#endif - i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); - if (i != PKEY_OK) { - return i; - } - opt->rsa_oaep_md = opt_md; - } else { - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } - } - } else { - return PKEY_BADARG; - } - - return PKEY_OK; -} - -static size_t size_of_RSA(EVP_PKEY *pkey) { - size_t tmplen; - RSA *rsa = EVP_PKEY_get1_RSA(pkey); - if (rsa == NULL) return 0; - tmplen = RSA_size(rsa); - RSA_free(rsa); - return tmplen; -} - -static ERL_NIF_TERM pkey_crypt_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{/* (Algorithm, Data, PublKey=[E,N]|[E,N,D]|[E,N,D,P1,P2,E1,E2,C], Options, IsPrivate, IsEncrypt) */ - int i; - EVP_PKEY *pkey; -#ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX *ctx; -#else - RSA *rsa; -#endif - PKeyCryptOptions crypt_opt; - ErlNifBinary in_bin, out_bin, tmp_bin; - size_t outlen; -#ifdef HAVE_RSA_SSLV23_PADDING - size_t tmplen; -#endif - int is_private = (argv[4] == atom_true), - is_encrypt = (argv[5] == atom_true); - int algo_init = 0; - -/* char algo[1024]; */ - -#ifndef HAS_ENGINE_SUPPORT - if (enif_is_map(env, argv[2])) { - return atom_notsup; - } -#endif - - if (!enif_inspect_binary(env, argv[1], &in_bin)) { - return enif_make_badarg(env); - } - - i = get_pkey_crypt_options(env, argv[0], argv[3], &crypt_opt); - if (i != PKEY_OK) { - if (i == PKEY_NOTSUP) - return atom_notsup; - else - return enif_make_badarg(env); - } - - if (is_private) { - if (get_pkey_private_key(env, argv[0], argv[2], &pkey) != PKEY_OK) { - return enif_make_badarg(env); - } - } else { - if (get_pkey_public_key(env, argv[0], argv[2], &pkey) != PKEY_OK) { - return enif_make_badarg(env); - } - } - - out_bin.data = NULL; - out_bin.size = 0; - tmp_bin.data = NULL; - tmp_bin.size = 0; - -#ifdef HAS_EVP_PKEY_CTX - ctx = EVP_PKEY_CTX_new(pkey, NULL); - if (!ctx) goto badarg; - -/* enif_get_atom(env,argv[0],algo,1024,ERL_NIF_LATIN1); */ - - if (is_private) { - if (is_encrypt) { - /* private encrypt */ - if ((algo_init=EVP_PKEY_sign_init(ctx)) <= 0) { - /* fprintf(stderr,"BADARG %s private encrypt algo_init=%d %s:%d\r\n", algo, algo_init, __FILE__, __LINE__); */ - goto badarg; - } - } else { - /* private decrypt */ - if ((algo_init=EVP_PKEY_decrypt_init(ctx)) <= 0) { - /* fprintf(stderr,"BADARG %s private decrypt algo_init=%d %s:%d\r\n", algo, algo_init, __FILE__, __LINE__); */ - goto badarg; - } - } - } else { - if (is_encrypt) { - /* public encrypt */ - if ((algo_init=EVP_PKEY_encrypt_init(ctx)) <= 0) { - /* fprintf(stderr,"BADARG %s public encrypt algo_init=%d %s:%d\r\n", algo,algo_init,__FILE__, __LINE__); */ - goto badarg; - } - } else { - /* public decrypt */ - if ((algo_init=EVP_PKEY_verify_recover_init(ctx)) <= 0) { - /* fprintf(stderr,"BADARG %s public decrypt algo_init=%d %s:%d\r\n", algo,algo_init,__FILE__, __LINE__); */ - goto badarg; - } - } - } - - if (argv[0] == atom_rsa) { - if (crypt_opt.signature_md != NULL - && EVP_PKEY_CTX_set_signature_md(ctx, crypt_opt.signature_md) <= 0) - goto badarg; -#ifdef HAVE_RSA_SSLV23_PADDING - if (crypt_opt.rsa_padding == RSA_SSLV23_PADDING) { - if (is_encrypt) { - tmplen = size_of_RSA(pkey); - if (tmplen == 0) goto badarg; - if (!enif_alloc_binary(tmplen, &tmp_bin)) goto badarg; - if (RSA_padding_add_SSLv23(tmp_bin.data, tmplen, in_bin.data, in_bin.size) <= 0) - goto badarg; - in_bin = tmp_bin; - } - if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_NO_PADDING) <= 0) goto badarg; - } else -#endif - { - if (EVP_PKEY_CTX_set_rsa_padding(ctx, crypt_opt.rsa_padding) <= 0) goto badarg; - } -#ifdef HAVE_RSA_OAEP_MD - if (crypt_opt.rsa_padding == RSA_PKCS1_OAEP_PADDING) { - if (crypt_opt.rsa_oaep_md != NULL - && EVP_PKEY_CTX_set_rsa_oaep_md(ctx, crypt_opt.rsa_oaep_md) <= 0) - goto badarg; - if (crypt_opt.rsa_mgf1_md != NULL - && EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, crypt_opt.rsa_mgf1_md) <= 0) goto badarg; - if (crypt_opt.rsa_oaep_label.data != NULL && crypt_opt.rsa_oaep_label.size > 0) { - unsigned char *label_copy = NULL; - label_copy = OPENSSL_malloc(crypt_opt.rsa_oaep_label.size); - if (label_copy == NULL) goto badarg; - memcpy((void *)(label_copy), (const void *)(crypt_opt.rsa_oaep_label.data), - crypt_opt.rsa_oaep_label.size); - if (EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, label_copy, - crypt_opt.rsa_oaep_label.size) <= 0) { - OPENSSL_free(label_copy); - label_copy = NULL; - goto badarg; - } - } - } -#endif - } - - if (is_private) { - if (is_encrypt) { - /* private_encrypt */ - i = EVP_PKEY_sign(ctx, NULL, &outlen, in_bin.data, in_bin.size); - } else { - /* private_decrypt */ - i = EVP_PKEY_decrypt(ctx, NULL, &outlen, in_bin.data, in_bin.size); - } - } else { - if (is_encrypt) { - /* public_encrypt */ - i = EVP_PKEY_encrypt(ctx, NULL, &outlen, in_bin.data, in_bin.size); - } else { - /* public_decrypt */ - i = EVP_PKEY_verify_recover(ctx, NULL, &outlen, in_bin.data, in_bin.size); - } - } - /* fprintf(stderr,"i = %d %s:%d\r\n", i, __FILE__, __LINE__); */ - - if (i != 1) goto badarg; - - enif_alloc_binary(outlen, &out_bin); - - if (is_private) { - if (is_encrypt) { - /* private_encrypt */ - i = EVP_PKEY_sign(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); - } else { - /* private_decrypt */ - i = EVP_PKEY_decrypt(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); - } - } else { - if (is_encrypt) { - /* public_encrypt */ - i = EVP_PKEY_encrypt(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); - } else { - /* public_decrypt */ - i = EVP_PKEY_verify_recover(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); - } - } - -#else - /* Non-EVP cryptolib. Only support RSA */ - - if (argv[0] != atom_rsa) { - algo_init = -2; /* exitcode: notsup */ - goto badarg; - } - rsa = EVP_PKEY_get1_RSA(pkey); - enif_alloc_binary(RSA_size(rsa), &out_bin); - - if (is_private) { - if (is_encrypt) { - /* non-evp rsa private encrypt */ - ERL_VALGRIND_ASSERT_MEM_DEFINED(in_bin.data,in_bin.size); - i = RSA_private_encrypt(in_bin.size, in_bin.data, - out_bin.data, rsa, crypt_opt.rsa_padding); - if (i > 0) { - ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, i); - } - } else { - /* non-evp rsa private decrypt */ - i = RSA_private_decrypt(in_bin.size, in_bin.data, - out_bin.data, rsa, crypt_opt.rsa_padding); - if (i > 0) { - ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, i); - enif_realloc_binary(&out_bin, i); - } - } - } else { - if (is_encrypt) { - /* non-evp rsa public encrypt */ - ERL_VALGRIND_ASSERT_MEM_DEFINED(in_bin.data,in_bin.size); - i = RSA_public_encrypt(in_bin.size, in_bin.data, - out_bin.data, rsa, crypt_opt.rsa_padding); - if (i > 0) { - ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, i); - } - } else { - /* non-evp rsa public decrypt */ - i = RSA_public_decrypt(in_bin.size, in_bin.data, - out_bin.data, rsa, crypt_opt.rsa_padding); - if (i > 0) { - ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, i); - enif_realloc_binary(&out_bin, i); - } - } - } - - outlen = i; - RSA_free(rsa); -#endif - - if ((i > 0) && argv[0] == atom_rsa && !is_encrypt) { -#ifdef HAVE_RSA_SSLV23_PADDING - if (crypt_opt.rsa_padding == RSA_SSLV23_PADDING) { - unsigned char *p; - tmplen = size_of_RSA(pkey); - if (tmplen == 0) goto badarg; - if (!enif_alloc_binary(tmplen, &tmp_bin)) - goto badarg; - p = out_bin.data; - p++; - i = RSA_padding_check_SSLv23(tmp_bin.data, tmplen, p, out_bin.size - 1, tmplen); - if (i >= 0) { - outlen = i; - in_bin = out_bin; - out_bin = tmp_bin; - tmp_bin = in_bin; - i = 1; - } - } -#endif - } - - if (tmp_bin.data != NULL) { - enif_release_binary(&tmp_bin); - } - -#ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX_free(ctx); -#else -#endif - EVP_PKEY_free(pkey); - if (i > 0) { - ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, outlen); - if (outlen != out_bin.size) { - enif_realloc_binary(&out_bin, outlen); - ERL_VALGRIND_ASSERT_MEM_DEFINED(out_bin.data, outlen); - } - return enif_make_binary(env, &out_bin); - } else { - enif_release_binary(&out_bin); - return atom_error; - } - - badarg: - if (out_bin.data != NULL) { - enif_release_binary(&out_bin); - } - if (tmp_bin.data != NULL) { - enif_release_binary(&tmp_bin); - } -#ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX_free(ctx); -#else -#endif - EVP_PKEY_free(pkey); - if (algo_init == -2) - return atom_notsup; - else - return enif_make_badarg(env); -} - - - -/*--------------------------------*/ -static ERL_NIF_TERM privkey_to_pubkey_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ /* (Algorithm, PrivKey | KeyMap) */ - EVP_PKEY *pkey; - ERL_NIF_TERM alg = argv[0]; - ERL_NIF_TERM result[8]; - if (get_pkey_private_key(env, alg, argv[1], &pkey) != PKEY_OK) { - return enif_make_badarg(env); - } - - if (alg == atom_rsa) { - const BIGNUM *n = NULL, *e = NULL, *d = NULL; - RSA *rsa = EVP_PKEY_get1_RSA(pkey); - if (rsa) { - RSA_get0_key(rsa, &n, &e, &d); - result[0] = bin_from_bn(env, e); // Exponent E - result[1] = bin_from_bn(env, n); // Modulus N = p*q - RSA_free(rsa); - EVP_PKEY_free(pkey); - return enif_make_list_from_array(env, result, 2); - } - - } else if (argv[0] == atom_dss) { - const BIGNUM *p = NULL, *q = NULL, *g = NULL, *pub_key = NULL; - DSA *dsa = EVP_PKEY_get1_DSA(pkey); - if (dsa) { - DSA_get0_pqg(dsa, &p, &q, &g); - DSA_get0_key(dsa, &pub_key, NULL); - result[0] = bin_from_bn(env, p); - result[1] = bin_from_bn(env, q); - result[2] = bin_from_bn(env, g); - result[3] = bin_from_bn(env, pub_key); - DSA_free(dsa); - EVP_PKEY_free(pkey); - return enif_make_list_from_array(env, result, 4); - } - - } else if (argv[0] == atom_ecdsa) { -#if defined(HAVE_EC) - /* not yet implemented - EC_KEY *ec = EVP_PKEY_get1_EC_KEY(pkey); - if (ec) { - / * Example of result: - { - Curve = {Field, Prime, Point, Order, CoFactor} = - { - Field = {prime_field,<<255,...,255>>}, - Prime = {<<255,...,252>>, - <<90,...,75>>, - <<196,...,144>> - }, - Point = <<4,...,245>>, - Order = <<255,...,81>>, - CoFactor = <<1>> - }, - Key = <<151,...,62>> - } - or - { - Curve = - {characteristic_two_field, - M, - Basis = {tpbasis, _} - | {ppbasis, k1, k2, k3} - }, - Key - } - * / - EVP_PKEY_free(pkey); - return enif_make_list_from_array(env, ..., ...); - */ -#endif - } - - if (pkey) EVP_PKEY_free(pkey); - return enif_make_badarg(env); -} - -/*================================================================*/ - -static ERL_NIF_TERM rand_seed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - ErlNifBinary seed_bin; - - if (!enif_inspect_binary(env, argv[0], &seed_bin)) - return enif_make_badarg(env); - RAND_seed(seed_bin.data,seed_bin.size); - return atom_ok; -} - -/*================================================================*/ -/* Engine */ -/*================================================================*/ -static ERL_NIF_TERM engine_by_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (EngineId) */ -#ifdef HAS_ENGINE_SUPPORT - ERL_NIF_TERM ret; - ErlNifBinary engine_id_bin; - char *engine_id; - ENGINE *engine; - struct engine_ctx *ctx; - - // Get Engine Id - if(!enif_inspect_binary(env, argv[0], &engine_id_bin)) { - PRINTF_ERR0("engine_by_id_nif Leaved: badarg"); - return enif_make_badarg(env); - } else { - engine_id = enif_alloc(engine_id_bin.size+1); - (void) memcpy(engine_id, engine_id_bin.data, engine_id_bin.size); - engine_id[engine_id_bin.size] = '\0'; - } - - engine = ENGINE_by_id(engine_id); - if(!engine) { - enif_free(engine_id); - PRINTF_ERR0("engine_by_id_nif Leaved: {error, bad_engine_id}"); - return enif_make_tuple2(env, atom_error, atom_bad_engine_id); - } - - ctx = enif_alloc_resource(engine_ctx_rtype, sizeof(struct engine_ctx)); - ctx->engine = engine; - ctx->id = engine_id; - - ret = enif_make_resource(env, ctx); - enif_release_resource(ctx); - - return enif_make_tuple2(env, atom_ok, ret); -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine) */ -#ifdef HAS_ENGINE_SUPPORT - ERL_NIF_TERM ret = atom_ok; - struct engine_ctx *ctx; - - // Get Engine - if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { - PRINTF_ERR0("engine_init_nif Leaved: Parameter not an engine resource object"); - return enif_make_badarg(env); - } - if (!ENGINE_init(ctx->engine)) { - //ERR_print_errors_fp(stderr); - PRINTF_ERR0("engine_init_nif Leaved: {error, engine_init_failed}"); - return enif_make_tuple2(env, atom_error, atom_engine_init_failed); - } - - return ret; -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_free_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine) */ -#ifdef HAS_ENGINE_SUPPORT - struct engine_ctx *ctx; - - // Get Engine - if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { - PRINTF_ERR0("engine_free_nif Leaved: Parameter not an engine resource object"); - return enif_make_badarg(env); - } - - ENGINE_free(ctx->engine); - return atom_ok; -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_finish_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine) */ -#ifdef HAS_ENGINE_SUPPORT - struct engine_ctx *ctx; - - // Get Engine - if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { - PRINTF_ERR0("engine_finish_nif Leaved: Parameter not an engine resource object"); - return enif_make_badarg(env); - } - - ENGINE_finish(ctx->engine); - return atom_ok; -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_load_dynamic_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* () */ -#ifdef HAS_ENGINE_SUPPORT - ENGINE_load_dynamic(); - return atom_ok; -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_ctrl_cmd_strings_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine, Commands) */ -#ifdef HAS_ENGINE_SUPPORT - ERL_NIF_TERM ret = atom_ok; - unsigned int cmds_len = 0; - char **cmds = NULL; - struct engine_ctx *ctx; - int i, optional = 0; - - // Get Engine - if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { - PRINTF_ERR0("engine_ctrl_cmd_strings_nif Leaved: Parameter not an engine resource object"); - return enif_make_badarg(env); - } - - PRINTF_ERR1("Engine Id: %s\r\n", ENGINE_get_id(ctx->engine)); - - // Get Command List - if(!enif_get_list_length(env, argv[1], &cmds_len)) { - PRINTF_ERR0("engine_ctrl_cmd_strings_nif Leaved: Bad Command List"); - return enif_make_badarg(env); - } else { - cmds_len *= 2; // Key-Value list from erlang - cmds = enif_alloc((cmds_len+1)*sizeof(char*)); - if(get_engine_load_cmd_list(env, argv[1], cmds, 0)) { - PRINTF_ERR0("engine_ctrl_cmd_strings_nif Leaved: Couldn't read Command List"); - ret = enif_make_badarg(env); - goto error; - } - } - - if(!enif_get_int(env, argv[2], &optional)) { - PRINTF_ERR0("engine_ctrl_cmd_strings_nif Leaved: Parameter optional not an integer"); - return enif_make_badarg(env); - } - - for(i = 0; i < cmds_len; i+=2) { - PRINTF_ERR2("Cmd: %s:%s\r\n", - cmds[i] ? cmds[i] : "(NULL)", - cmds[i+1] ? cmds[i+1] : "(NULL)"); - if(!ENGINE_ctrl_cmd_string(ctx->engine, cmds[i], cmds[i+1], optional)) { - PRINTF_ERR2("Command failed: %s:%s\r\n", - cmds[i] ? cmds[i] : "(NULL)", - cmds[i+1] ? cmds[i+1] : "(NULL)"); - //ENGINE_free(ctx->engine); - ret = enif_make_tuple2(env, atom_error, atom_ctrl_cmd_failed); - PRINTF_ERR0("engine_ctrl_cmd_strings_nif Leaved: {error, ctrl_cmd_failed}"); - goto error; - } - } - - error: - for(i = 0; cmds != NULL && cmds[i] != NULL; i++) - enif_free(cmds[i]); - enif_free(cmds); - return ret; -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_add_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine) */ -#ifdef HAS_ENGINE_SUPPORT - struct engine_ctx *ctx; - - // Get Engine - if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { - PRINTF_ERR0("engine_add_nif Leaved: Parameter not an engine resource object"); - return enif_make_badarg(env); - } - - if (!ENGINE_add(ctx->engine)) { - PRINTF_ERR0("engine_add_nif Leaved: {error, add_engine_failed}"); - return enif_make_tuple2(env, atom_error, atom_add_engine_failed); - } - return atom_ok; -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_remove_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine) */ -#ifdef HAS_ENGINE_SUPPORT - struct engine_ctx *ctx; - - // Get Engine - if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { - PRINTF_ERR0("engine_remove_nif Leaved: Parameter not an engine resource object"); - return enif_make_badarg(env); - } - - if (!ENGINE_remove(ctx->engine)) { - PRINTF_ERR0("engine_remove_nif Leaved: {error, remove_engine_failed}"); - return enif_make_tuple2(env, atom_error, atom_remove_engine_failed); - } - return atom_ok; -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_register_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine, EngineMethod) */ -#ifdef HAS_ENGINE_SUPPORT - struct engine_ctx *ctx; - unsigned int method; - - // Get Engine - if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { - PRINTF_ERR0("engine_register_nif Leaved: Parameter not an engine resource object"); - return enif_make_badarg(env); - } - // Get Method - if (!enif_get_uint(env, argv[1], &method)) { - PRINTF_ERR0("engine_register_nif Leaved: Parameter Method not an uint"); - return enif_make_badarg(env); - } - - switch(method) - { -#ifdef ENGINE_METHOD_RSA - case ENGINE_METHOD_RSA: - if (!ENGINE_register_RSA(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif -#ifdef ENGINE_METHOD_DSA - case ENGINE_METHOD_DSA: - if (!ENGINE_register_DSA(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif -#ifdef ENGINE_METHOD_DH - case ENGINE_METHOD_DH: - if (!ENGINE_register_DH(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif -#ifdef ENGINE_METHOD_RAND - case ENGINE_METHOD_RAND: - if (!ENGINE_register_RAND(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif -#ifdef ENGINE_METHOD_ECDH - case ENGINE_METHOD_ECDH: - if (!ENGINE_register_ECDH(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif -#ifdef ENGINE_METHOD_ECDSA - case ENGINE_METHOD_ECDSA: - if (!ENGINE_register_ECDSA(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif -#ifdef ENGINE_METHOD_STORE - case ENGINE_METHOD_STORE: - if (!ENGINE_register_STORE(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif -#ifdef ENGINE_METHOD_CIPHERS - case ENGINE_METHOD_CIPHERS: - if (!ENGINE_register_ciphers(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif -#ifdef ENGINE_METHOD_DIGESTS - case ENGINE_METHOD_DIGESTS: - if (!ENGINE_register_digests(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif -#ifdef ENGINE_METHOD_PKEY_METHS - case ENGINE_METHOD_PKEY_METHS: - if (!ENGINE_register_pkey_meths(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif -#ifdef ENGINE_METHOD_PKEY_ASN1_METHS - case ENGINE_METHOD_PKEY_ASN1_METHS: - if (!ENGINE_register_pkey_asn1_meths(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif -#ifdef ENGINE_METHOD_EC - case ENGINE_METHOD_EC: - if (!ENGINE_register_EC(ctx->engine)) - return enif_make_tuple2(env, atom_error, atom_register_engine_failed); - break; -#endif - default: - return enif_make_tuple2(env, atom_error, atom_engine_method_not_supported); - break; - } - return atom_ok; -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_unregister_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine, EngineMethod) */ -#ifdef HAS_ENGINE_SUPPORT - struct engine_ctx *ctx; - unsigned int method; - - // Get Engine - if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { - PRINTF_ERR0("engine_unregister_nif Leaved: Parameter not an engine resource object"); - return enif_make_badarg(env); - } - // Get Method - if (!enif_get_uint(env, argv[1], &method)) { - PRINTF_ERR0("engine_unregister_nif Leaved: Parameter Method not an uint"); - return enif_make_badarg(env); - } - - switch(method) - { -#ifdef ENGINE_METHOD_RSA - case ENGINE_METHOD_RSA: - ENGINE_unregister_RSA(ctx->engine); - break; -#endif -#ifdef ENGINE_METHOD_DSA - case ENGINE_METHOD_DSA: - ENGINE_unregister_DSA(ctx->engine); - break; -#endif -#ifdef ENGINE_METHOD_DH - case ENGINE_METHOD_DH: - ENGINE_unregister_DH(ctx->engine); - break; -#endif -#ifdef ENGINE_METHOD_RAND - case ENGINE_METHOD_RAND: - ENGINE_unregister_RAND(ctx->engine); - break; -#endif -#ifdef ENGINE_METHOD_ECDH - case ENGINE_METHOD_ECDH: - ENGINE_unregister_ECDH(ctx->engine); - break; -#endif -#ifdef ENGINE_METHOD_ECDSA - case ENGINE_METHOD_ECDSA: - ENGINE_unregister_ECDSA(ctx->engine); - break; -#endif -#ifdef ENGINE_METHOD_STORE - case ENGINE_METHOD_STORE: - ENGINE_unregister_STORE(ctx->engine); - break; -#endif -#ifdef ENGINE_METHOD_CIPHERS - case ENGINE_METHOD_CIPHERS: - ENGINE_unregister_ciphers(ctx->engine); - break; -#endif -#ifdef ENGINE_METHOD_DIGESTS - case ENGINE_METHOD_DIGESTS: - ENGINE_unregister_digests(ctx->engine); - break; -#endif -#ifdef ENGINE_METHOD_PKEY_METHS - case ENGINE_METHOD_PKEY_METHS: - ENGINE_unregister_pkey_meths(ctx->engine); - break; -#endif -#ifdef ENGINE_METHOD_PKEY_ASN1_METHS - case ENGINE_METHOD_PKEY_ASN1_METHS: - ENGINE_unregister_pkey_asn1_meths(ctx->engine); - break; -#endif -#ifdef ENGINE_METHOD_EC - case ENGINE_METHOD_EC: - ENGINE_unregister_EC(ctx->engine); - break; -#endif - default: - break; - } - return atom_ok; -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_get_first_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine) */ -#ifdef HAS_ENGINE_SUPPORT - ERL_NIF_TERM ret; - ENGINE *engine; - ErlNifBinary engine_bin; - struct engine_ctx *ctx; - - engine = ENGINE_get_first(); - if(!engine) { - enif_alloc_binary(0, &engine_bin); - engine_bin.size = 0; - return enif_make_tuple2(env, atom_ok, enif_make_binary(env, &engine_bin)); - } - - ctx = enif_alloc_resource(engine_ctx_rtype, sizeof(struct engine_ctx)); - ctx->engine = engine; - ctx->id = NULL; - - ret = enif_make_resource(env, ctx); - enif_release_resource(ctx); - - return enif_make_tuple2(env, atom_ok, ret); -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_get_next_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine) */ -#ifdef HAS_ENGINE_SUPPORT - ERL_NIF_TERM ret; - ENGINE *engine; - ErlNifBinary engine_bin; - struct engine_ctx *ctx, *next_ctx; - - // Get Engine - if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { - PRINTF_ERR0("engine_get_next_nif Leaved: Parameter not an engine resource object"); - return enif_make_badarg(env); - } - engine = ENGINE_get_next(ctx->engine); - if (!engine) { - enif_alloc_binary(0, &engine_bin); - engine_bin.size = 0; - return enif_make_tuple2(env, atom_ok, enif_make_binary(env, &engine_bin)); - } - - next_ctx = enif_alloc_resource(engine_ctx_rtype, sizeof(struct engine_ctx)); - next_ctx->engine = engine; - next_ctx->id = NULL; - - ret = enif_make_resource(env, next_ctx); - enif_release_resource(next_ctx); - - return enif_make_tuple2(env, atom_ok, ret); -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_get_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine) */ -#ifdef HAS_ENGINE_SUPPORT - ErlNifBinary engine_id_bin; - const char *engine_id; - int size; - struct engine_ctx *ctx; - - // Get Engine - if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { - PRINTF_ERR0("engine_get_id_nif Leaved: Parameter not an engine resource object"); - return enif_make_badarg(env); - } - - engine_id = ENGINE_get_id(ctx->engine); - if (!engine_id) { - enif_alloc_binary(0, &engine_id_bin); - engine_id_bin.size = 0; - return enif_make_binary(env, &engine_id_bin); - } - - size = strlen(engine_id); - enif_alloc_binary(size, &engine_id_bin); - engine_id_bin.size = size; - memcpy(engine_id_bin.data, engine_id, size); - - return enif_make_binary(env, &engine_id_bin); -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM engine_get_name_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Engine) */ -#ifdef HAS_ENGINE_SUPPORT - ErlNifBinary engine_name_bin; - const char *engine_name; - int size; - struct engine_ctx *ctx; - - // Get Engine - if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) { - PRINTF_ERR0("engine_get_id_nif Leaved: Parameter not an engine resource object"); - return enif_make_badarg(env); - } - - engine_name = ENGINE_get_name(ctx->engine); - if (!engine_name) { - enif_alloc_binary(0, &engine_name_bin); - engine_name_bin.size = 0; - return enif_make_binary(env, &engine_name_bin); - } - - size = strlen(engine_name); - enif_alloc_binary(size, &engine_name_bin); - engine_name_bin.size = size; - memcpy(engine_name_bin.data, engine_name, size); - - return enif_make_binary(env, &engine_name_bin); -#else - return atom_notsup; -#endif -} - -#ifdef HAS_ENGINE_SUPPORT -static int get_engine_load_cmd_list(ErlNifEnv* env, const ERL_NIF_TERM term, char **cmds, int i) -{ - ERL_NIF_TERM head, tail; - const ERL_NIF_TERM *tmp_tuple; - ErlNifBinary tmpbin; - int arity; - char* tmpstr; - - if(!enif_is_empty_list(env, term)) { - if(!enif_get_list_cell(env, term, &head, &tail)) { - cmds[i] = NULL; - return -1; - } else { - if(!enif_get_tuple(env, head, &arity, &tmp_tuple) || arity != 2) { - cmds[i] = NULL; - return -1; - } else { - if(!enif_inspect_binary(env, tmp_tuple[0], &tmpbin)) { - cmds[i] = NULL; - return -1; - } else { - tmpstr = enif_alloc(tmpbin.size+1); - (void) memcpy(tmpstr, tmpbin.data, tmpbin.size); - tmpstr[tmpbin.size] = '\0'; - cmds[i++] = tmpstr; - } - if(!enif_inspect_binary(env, tmp_tuple[1], &tmpbin)) { - cmds[i] = NULL; - return -1; - } else { - if(tmpbin.size == 0) - cmds[i++] = NULL; - else { - tmpstr = enif_alloc(tmpbin.size+1); - (void) memcpy(tmpstr, tmpbin.data, tmpbin.size); - tmpstr[tmpbin.size] = '\0'; - cmds[i++] = tmpstr; - } - } - return get_engine_load_cmd_list(env, tail, cmds, i); - } - } - } else { - cmds[i] = NULL; - return 0; - } -} -#endif - -static ERL_NIF_TERM engine_get_all_methods_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* () */ -#ifdef HAS_ENGINE_SUPPORT - ERL_NIF_TERM method_array[12]; - int i = 0; - -#ifdef ENGINE_METHOD_RSA - method_array[i++] = atom_engine_method_rsa; -#endif -#ifdef ENGINE_METHOD_DSA - method_array[i++] = atom_engine_method_dsa; -#endif -#ifdef ENGINE_METHOD_DH - method_array[i++] = atom_engine_method_dh; -#endif -#ifdef ENGINE_METHOD_RAND - method_array[i++] = atom_engine_method_rand; -#endif -#ifdef ENGINE_METHOD_ECDH - method_array[i++] = atom_engine_method_ecdh; -#endif -#ifdef ENGINE_METHOD_ECDSA - method_array[i++] = atom_engine_method_ecdsa; -#endif -#ifdef ENGINE_METHOD_STORE - method_array[i++] = atom_engine_method_store; -#endif -#ifdef ENGINE_METHOD_CIPHERS - method_array[i++] = atom_engine_method_ciphers; -#endif -#ifdef ENGINE_METHOD_DIGESTS - method_array[i++] = atom_engine_method_digests; -#endif -#ifdef ENGINE_METHOD_PKEY_METHS - method_array[i++] = atom_engine_method_pkey_meths; -#endif -#ifdef ENGINE_METHOD_PKEY_ASN1_METHS - method_array[i++] = atom_engine_method_pkey_asn1_meths; -#endif -#ifdef ENGINE_METHOD_EC - method_array[i++] = atom_engine_method_ec; -#endif - - return enif_make_list_from_array(env, method_array, i); -#else - return atom_notsup; -#endif -} diff --git a/lib/crypto/c_src/crypto_callback.c b/lib/crypto/c_src/crypto_callback.c index 0cc7dd609d..0141ccd840 100644 --- a/lib/crypto/c_src/crypto_callback.c +++ b/lib/crypto/c_src/crypto_callback.c @@ -21,6 +21,7 @@ #include <stdio.h> #include <string.h> #include <openssl/opensslconf.h> +#include <stdint.h> #include <erl_nif.h> #include "crypto_callback.h" @@ -64,22 +65,36 @@ static void nomem(size_t size, const char* op) static void* crypto_alloc(size_t size CCB_FILE_LINE_ARGS) { - void *ret = enif_alloc(size); + void *ret; - if (!ret && size) - nomem(size, "allocate"); + if ((ret = enif_alloc(size)) == NULL) + goto err; return ret; + + err: + if (size) + nomem(size, "allocate"); + return NULL; } static void* crypto_realloc(void* ptr, size_t size CCB_FILE_LINE_ARGS) { - void* ret = enif_realloc(ptr, size); + void* ret; - if (!ret && size) - nomem(size, "reallocate"); + if ((ret = enif_realloc(ptr, size)) == NULL) + goto err; return ret; + + err: + if (size) + nomem(size, "reallocate"); + return NULL; } + static void crypto_free(void* ptr CCB_FILE_LINE_ARGS) { + if (ptr == NULL) + return; + enif_free(ptr); } @@ -160,19 +175,26 @@ DLLEXPORT struct crypto_callbacks* get_crypto_callbacks(int nlocks) #ifdef OPENSSL_THREADS if (nlocks > 0) { int i; - lock_vec = enif_alloc(nlocks*sizeof(*lock_vec)); - if (lock_vec==NULL) return NULL; - memset(lock_vec, 0, nlocks*sizeof(*lock_vec)); - + + if ((size_t)nlocks > SIZE_MAX / sizeof(*lock_vec)) + goto err; + if ((lock_vec = enif_alloc((size_t)nlocks * sizeof(*lock_vec))) == NULL) + goto err; + + memset(lock_vec, 0, (size_t)nlocks * sizeof(*lock_vec)); + for (i=nlocks-1; i>=0; --i) { - lock_vec[i] = enif_rwlock_create("crypto_stat"); - if (lock_vec[i]==NULL) return NULL; + if ((lock_vec[i] = enif_rwlock_create("crypto_stat")) == NULL) + goto err; } } #endif is_initialized = 1; } return &the_struct; + + err: + return NULL; } #ifdef HAVE_DYNAMIC_CRYPTO_LIB diff --git a/lib/crypto/c_src/dh.c b/lib/crypto/c_src/dh.c new file mode 100644 index 0000000000..13a2336f25 --- /dev/null +++ b/lib/crypto/c_src/dh.c @@ -0,0 +1,302 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "dh.h" +#include "bn.h" + +ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (PrivKey|undefined, DHParams=[P,G], Mpint, Len|0) */ +#ifdef HAVE_DH + DH *dh_params = NULL; + unsigned int mpint; /* 0 or 4 */ + ERL_NIF_TERM head, tail; + BIGNUM *dh_p = NULL; + BIGNUM *dh_p_shared; + BIGNUM *dh_g = NULL; + BIGNUM *priv_key_in = NULL; + unsigned long len = 0; + unsigned char *pub_ptr, *prv_ptr; + int pub_len, prv_len; + ERL_NIF_TERM ret_pub, ret_prv, ret; + const BIGNUM *pub_key_gen, *priv_key_gen; +#ifdef HAS_EVP_PKEY_CTX + EVP_PKEY_CTX *ctx = NULL; + EVP_PKEY *dhkey = NULL, *params = NULL; +#endif + + ASSERT(argc == 4); + + if (argv[0] != atom_undefined) { + if (!get_bn_from_bin(env, argv[0], &priv_key_in)) + goto bad_arg; + } + if (!enif_get_list_cell(env, argv[1], &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &dh_p)) + goto bad_arg; + + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &dh_g)) + goto bad_arg; + + if (!enif_is_empty_list(env, tail)) + goto bad_arg; + + if (!enif_get_uint(env, argv[2], &mpint)) + goto bad_arg; + if (mpint != 0 && mpint != 4) + goto bad_arg; + + if (!enif_get_ulong(env, argv[3], &len)) + goto bad_arg; + if (len > LONG_MAX) + goto bad_arg; + + /* Load dh_params with values to use by the generator. + Mem mgmnt transfered from dh_p etc to dh_params */ + if ((dh_params = DH_new()) == NULL) + goto bad_arg; + if (priv_key_in) { + if (!DH_set0_key(dh_params, NULL, priv_key_in)) + goto bad_arg; + /* On success, dh_params owns priv_key_in */ + priv_key_in = NULL; + } + if (!DH_set0_pqg(dh_params, dh_p, NULL, dh_g)) + goto bad_arg; + dh_p_shared = dh_p; /* Don't free this because dh_params owns it */ + /* On success, dh_params owns dh_p and dh_g */ + dh_p = NULL; + dh_g = NULL; + + if (len) { + int bn_len; + + if ((bn_len = BN_num_bits(dh_p_shared)) < 0) + goto bad_arg; + dh_p_shared = NULL; /* dh_params owns the reference */ + if (len >= (size_t)bn_len) + goto bad_arg; + + if (!DH_set_length(dh_params, (long)len)) + goto bad_arg; + } + +#ifdef HAS_EVP_PKEY_CTX + if ((params = EVP_PKEY_new()) == NULL) + goto err; + + /* set the key referenced by params to dh_params... */ + if (EVP_PKEY_set1_DH(params, dh_params) != 1) + goto err; + + if ((ctx = EVP_PKEY_CTX_new(params, NULL)) == NULL) + goto err; + + if (EVP_PKEY_keygen_init(ctx) != 1) + goto err; + + if ((dhkey = EVP_PKEY_new()) == NULL) + goto err; + + /* key gen op, key written to ppkey (=last arg) */ + if (EVP_PKEY_keygen(ctx, &dhkey) != 1) + goto err; + + DH_free(dh_params); + if ((dh_params = EVP_PKEY_get1_DH(dhkey)) == NULL) + goto err; + +#else + if (!DH_generate_key(dh_params)) + goto err; +#endif + + DH_get0_key(dh_params, &pub_key_gen, &priv_key_gen); + + if ((pub_len = BN_num_bytes(pub_key_gen)) < 0) + goto err; + if ((prv_len = BN_num_bytes(priv_key_gen)) < 0) + goto err; + + if ((pub_ptr = enif_make_new_binary(env, (size_t)pub_len+mpint, &ret_pub)) == NULL) + goto err; + if ((prv_ptr = enif_make_new_binary(env, (size_t)prv_len+mpint, &ret_prv)) == NULL) + goto err; + + if (mpint) { + put_uint32(pub_ptr, (unsigned int)pub_len); + pub_ptr += 4; + + put_uint32(prv_ptr, (unsigned int)prv_len); + prv_ptr += 4; + } + + if (BN_bn2bin(pub_key_gen, pub_ptr) < 0) + goto err; + if (BN_bn2bin(priv_key_gen, prv_ptr) < 0) + goto err; + + ERL_VALGRIND_MAKE_MEM_DEFINED(pub_ptr, pub_len); + ERL_VALGRIND_MAKE_MEM_DEFINED(prv_ptr, prv_len); + + ret = enif_make_tuple2(env, ret_pub, ret_prv); + goto done; + + bad_arg: + ret = enif_make_badarg(env); + goto done; + + err: + ret = atom_error; + + done: + if (priv_key_in) + BN_free(priv_key_in); + if (dh_p) + BN_free(dh_p); + if (dh_g) + BN_free(dh_g); + if (dh_params) + DH_free(dh_params); + +#ifdef HAS_EVP_PKEY_CTX + if (ctx) + EVP_PKEY_CTX_free(ctx); + if (dhkey) + EVP_PKEY_free(dhkey); + if (params) + EVP_PKEY_free(params); +#endif + + return ret; +#else + return enif_raise_exception(env, atom_notsup); +#endif +} + +ERL_NIF_TERM dh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (OthersPublicKey, MyPrivateKey, DHParams=[P,G]) */ +#ifdef HAVE_DH + BIGNUM *other_pub_key = NULL; + BIGNUM *dh_p = NULL; + BIGNUM *dh_g = NULL; + BIGNUM *dummy_pub_key = NULL; + BIGNUM *priv_key = NULL; + DH *dh_priv = NULL; + ERL_NIF_TERM head, tail, ret; + ErlNifBinary ret_bin; + int size; + int ret_bin_alloc = 0; + int dh_size; + + /* Check the arguments and get + my private key (dh_priv), + the peer's public key (other_pub_key), + the parameters p & q + */ + ASSERT(argc == 3); + + if (!get_bn_from_bin(env, argv[0], &other_pub_key)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[1], &priv_key)) + goto bad_arg; + + if (!enif_get_list_cell(env, argv[2], &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &dh_p)) + goto bad_arg; + + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &dh_g)) + goto bad_arg; + + if (!enif_is_empty_list(env, tail)) + goto bad_arg; + + /* Note: DH_set0_key() does not allow setting only the + * private key, although DH_compute_key() does not use the + * public key. Work around this limitation by setting + * the public key to a copy of the private key. + */ + if ((dummy_pub_key = BN_dup(priv_key)) == NULL) + goto err; + if ((dh_priv = DH_new()) == NULL) + goto err; + + if (!DH_set0_key(dh_priv, dummy_pub_key, priv_key)) + goto err; + /* dh_priv owns dummy_pub_key and priv_key now */ + dummy_pub_key = NULL; + priv_key = NULL; + + if (!DH_set0_pqg(dh_priv, dh_p, NULL, dh_g)) + goto err; + /* dh_priv owns dh_p and dh_g now */ + dh_p = NULL; + dh_g = NULL; + + if ((dh_size = DH_size(dh_priv)) < 0) + goto err; + if (!enif_alloc_binary((size_t)dh_size, &ret_bin)) + goto err; + ret_bin_alloc = 1; + + if ((size = DH_compute_key(ret_bin.data, other_pub_key, dh_priv)) < 0) + goto err; + if (size == 0) + goto err; + + if ((size_t)size != ret_bin.size) { + if (!enif_realloc_binary(&ret_bin, (size_t)size)) + goto err; + } + + ret = enif_make_binary(env, &ret_bin); + ret_bin_alloc = 0; + goto done; + + bad_arg: + err: + if (ret_bin_alloc) + enif_release_binary(&ret_bin); + ret = enif_make_badarg(env); + + done: + if (other_pub_key) + BN_free(other_pub_key); + if (priv_key) + BN_free(priv_key); + if (dh_p) + BN_free(dh_p); + if (dh_g) + BN_free(dh_g); + if (dummy_pub_key) + BN_free(dummy_pub_key); + if (dh_priv) + DH_free(dh_priv); + + return ret; +#else + return enif_raise_exception(env, atom_notsup); +#endif +} diff --git a/lib/crypto/c_src/dh.h b/lib/crypto/c_src/dh.h new file mode 100644 index 0000000000..a996b0ea28 --- /dev/null +++ b/lib/crypto/c_src/dh.h @@ -0,0 +1,29 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_DH_H__ +#define E_DH_H__ 1 + +#include "common.h" + +ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM dh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_DH_H__ */ diff --git a/lib/crypto/c_src/digest.c b/lib/crypto/c_src/digest.c new file mode 100644 index 0000000000..c987a664d5 --- /dev/null +++ b/lib/crypto/c_src/digest.c @@ -0,0 +1,156 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "digest.h" + +static struct digest_type_t digest_types[] = +{ + {{"md4"}, +#ifdef HAVE_MD4 + {&EVP_md4} +#else + {NULL} +#endif + }, + + {{"md5"}, +#ifdef HAVE_MD5 + {&EVP_md5} +#else + {NULL} +#endif + }, + + {{"ripemd160"}, +#ifdef HAVE_RIPEMD160 + {&EVP_ripemd160} +#else + {NULL} +#endif + }, + + {{"sha"}, {&EVP_sha1}}, + + {{"sha224"}, +#ifdef HAVE_SHA224 + {&EVP_sha224} +#else + {NULL} +#endif + }, + + {{"sha256"}, +#ifdef HAVE_SHA256 + {&EVP_sha256} +#else + {NULL} +#endif + }, + + {{"sha384"}, +#ifdef HAVE_SHA384 + {&EVP_sha384} +#else + {NULL} +#endif + }, + + {{"sha512"}, +#ifdef HAVE_SHA512 + {&EVP_sha512} +#else + {NULL} +#endif + }, + + {{"sha3_224"}, +#ifdef HAVE_SHA3_224 + {&EVP_sha3_224} +#else + {NULL} +#endif + }, + + {{"sha3_256"}, +#ifdef HAVE_SHA3_256 + {&EVP_sha3_256} +#else + {NULL} +#endif + }, + + {{"sha3_384"}, +#ifdef HAVE_SHA3_384 + {&EVP_sha3_384} +#else + {NULL} +#endif + }, + + {{"sha3_512"}, +#ifdef HAVE_SHA3_512 + {&EVP_sha3_512} +#else + {NULL} +#endif + }, + + {{"blake2b"}, +#ifdef HAVE_BLAKE2 + {&EVP_blake2b512} +#else + {NULL} +#endif + }, + + {{"blake2s"}, +#ifdef HAVE_BLAKE2 + {&EVP_blake2s256} +#else + {NULL} +#endif + }, + + {{NULL}, {NULL}} +}; + +void init_digest_types(ErlNifEnv* env) +{ + struct digest_type_t* p = digest_types; + + for (p = digest_types; p->type.str; p++) { + p->type.atom = enif_make_atom(env, p->type.str); + if (p->md.funcp) + p->md.p = p->md.funcp(); + } + p->type.atom = atom_false; /* end marker */ +} + +struct digest_type_t* get_digest_type(ERL_NIF_TERM type) +{ + struct digest_type_t* p = NULL; + for (p = digest_types; p->type.atom != atom_false; p++) { + if (type == p->type.atom) { + return p; + } + } + return NULL; +} + diff --git a/lib/crypto/c_src/digest.h b/lib/crypto/c_src/digest.h new file mode 100644 index 0000000000..06852416cf --- /dev/null +++ b/lib/crypto/c_src/digest.h @@ -0,0 +1,40 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_DIGEST_H__ +#define E_DIGEST_H__ 1 + +#include "common.h" + +struct digest_type_t { + union { + const char* str; /* before init, NULL for end-of-table */ + ERL_NIF_TERM atom; /* after init, 'false' for end-of-table */ + }type; + union { + const EVP_MD* (*funcp)(void); /* before init, NULL if notsup */ + const EVP_MD* p; /* after init, NULL if notsup */ + }md; +}; + +void init_digest_types(ErlNifEnv* env); +struct digest_type_t* get_digest_type(ERL_NIF_TERM type); + +#endif /* E_DIGEST_H__ */ diff --git a/lib/crypto/c_src/dss.c b/lib/crypto/c_src/dss.c new file mode 100644 index 0000000000..63268f0f2b --- /dev/null +++ b/lib/crypto/c_src/dss.c @@ -0,0 +1,148 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "dss.h" +#include "bn.h" + +#ifdef HAVE_DSA + +int get_dss_private_key(ErlNifEnv* env, ERL_NIF_TERM key, DSA *dsa) +{ + /* key=[P,Q,G,KEY] */ + ERL_NIF_TERM head, tail; + BIGNUM *dsa_p = NULL, *dsa_q = NULL, *dsa_g = NULL; + BIGNUM *dummy_pub_key = NULL, *priv_key = NULL; + + if (!enif_get_list_cell(env, key, &head, &tail)) + goto err; + if (!get_bn_from_bin(env, head, &dsa_p)) + goto err; + + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto err; + if (!get_bn_from_bin(env, head, &dsa_q)) + goto err; + + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto err; + if (!get_bn_from_bin(env, head, &dsa_g)) + goto err; + + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto err; + if (!get_bn_from_bin(env, head, &priv_key)) + goto err; + + if (!enif_is_empty_list(env, tail)) + goto err; + + /* Note: DSA_set0_key() does not allow setting only the + * private key, although DSA_sign() does not use the + * public key. Work around this limitation by setting + * the public key to a copy of the private key. + */ + if ((dummy_pub_key = BN_dup(priv_key)) == NULL) + goto err; + + if (!DSA_set0_pqg(dsa, dsa_p, dsa_q, dsa_g)) + goto err; + /* dsa takes ownership on success */ + dsa_p = NULL; + dsa_q = NULL; + dsa_g = NULL; + + if (!DSA_set0_key(dsa, dummy_pub_key, priv_key)) + goto err; + /* dsa takes ownership on success */ + dummy_pub_key = NULL; + priv_key = NULL; + + return 1; + + err: + if (dsa_p) + BN_free(dsa_p); + if (dsa_q) + BN_free(dsa_q); + if (dsa_g) + BN_free(dsa_g); + if (priv_key) + BN_free(priv_key); + if (dummy_pub_key) + BN_free(dummy_pub_key); + return 0; +} + +int get_dss_public_key(ErlNifEnv* env, ERL_NIF_TERM key, DSA *dsa) +{ + /* key=[P, Q, G, Y] */ + ERL_NIF_TERM head, tail; + BIGNUM *dsa_p = NULL, *dsa_q = NULL, *dsa_g = NULL, *dsa_y = NULL; + + if (!enif_get_list_cell(env, key, &head, &tail)) + goto err; + if (!get_bn_from_bin(env, head, &dsa_p)) + goto err; + + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto err; + if (!get_bn_from_bin(env, head, &dsa_q)) + goto err; + + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto err; + if (!get_bn_from_bin(env, head, &dsa_g)) + goto err; + + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto err; + if (!get_bn_from_bin(env, head, &dsa_y)) + goto err; + + if (!enif_is_empty_list(env,tail)) + goto err; + + if (!DSA_set0_pqg(dsa, dsa_p, dsa_q, dsa_g)) + goto err; + /* dsa takes ownership on success */ + dsa_p = NULL; + dsa_q = NULL; + dsa_g = NULL; + + if (!DSA_set0_key(dsa, dsa_y, NULL)) + goto err; + /* dsa takes ownership on success */ + dsa_y = NULL; + + return 1; + + err: + if (dsa_p) + BN_free(dsa_p); + if (dsa_q) + BN_free(dsa_q); + if (dsa_g) + BN_free(dsa_g); + if (dsa_y) + BN_free(dsa_y); + return 0; +} + +#endif diff --git a/lib/crypto/c_src/dss.h b/lib/crypto/c_src/dss.h new file mode 100644 index 0000000000..07e28ca7c5 --- /dev/null +++ b/lib/crypto/c_src/dss.h @@ -0,0 +1,31 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_DSS_H__ +#define E_DSS_H__ 1 + +#include "common.h" + +#ifdef HAVE_DSA +int get_dss_private_key(ErlNifEnv* env, ERL_NIF_TERM key, DSA *dsa); +int get_dss_public_key(ErlNifEnv* env, ERL_NIF_TERM key, DSA *dsa); +#endif + +#endif /* E_DSS_H__ */ diff --git a/lib/crypto/c_src/ec.c b/lib/crypto/c_src/ec.c new file mode 100644 index 0000000000..51a3547694 --- /dev/null +++ b/lib/crypto/c_src/ec.c @@ -0,0 +1,414 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "ec.h" +#include "bn.h" + +#ifdef HAVE_EC +static EC_KEY* ec_key_new(ErlNifEnv* env, ERL_NIF_TERM curve_arg); +static ERL_NIF_TERM point2term(ErlNifEnv* env, + const EC_GROUP *group, + const EC_POINT *point, + point_conversion_form_t form); + +ERL_NIF_TERM make_badarg_maybe(ErlNifEnv* env) +{ + ERL_NIF_TERM reason; + if (enif_has_pending_exception(env, &reason)) + return reason; /* dummy return value ignored */ + else + return enif_make_badarg(env); +} + +static EC_KEY* ec_key_new(ErlNifEnv* env, ERL_NIF_TERM curve_arg) +{ + EC_KEY *key = NULL; + int c_arity = -1; + const ERL_NIF_TERM* curve; + ErlNifBinary seed; + BIGNUM *p = NULL; + BIGNUM *a = NULL; + BIGNUM *b = NULL; + BIGNUM *bn_order = NULL; + BIGNUM *cofactor = NULL; + EC_GROUP *group = NULL; + EC_POINT *point = NULL; + int f_arity = -1; + const ERL_NIF_TERM *field; + int p_arity = -1; + const ERL_NIF_TERM *prime; + long field_bits; + + /* {Field, Prime, Point, Order, CoFactor} = Curve */ + if (!enif_get_tuple(env, curve_arg, &c_arity, &curve)) + goto err; + if (c_arity != 5) + goto err; + if (!get_bn_from_bin(env, curve[3], &bn_order)) + goto err; + if (curve[4] != atom_none) { + if (!get_bn_from_bin(env, curve[4], &cofactor)) + goto err; + } + + /* {A, B, Seed} = Prime */ + if (!enif_get_tuple(env, curve[1], &p_arity, &prime)) + goto err; + if (!get_bn_from_bin(env, prime[0], &a)) + goto err; + if (!get_bn_from_bin(env, prime[1], &b)) + goto err; + + if (!enif_get_tuple(env, curve[0], &f_arity, &field)) + goto err; + + if (f_arity == 2 && field[0] == atom_prime_field) { + /* {prime_field, Prime} */ + if (!get_bn_from_bin(env, field[1], &p)) + goto err; + if (BN_is_negative(p)) + goto err; + if (BN_is_zero(p)) + goto err; + + field_bits = BN_num_bits(p); + if (field_bits > OPENSSL_ECC_MAX_FIELD_BITS) + goto err; + + /* create the EC_GROUP structure */ + if ((group = EC_GROUP_new_curve_GFp(p, a, b, NULL)) == NULL) + goto err; + + } else if (f_arity == 3 && field[0] == atom_characteristic_two_field) { +#if defined(OPENSSL_NO_EC2M) + enif_raise_exception(env, atom_notsup); + goto err; +#else + /* {characteristic_two_field, M, Basis} */ + int b_arity = -1; + const ERL_NIF_TERM* basis; + + if ((p = BN_new()) == NULL) + goto err; + if (!enif_get_long(env, field[1], &field_bits)) + goto err; + if (field_bits > OPENSSL_ECC_MAX_FIELD_BITS || field_bits > INT_MAX) + goto err; + + if (enif_get_tuple(env, field[2], &b_arity, &basis)) { + if (b_arity == 2) { + unsigned int k1; + + if (basis[0] != atom_tpbasis) + goto err; + if (!enif_get_uint(env, basis[1], &k1)) + goto err; + + /* {tpbasis, k} = Basis */ + if (field_bits <= k1 || k1 == 0 || k1 > INT_MAX) + goto err; + + /* create the polynomial */ + if (!BN_set_bit(p, (int)field_bits)) + goto err; + if (!BN_set_bit(p, (int)k1)) + goto err; + if (!BN_set_bit(p, 0)) + goto err; + + } else if (b_arity == 4) { + unsigned int k1, k2, k3; + + if (basis[0] != atom_ppbasis) + goto err; + if (!enif_get_uint(env, basis[1], &k1)) + goto err; + if (!enif_get_uint(env, basis[2], &k2)) + goto err; + if (!enif_get_uint(env, basis[3], &k3)) + goto err; + + /* {ppbasis, k1, k2, k3} = Basis */ + if (field_bits <= k3 || k3 <= k2 || k2 <= k1 || k1 == 0 || k3 > INT_MAX || k2 > INT_MAX || k1 > INT_MAX) + goto err; + + /* create the polynomial */ + if (!BN_set_bit(p, (int)field_bits)) + goto err; + if (!BN_set_bit(p, (int)k1)) + goto err; + if (!BN_set_bit(p, (int)k2)) + goto err; + if (!BN_set_bit(p, (int)k3)) + goto err; + if (!BN_set_bit(p, 0)) + goto err; + + } else + goto err; + } else if (field[2] == atom_onbasis) { + /* onbasis = Basis */ + /* no parameters */ + goto err; + + } else + goto err; + + if ((group = EC_GROUP_new_curve_GF2m(p, a, b, NULL)) == NULL) + goto err; +#endif + } else + goto err; + + if (enif_inspect_binary(env, prime[2], &seed)) { + if (!EC_GROUP_set_seed(group, seed.data, seed.size)) + goto err; + } + + if (!term2point(env, curve[2], group, &point)) + goto err; + + if (BN_is_negative(bn_order)) + goto err; + if (BN_is_zero(bn_order)) + goto err; + if (BN_num_bits(bn_order) > (int)field_bits + 1) + goto err; + + if (!EC_GROUP_set_generator(group, point, bn_order, cofactor)) + goto err; + + EC_GROUP_set_asn1_flag(group, 0x0); + + if ((key = EC_KEY_new()) == NULL) + goto err; + + if (!EC_KEY_set_group(key, group)) + goto err; + + goto done; + + err: + if (key) + EC_KEY_free(key); + key = NULL; + + done: + /* some OpenSSL structures are mem-dup'ed into the key, + so we have to free our copies here */ + if (bn_order) + BN_free(bn_order); + if (cofactor) + BN_free(cofactor); + if (a) + BN_free(a); + if (b) + BN_free(b); + if (p) + BN_free(p); + if (group) + EC_GROUP_free(group); + if (point) + EC_POINT_free(point); + + return key; +} + +static ERL_NIF_TERM point2term(ErlNifEnv* env, + const EC_GROUP *group, + const EC_POINT *point, + point_conversion_form_t form) +{ + ERL_NIF_TERM ret; + size_t dlen; + ErlNifBinary bin; + int bin_alloc = 0; + + if ((dlen = EC_POINT_point2oct(group, point, form, NULL, 0, NULL)) == 0) + return atom_undefined; + + if (!enif_alloc_binary(dlen, &bin)) + goto err; + bin_alloc = 1; + + if (!EC_POINT_point2oct(group, point, form, bin.data, bin.size, NULL)) + goto err; + + ERL_VALGRIND_MAKE_MEM_DEFINED(bin.data, bin.size); + + ret = enif_make_binary(env, &bin); + bin_alloc = 0; + goto done; + + err: + if (bin_alloc) + enif_release_binary(&bin); + ret = enif_make_badarg(env); + + done: + return ret; +} + +int term2point(ErlNifEnv* env, ERL_NIF_TERM term, EC_GROUP *group, EC_POINT **pptr) +{ + ErlNifBinary bin; + EC_POINT *point = NULL; + + if (!enif_inspect_binary(env, term, &bin)) + goto err; + + if ((point = EC_POINT_new(group)) == NULL) + goto err; + + /* set the point conversion form */ + EC_GROUP_set_point_conversion_form(group, (point_conversion_form_t)(bin.data[0] & ~0x01)); + + /* extract the ec point */ + if (!EC_POINT_oct2point(group, point, bin.data, bin.size, NULL)) + goto err; + + *pptr = point; + return 1; + + err: + if (point) + EC_POINT_free(point); + return 0; +} + +int get_ec_key(ErlNifEnv* env, + ERL_NIF_TERM curve, ERL_NIF_TERM priv, ERL_NIF_TERM pub, + EC_KEY** res) +{ + EC_KEY *key = NULL; + BIGNUM *priv_key = NULL; + EC_POINT *pub_key = NULL; + EC_GROUP *group = NULL; + + if (priv != atom_undefined) { + if (!get_bn_from_bin(env, priv, &priv_key)) + goto err; + } + if (pub != atom_undefined) { + if (!enif_is_binary(env, pub)) + goto err; + } + + if ((key = ec_key_new(env, curve)) == NULL) + goto err; + + if ((group = EC_GROUP_dup(EC_KEY_get0_group(key))) == NULL) + goto err; + + if (term2point(env, pub, group, &pub_key)) { + if (!EC_KEY_set_public_key(key, pub_key)) + goto err; + } + + if (priv != atom_undefined && !BN_is_zero(priv_key)) { + if (!EC_KEY_set_private_key(key, priv_key)) + goto err; + + /* calculate public key (if necessary) */ + if (EC_KEY_get0_public_key(key) == NULL) { + /* the public key was not included in the SEC1 private + * key => calculate the public key */ + if ((pub_key = EC_POINT_new(group)) == NULL) + goto err; + if (!EC_POINT_copy(pub_key, EC_GROUP_get0_generator(group))) + goto err; + if (!EC_POINT_mul(group, pub_key, priv_key, NULL, NULL, NULL)) + goto err; + if (!EC_KEY_set_public_key(key, pub_key)) + goto err; + } + } + goto done; + + err: + if (key) + EC_KEY_free(key); + key = NULL; + + done: + /* some OpenSSL structures are mem-dup'ed into the key, + so we have to free our copies here */ + if (priv_key) + BN_clear_free(priv_key); + if (group) + EC_GROUP_free(group); + if (pub_key) + EC_POINT_free(pub_key); + + if (key == NULL) + return 0; + + *res = key; + return 1; +} + +#endif /* HAVE_EC */ + +ERL_NIF_TERM ec_key_generate(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ +#if defined(HAVE_EC) + EC_KEY *key = NULL; + const EC_GROUP *group; + const EC_POINT *public_key; + ERL_NIF_TERM priv_key; + ERL_NIF_TERM pub_key; + ERL_NIF_TERM ret; + + if (!get_ec_key(env, argv[0], argv[1], atom_undefined, &key)) + goto bad_arg; + + if (argv[1] == atom_undefined) { + if (!EC_KEY_generate_key(key)) + goto err; + } + + group = EC_KEY_get0_group(key); + public_key = EC_KEY_get0_public_key(key); + + if (group == NULL || public_key == NULL) { + pub_key = atom_undefined; + + } else { + pub_key = point2term(env, group, public_key, + EC_KEY_get_conv_form(key)); + } + + priv_key = bn2term(env, EC_KEY_get0_private_key(key)); + ret = enif_make_tuple2(env, pub_key, priv_key); + goto done; + + err: + bad_arg: + ret = make_badarg_maybe(env); + + done: + if (key) + EC_KEY_free(key); + return ret; + +#else + return atom_notsup; +#endif +} diff --git a/lib/crypto/c_src/ec.h b/lib/crypto/c_src/ec.h new file mode 100644 index 0000000000..b7e1cc5a46 --- /dev/null +++ b/lib/crypto/c_src/ec.h @@ -0,0 +1,35 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_EC_H__ +#define E_EC_H__ 1 + +#include "common.h" + +#if defined(HAVE_EC) +int get_ec_key(ErlNifEnv* env, ERL_NIF_TERM curve, ERL_NIF_TERM priv, ERL_NIF_TERM pub, + EC_KEY** res); +int term2point(ErlNifEnv* env, ERL_NIF_TERM term, EC_GROUP *group, EC_POINT **pptr); +ERL_NIF_TERM make_badarg_maybe(ErlNifEnv* env); +#endif + +ERL_NIF_TERM ec_key_generate(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_EC_H__ */ diff --git a/lib/crypto/c_src/ecdh.c b/lib/crypto/c_src/ecdh.c new file mode 100644 index 0000000000..9e3f460519 --- /dev/null +++ b/lib/crypto/c_src/ecdh.c @@ -0,0 +1,94 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "ecdh.h" +#include "ec.h" + +/* + (_OthersPublicKey, _MyPrivateKey) + (_OthersPublicKey, _MyEC_Point) +*/ +ERL_NIF_TERM ecdh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +/* (OtherPublicKey, Curve, My) */ +{ +#if defined(HAVE_EC) + ERL_NIF_TERM ret; + unsigned char *p; + EC_KEY* key = NULL; + int degree; + size_t field_size; + EC_GROUP *group = NULL; + const BIGNUM *priv_key; + EC_POINT *my_ecpoint = NULL; + EC_KEY *other_ecdh = NULL; + + ASSERT(argc == 3); + + if (!get_ec_key(env, argv[1], argv[2], atom_undefined, &key)) + goto bad_arg; + if ((group = EC_GROUP_dup(EC_KEY_get0_group(key))) == NULL) + goto bad_arg; + priv_key = EC_KEY_get0_private_key(key); + + if (!term2point(env, argv[0], group, &my_ecpoint)) { + goto err; + } + + if ((other_ecdh = EC_KEY_new()) == NULL) + goto err; + if (!EC_KEY_set_group(other_ecdh, group)) + goto err; + if (!EC_KEY_set_private_key(other_ecdh, priv_key)) + goto err; + + if ((degree = EC_GROUP_get_degree(group)) <= 0) + goto err; + + field_size = (size_t)degree; + if ((p = enif_make_new_binary(env, (field_size+7)/8, &ret)) == NULL) + goto err; + if (ECDH_compute_key(p, (field_size+7)/8, my_ecpoint, other_ecdh, NULL) < 1) + goto err; + + goto done; + + bad_arg: + ret = make_badarg_maybe(env); + goto done; + + err: + ret = enif_make_badarg(env); + + done: + if (group) + EC_GROUP_free(group); + if (my_ecpoint) + EC_POINT_free(my_ecpoint); + if (other_ecdh) + EC_KEY_free(other_ecdh); + if (key) + EC_KEY_free(key); + + return ret; + +#else + return atom_notsup; +#endif +} diff --git a/lib/crypto/c_src/ecdh.h b/lib/crypto/c_src/ecdh.h new file mode 100644 index 0000000000..5ed331e676 --- /dev/null +++ b/lib/crypto/c_src/ecdh.h @@ -0,0 +1,28 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_ECDH_H__ +#define E_ECDH_H__ 1 + +#include "common.h" + +ERL_NIF_TERM ecdh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_ECDH_H__ */ diff --git a/lib/crypto/c_src/eddsa.c b/lib/crypto/c_src/eddsa.c new file mode 100644 index 0000000000..0c89f9f6db --- /dev/null +++ b/lib/crypto/c_src/eddsa.c @@ -0,0 +1,63 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "eddsa.h" + +#ifdef HAVE_EDDSA +int get_eddsa_key(ErlNifEnv* env, int public, ERL_NIF_TERM key, EVP_PKEY **pkey) +{ + /* key=[K] */ + EVP_PKEY *result; + ERL_NIF_TERM head, tail, tail2, algo; + ErlNifBinary bin; + int type; + + if (!enif_get_list_cell(env, key, &head, &tail)) + goto err; + if (!enif_inspect_binary(env, head, &bin)) + goto err; + if (!enif_get_list_cell(env, tail, &algo, &tail2)) + goto err; + if (!enif_is_empty_list(env, tail2)) + goto err; + + if (algo == atom_ed25519) { + type = EVP_PKEY_ED25519; + } else if (algo == atom_ed448) { + type = EVP_PKEY_ED448; + } else { + goto err; + } + + if (public) + result = EVP_PKEY_new_raw_public_key(type, NULL, bin.data, bin.size); + else + result = EVP_PKEY_new_raw_private_key(type, NULL, bin.data, bin.size); + + if (result == NULL) + goto err; + + *pkey = result; + return 1; + + err: + return 0; +} +#endif diff --git a/lib/crypto/c_src/eddsa.h b/lib/crypto/c_src/eddsa.h new file mode 100644 index 0000000000..4b30247cab --- /dev/null +++ b/lib/crypto/c_src/eddsa.h @@ -0,0 +1,30 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_EDDSA_H__ +#define E_EDDSA_H__ 1 + +#include "common.h" + +#ifdef HAVE_EDDSA +int get_eddsa_key(ErlNifEnv* env, int public, ERL_NIF_TERM key, EVP_PKEY **pkey); +#endif + +#endif /* E_EDDSA_H__ */ diff --git a/lib/crypto/c_src/engine.c b/lib/crypto/c_src/engine.c new file mode 100644 index 0000000000..ea5d9a588f --- /dev/null +++ b/lib/crypto/c_src/engine.c @@ -0,0 +1,840 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "engine.h" + +#ifdef HAS_ENGINE_SUPPORT +struct engine_ctx { + ENGINE *engine; + char *id; +}; + +#define ERROR_Term(Env, ReasonTerm) enif_make_tuple2((Env), atom_error, (ReasonTerm)) +#define ERROR_Atom(Env, ReasonString) ERROR_Term((Env), enif_make_atom((Env),(ReasonString))) + +static ErlNifResourceType* engine_ctx_rtype; + +static int get_engine_load_cmd_list(ErlNifEnv* env, const ERL_NIF_TERM term, char **cmds, int i); +static int zero_terminate(ErlNifBinary bin, char **buf); + +static void engine_ctx_dtor(ErlNifEnv* env, struct engine_ctx* ctx) { + if (ctx == NULL) + return; + + PRINTF_ERR0("engine_ctx_dtor"); + if(ctx->id) { + PRINTF_ERR1(" non empty ctx->id=%s", ctx->id); + enif_free(ctx->id); + } else + PRINTF_ERR0(" empty ctx->id=NULL"); +} + +int get_engine_and_key_id(ErlNifEnv *env, ERL_NIF_TERM key, char ** id, ENGINE **e) +{ + ERL_NIF_TERM engine_res, key_id_term; + struct engine_ctx *ctx; + ErlNifBinary key_id_bin; + + if (!enif_get_map_value(env, key, atom_engine, &engine_res)) + goto err; + if (!enif_get_resource(env, engine_res, engine_ctx_rtype, (void**)&ctx)) + goto err; + if (!enif_get_map_value(env, key, atom_key_id, &key_id_term)) + goto err; + if (!enif_inspect_binary(env, key_id_term, &key_id_bin)) + goto err; + + *e = ctx->engine; + return zero_terminate(key_id_bin, id); + + err: + return 0; +} + +char *get_key_password(ErlNifEnv *env, ERL_NIF_TERM key) { + ERL_NIF_TERM tmp_term; + ErlNifBinary pwd_bin; + char *pwd = NULL; + + if (!enif_get_map_value(env, key, atom_password, &tmp_term)) + goto err; + if (!enif_inspect_binary(env, tmp_term, &pwd_bin)) + goto err; + if (!zero_terminate(pwd_bin, &pwd)) + goto err; + + return pwd; + + err: + return NULL; +} + +static int zero_terminate(ErlNifBinary bin, char **buf) { + if ((*buf = enif_alloc(bin.size + 1)) == NULL) + goto err; + + memcpy(*buf, bin.data, bin.size); + *(*buf + bin.size) = 0; + + return 1; + + err: + return 0; +} +#endif /* HAS_ENGINE_SUPPORT */ + +int init_engine_ctx(ErlNifEnv *env) { +#ifdef HAS_ENGINE_SUPPORT + engine_ctx_rtype = enif_open_resource_type(env, NULL, "ENGINE_CTX", + (ErlNifResourceDtor*) engine_ctx_dtor, + ERL_NIF_RT_CREATE|ERL_NIF_RT_TAKEOVER, + NULL); + if (engine_ctx_rtype == NULL) { + PRINTF_ERR0("CRYPTO: Could not open resource type 'ENGINE_CTX'"); + return 0; + } +#endif + + return 1; +} + +ERL_NIF_TERM engine_by_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (EngineId) */ +#ifdef HAS_ENGINE_SUPPORT + ERL_NIF_TERM ret, result; + ErlNifBinary engine_id_bin; + char *engine_id = NULL; + ENGINE *engine; + struct engine_ctx *ctx = NULL; + + // Get Engine Id + ASSERT(argc == 1); + + if (!enif_inspect_binary(env, argv[0], &engine_id_bin)) + goto bad_arg; + + if ((engine_id = enif_alloc(engine_id_bin.size+1)) == NULL) + goto err; + (void) memcpy(engine_id, engine_id_bin.data, engine_id_bin.size); + engine_id[engine_id_bin.size] = '\0'; + + if ((engine = ENGINE_by_id(engine_id)) == NULL) { + PRINTF_ERR0("engine_by_id_nif Leaved: {error, bad_engine_id}"); + ret = ERROR_Atom(env, "bad_engine_id"); + goto done; + } + + if ((ctx = enif_alloc_resource(engine_ctx_rtype, sizeof(struct engine_ctx))) == NULL) + goto err; + ctx->engine = engine; + ctx->id = engine_id; + /* ctx now owns engine_id */ + engine_id = NULL; + + result = enif_make_resource(env, ctx); + ret = enif_make_tuple2(env, atom_ok, result); + goto done; + + bad_arg: + err: + ret = enif_make_badarg(env); + + done: + if (engine_id) + enif_free(engine_id); + if (ctx) + enif_release_resource(ctx); + return ret; + +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Engine) */ +#ifdef HAS_ENGINE_SUPPORT + struct engine_ctx *ctx; + + // Get Engine + ASSERT(argc == 1); + + if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) + goto bad_arg; + + if (!ENGINE_init(ctx->engine)) + return ERROR_Atom(env, "engine_init_failed"); + + return atom_ok; + + bad_arg: + return enif_make_badarg(env); + +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_free_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Engine) */ +#ifdef HAS_ENGINE_SUPPORT + struct engine_ctx *ctx; + + // Get Engine + ASSERT(argc == 1); + + if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) + goto bad_arg; + + if (!ENGINE_free(ctx->engine)) + goto err; + return atom_ok; + + bad_arg: + err: + return enif_make_badarg(env); +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_finish_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Engine) */ +#ifdef HAS_ENGINE_SUPPORT + struct engine_ctx *ctx; + + // Get Engine + ASSERT(argc == 1); + + if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) + goto bad_arg; + + if (!ENGINE_finish(ctx->engine)) + goto err; + return atom_ok; + + bad_arg: + err: + return enif_make_badarg(env); + +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_load_dynamic_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* () */ +#ifdef HAS_ENGINE_SUPPORT + ASSERT(argc == 0); + + ENGINE_load_dynamic(); + return atom_ok; +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_ctrl_cmd_strings_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Engine, Commands, Optional) */ +#ifdef HAS_ENGINE_SUPPORT + ERL_NIF_TERM ret; + unsigned int cmds_len = 0; + char **cmds = NULL; + struct engine_ctx *ctx; + unsigned int i; + int optional = 0; + int cmds_loaded = 0; + + // Get Engine + ASSERT(argc == 3); + + if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) + goto bad_arg; + + PRINTF_ERR1("Engine Id: %s\r\n", ENGINE_get_id(ctx->engine)); + // Get Command List + if (!enif_get_list_length(env, argv[1], &cmds_len)) + goto bad_arg; + + if (cmds_len > (UINT_MAX / 2) - 1) + goto err; + cmds_len *= 2; // Key-Value list from erlang + + if ((size_t)cmds_len + 1 > SIZE_MAX / sizeof(char*)) + goto err; + if ((cmds = enif_alloc((cmds_len + 1) * sizeof(char*))) == NULL) + goto err; + if (get_engine_load_cmd_list(env, argv[1], cmds, 0)) + goto err; + cmds_loaded = 1; + if (!enif_get_int(env, argv[2], &optional)) + goto err; + + for(i = 0; i < cmds_len; i+=2) { + PRINTF_ERR2("Cmd: %s:%s\r\n", + cmds[i] ? cmds[i] : "(NULL)", + cmds[i+1] ? cmds[i+1] : "(NULL)"); + if(!ENGINE_ctrl_cmd_string(ctx->engine, cmds[i], cmds[i+1], optional)) { + PRINTF_ERR2("Command failed: %s:%s\r\n", + cmds[i] ? cmds[i] : "(NULL)", + cmds[i+1] ? cmds[i+1] : "(NULL)"); + goto cmd_failed; + } + } + ret = atom_ok; + goto done; + + bad_arg: + err: + ret = enif_make_badarg(env); + goto done; + + cmd_failed: + ret = ERROR_Atom(env, "ctrl_cmd_failed"); + + done: + if (cmds_loaded) { + for (i = 0; cmds != NULL && cmds[i] != NULL; i++) + enif_free(cmds[i]); + } + + if (cmds != NULL) + enif_free(cmds); + + return ret; + +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_add_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Engine) */ +#ifdef HAS_ENGINE_SUPPORT + struct engine_ctx *ctx; + + // Get Engine + ASSERT(argc == 1); + + if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) + goto bad_arg; + + if (!ENGINE_add(ctx->engine)) + goto failed; + + return atom_ok; + + bad_arg: + return enif_make_badarg(env); + + failed: + return ERROR_Atom(env, "add_engine_failed"); + +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_remove_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Engine) */ +#ifdef HAS_ENGINE_SUPPORT + struct engine_ctx *ctx; + + // Get Engine + ASSERT(argc == 1); + + if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) + goto bad_arg; + + if (!ENGINE_remove(ctx->engine)) + goto failed; + + return atom_ok; + + bad_arg: + return enif_make_badarg(env); + + failed: + return ERROR_Atom(env, "remove_engine_failed"); +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_register_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Engine, EngineMethod) */ +#ifdef HAS_ENGINE_SUPPORT + struct engine_ctx *ctx; + unsigned int method; + + // Get Engine + ASSERT(argc == 2); + + if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) + goto bad_arg; + if (!enif_get_uint(env, argv[1], &method)) + goto bad_arg; + + switch(method) + { +#ifdef ENGINE_METHOD_RSA + case ENGINE_METHOD_RSA: + if (!ENGINE_register_RSA(ctx->engine)) + goto failed; + break; +#endif +#ifdef ENGINE_METHOD_DSA + case ENGINE_METHOD_DSA: + if (!ENGINE_register_DSA(ctx->engine)) + goto failed; + break; +#endif +#ifdef ENGINE_METHOD_DH + case ENGINE_METHOD_DH: + if (!ENGINE_register_DH(ctx->engine)) + goto failed; + break; +#endif +#ifdef ENGINE_METHOD_RAND + case ENGINE_METHOD_RAND: + if (!ENGINE_register_RAND(ctx->engine)) + goto failed; + break; +#endif +#ifdef ENGINE_METHOD_ECDH + case ENGINE_METHOD_ECDH: + if (!ENGINE_register_ECDH(ctx->engine)) + goto failed; + break; +#endif +#ifdef ENGINE_METHOD_ECDSA + case ENGINE_METHOD_ECDSA: + if (!ENGINE_register_ECDSA(ctx->engine)) + goto failed; + break; +#endif +#ifdef ENGINE_METHOD_STORE + case ENGINE_METHOD_STORE: + if (!ENGINE_register_STORE(ctx->engine)) + goto failed; + break; +#endif +#ifdef ENGINE_METHOD_CIPHERS + case ENGINE_METHOD_CIPHERS: + if (!ENGINE_register_ciphers(ctx->engine)) + goto failed; + break; +#endif +#ifdef ENGINE_METHOD_DIGESTS + case ENGINE_METHOD_DIGESTS: + if (!ENGINE_register_digests(ctx->engine)) + goto failed; + break; +#endif +#ifdef ENGINE_METHOD_PKEY_METHS + case ENGINE_METHOD_PKEY_METHS: + if (!ENGINE_register_pkey_meths(ctx->engine)) + goto failed; + break; +#endif +#ifdef ENGINE_METHOD_PKEY_ASN1_METHS + case ENGINE_METHOD_PKEY_ASN1_METHS: + if (!ENGINE_register_pkey_asn1_meths(ctx->engine)) + goto failed; + break; +#endif +#ifdef ENGINE_METHOD_EC + case ENGINE_METHOD_EC: + if (!ENGINE_register_EC(ctx->engine)) + goto failed; + break; +#endif + default: + return ERROR_Atom(env, "engine_method_not_supported"); + } + + return atom_ok; + + bad_arg: + return enif_make_badarg(env); + + failed: + return ERROR_Atom(env, "register_engine_failed"); + +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_unregister_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Engine, EngineMethod) */ +#ifdef HAS_ENGINE_SUPPORT + struct engine_ctx *ctx; + unsigned int method; + + // Get Engine + ASSERT(argc == 2); + + if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) + goto bad_arg; + if (!enif_get_uint(env, argv[1], &method)) + goto bad_arg; + + switch(method) + { +#ifdef ENGINE_METHOD_RSA + case ENGINE_METHOD_RSA: + ENGINE_unregister_RSA(ctx->engine); + break; +#endif +#ifdef ENGINE_METHOD_DSA + case ENGINE_METHOD_DSA: + ENGINE_unregister_DSA(ctx->engine); + break; +#endif +#ifdef ENGINE_METHOD_DH + case ENGINE_METHOD_DH: + ENGINE_unregister_DH(ctx->engine); + break; +#endif +#ifdef ENGINE_METHOD_RAND + case ENGINE_METHOD_RAND: + ENGINE_unregister_RAND(ctx->engine); + break; +#endif +#ifdef ENGINE_METHOD_ECDH + case ENGINE_METHOD_ECDH: + ENGINE_unregister_ECDH(ctx->engine); + break; +#endif +#ifdef ENGINE_METHOD_ECDSA + case ENGINE_METHOD_ECDSA: + ENGINE_unregister_ECDSA(ctx->engine); + break; +#endif +#ifdef ENGINE_METHOD_STORE + case ENGINE_METHOD_STORE: + ENGINE_unregister_STORE(ctx->engine); + break; +#endif +#ifdef ENGINE_METHOD_CIPHERS + case ENGINE_METHOD_CIPHERS: + ENGINE_unregister_ciphers(ctx->engine); + break; +#endif +#ifdef ENGINE_METHOD_DIGESTS + case ENGINE_METHOD_DIGESTS: + ENGINE_unregister_digests(ctx->engine); + break; +#endif +#ifdef ENGINE_METHOD_PKEY_METHS + case ENGINE_METHOD_PKEY_METHS: + ENGINE_unregister_pkey_meths(ctx->engine); + break; +#endif +#ifdef ENGINE_METHOD_PKEY_ASN1_METHS + case ENGINE_METHOD_PKEY_ASN1_METHS: + ENGINE_unregister_pkey_asn1_meths(ctx->engine); + break; +#endif +#ifdef ENGINE_METHOD_EC + case ENGINE_METHOD_EC: + ENGINE_unregister_EC(ctx->engine); + break; +#endif + default: + break; + } + + return atom_ok; + + bad_arg: + return enif_make_badarg(env); + +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_get_first_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* () */ +#ifdef HAS_ENGINE_SUPPORT + ERL_NIF_TERM ret, result; + ENGINE *engine; + ErlNifBinary engine_bin; + struct engine_ctx *ctx = NULL; + + ASSERT(argc == 0); + + if ((engine = ENGINE_get_first()) == NULL) { + if (!enif_alloc_binary(0, &engine_bin)) + goto err; + engine_bin.size = 0; + return enif_make_tuple2(env, atom_ok, enif_make_binary(env, &engine_bin)); + } + + if ((ctx = enif_alloc_resource(engine_ctx_rtype, sizeof(struct engine_ctx))) == NULL) + goto err; + ctx->engine = engine; + ctx->id = NULL; + + result = enif_make_resource(env, ctx); + ret = enif_make_tuple2(env, atom_ok, result); + goto done; + + err: + ret = enif_make_badarg(env); + + done: + if (ctx) + enif_release_resource(ctx); + return ret; + +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_get_next_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Engine) */ +#ifdef HAS_ENGINE_SUPPORT + ERL_NIF_TERM ret, result; + ENGINE *engine; + ErlNifBinary engine_bin; + struct engine_ctx *ctx, *next_ctx = NULL; + + // Get Engine + ASSERT(argc == 1); + + if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) + goto bad_arg; + + if ((engine = ENGINE_get_next(ctx->engine)) == NULL) { + if (!enif_alloc_binary(0, &engine_bin)) + goto err; + engine_bin.size = 0; + return enif_make_tuple2(env, atom_ok, enif_make_binary(env, &engine_bin)); + } + + if ((next_ctx = enif_alloc_resource(engine_ctx_rtype, sizeof(struct engine_ctx))) == NULL) + goto err; + next_ctx->engine = engine; + next_ctx->id = NULL; + + result = enif_make_resource(env, next_ctx); + ret = enif_make_tuple2(env, atom_ok, result); + goto done; + + bad_arg: + err: + ret = enif_make_badarg(env); + + done: + if (next_ctx) + enif_release_resource(next_ctx); + return ret; + +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_get_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Engine) */ +#ifdef HAS_ENGINE_SUPPORT + ErlNifBinary engine_id_bin; + const char *engine_id; + size_t size; + struct engine_ctx *ctx = NULL; + + // Get Engine + ASSERT(argc == 1); + + if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) + goto bad_arg; + + if ((engine_id = ENGINE_get_id(ctx->engine)) == NULL) { + if (!enif_alloc_binary(0, &engine_id_bin)) + goto err; + engine_id_bin.size = 0; + return enif_make_binary(env, &engine_id_bin); + } + + size = strlen(engine_id); + if (!enif_alloc_binary(size, &engine_id_bin)) + goto err; + engine_id_bin.size = size; + memcpy(engine_id_bin.data, engine_id, size); + + return enif_make_binary(env, &engine_id_bin); + + bad_arg: + err: + return enif_make_badarg(env); + +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM engine_get_name_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Engine) */ +#ifdef HAS_ENGINE_SUPPORT + ErlNifBinary engine_name_bin; + const char *engine_name; + size_t size; + struct engine_ctx *ctx; + + // Get Engine + ASSERT(argc == 1); + + if (!enif_get_resource(env, argv[0], engine_ctx_rtype, (void**)&ctx)) + goto bad_arg; + + if ((engine_name = ENGINE_get_name(ctx->engine)) == NULL) { + if (!enif_alloc_binary(0, &engine_name_bin)) + goto err; + engine_name_bin.size = 0; + return enif_make_binary(env, &engine_name_bin); + } + + size = strlen(engine_name); + if (!enif_alloc_binary(size, &engine_name_bin)) + goto err; + engine_name_bin.size = size; + memcpy(engine_name_bin.data, engine_name, size); + + return enif_make_binary(env, &engine_name_bin); + + bad_arg: + err: + return enif_make_badarg(env); + +#else + return atom_notsup; +#endif +} + +#ifdef HAS_ENGINE_SUPPORT +static int get_engine_load_cmd_list(ErlNifEnv* env, const ERL_NIF_TERM term, char **cmds, int i) +{ + ERL_NIF_TERM head, tail; + const ERL_NIF_TERM *tmp_tuple; + ErlNifBinary tmpbin; + int arity; + char *tuple1 = NULL, *tuple2 = NULL; + + if (enif_is_empty_list(env, term)) { + cmds[i] = NULL; + return 0; + } + + if (!enif_get_list_cell(env, term, &head, &tail)) + goto err; + if (!enif_get_tuple(env, head, &arity, &tmp_tuple)) + goto err; + if (arity != 2) + goto err; + if (!enif_inspect_binary(env, tmp_tuple[0], &tmpbin)) + goto err; + + if ((tuple1 = enif_alloc(tmpbin.size + 1)) == NULL) + goto err; + + (void) memcpy(tuple1, tmpbin.data, tmpbin.size); + tuple1[tmpbin.size] = '\0'; + cmds[i] = tuple1; + i++; + + if (!enif_inspect_binary(env, tmp_tuple[1], &tmpbin)) + goto err; + + if (tmpbin.size == 0) { + cmds[i] = NULL; + } else { + if ((tuple2 = enif_alloc(tmpbin.size + 1)) == NULL) + goto err; + (void) memcpy(tuple2, tmpbin.data, tmpbin.size); + tuple2[tmpbin.size] = '\0'; + cmds[i] = tuple2; + } + i++; + return get_engine_load_cmd_list(env, tail, cmds, i); + + err: + if (tuple1 != NULL) { + i--; + enif_free(tuple1); + } + cmds[i] = NULL; + return -1; +} +#endif /* HAS_ENGINE_SUPPORT */ + +ERL_NIF_TERM engine_get_all_methods_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* () */ +#ifdef HAS_ENGINE_SUPPORT + ERL_NIF_TERM method_array[12]; + unsigned int i = 0; + + ASSERT(argc == 0); + +#ifdef ENGINE_METHOD_RSA + method_array[i++] = atom_engine_method_rsa; +#endif +#ifdef ENGINE_METHOD_DSA + method_array[i++] = atom_engine_method_dsa; +#endif +#ifdef ENGINE_METHOD_DH + method_array[i++] = atom_engine_method_dh; +#endif +#ifdef ENGINE_METHOD_RAND + method_array[i++] = atom_engine_method_rand; +#endif +#ifdef ENGINE_METHOD_ECDH + method_array[i++] = atom_engine_method_ecdh; +#endif +#ifdef ENGINE_METHOD_ECDSA + method_array[i++] = atom_engine_method_ecdsa; +#endif +#ifdef ENGINE_METHOD_STORE + method_array[i++] = atom_engine_method_store; +#endif +#ifdef ENGINE_METHOD_CIPHERS + method_array[i++] = atom_engine_method_ciphers; +#endif +#ifdef ENGINE_METHOD_DIGESTS + method_array[i++] = atom_engine_method_digests; +#endif +#ifdef ENGINE_METHOD_PKEY_METHS + method_array[i++] = atom_engine_method_pkey_meths; +#endif +#ifdef ENGINE_METHOD_PKEY_ASN1_METHS + method_array[i++] = atom_engine_method_pkey_asn1_meths; +#endif +#ifdef ENGINE_METHOD_EC + method_array[i++] = atom_engine_method_ec; +#endif + + return enif_make_list_from_array(env, method_array, i); +#else + return atom_notsup; +#endif +} diff --git a/lib/crypto/c_src/engine.h b/lib/crypto/c_src/engine.h new file mode 100644 index 0000000000..4a2eed9672 --- /dev/null +++ b/lib/crypto/c_src/engine.h @@ -0,0 +1,49 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_ENGINE_H__ +#define E_ENGINE_H__ 1 + +#include "common.h" + +#ifdef HAS_ENGINE_SUPPORT +int get_engine_and_key_id(ErlNifEnv *env, ERL_NIF_TERM key, char ** id, ENGINE **e); +char *get_key_password(ErlNifEnv *env, ERL_NIF_TERM key); +#endif /* HAS_ENGINE_SUPPORT */ + +int init_engine_ctx(ErlNifEnv *env); + +ERL_NIF_TERM engine_by_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_finish_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_free_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_load_dynamic_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_ctrl_cmd_strings_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_register_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_unregister_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_add_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_remove_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_get_first_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_get_next_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_get_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_get_name_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM engine_get_all_methods_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_ENGINE_H__ */ diff --git a/lib/crypto/c_src/evp.c b/lib/crypto/c_src/evp.c new file mode 100644 index 0000000000..3bf66bfffe --- /dev/null +++ b/lib/crypto/c_src/evp.c @@ -0,0 +1,164 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "evp.h" + +ERL_NIF_TERM evp_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) + /* (Curve, PeerBin, MyBin) */ +{ +#ifdef HAVE_ED_CURVE_DH + ERL_NIF_TERM ret; + int type; + EVP_PKEY_CTX *ctx = NULL; + ErlNifBinary peer_bin, my_bin, key_bin; + EVP_PKEY *peer_key = NULL, *my_key = NULL; + size_t max_size; + int key_bin_alloc = 0; + + ASSERT(argc == 3); + + if (argv[0] == atom_x25519) + type = EVP_PKEY_X25519; + else if (argv[0] == atom_x448) + type = EVP_PKEY_X448; + else + goto bad_arg; + + if (!enif_inspect_binary(env, argv[1], &peer_bin)) + goto bad_arg; + if (!enif_inspect_binary(env, argv[2], &my_bin)) + goto bad_arg; + + if ((my_key = EVP_PKEY_new_raw_private_key(type, NULL, my_bin.data, my_bin.size)) == NULL) + goto err; + if ((ctx = EVP_PKEY_CTX_new(my_key, NULL)) == NULL) + goto err; + + if (EVP_PKEY_derive_init(ctx) != 1) + goto err; + + if ((peer_key = EVP_PKEY_new_raw_public_key(type, NULL, peer_bin.data, peer_bin.size)) == NULL) + goto err; + if (EVP_PKEY_derive_set_peer(ctx, peer_key) != 1) + goto err; + + if (EVP_PKEY_derive(ctx, NULL, &max_size) != 1) + goto err; + + if (!enif_alloc_binary(max_size, &key_bin)) + goto err; + key_bin_alloc = 1; + if (EVP_PKEY_derive(ctx, key_bin.data, &key_bin.size) != 1) + goto err; + + if (key_bin.size < max_size) { + if (!enif_realloc_binary(&key_bin, (size_t)key_bin.size)) + goto err; + } + + ret = enif_make_binary(env, &key_bin); + key_bin_alloc = 0; + goto done; + + bad_arg: + err: + if (key_bin_alloc) + enif_release_binary(&key_bin); + ret = enif_make_badarg(env); + + done: + if (my_key) + EVP_PKEY_free(my_key); + if (peer_key) + EVP_PKEY_free(peer_key); + if (ctx) + EVP_PKEY_CTX_free(ctx); + + return ret; + +#else + return atom_notsup; +#endif +} + +ERL_NIF_TERM evp_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +/* (Curve) */ +{ +#ifdef HAVE_ED_CURVE_DH + int type; + EVP_PKEY_CTX *ctx = NULL; + EVP_PKEY *pkey = NULL; + ERL_NIF_TERM ret_pub, ret_prv, ret; + size_t key_len; + unsigned char *out_pub = NULL, *out_priv = NULL; + + ASSERT(argc == 1); + + if (argv[0] == atom_x25519) + type = EVP_PKEY_X25519; + else if (argv[0] == atom_x448) + type = EVP_PKEY_X448; + else + goto bad_arg; + + if ((ctx = EVP_PKEY_CTX_new_id(type, NULL)) == NULL) + goto bad_arg; + + if (EVP_PKEY_keygen_init(ctx) != 1) + goto err; + if (EVP_PKEY_keygen(ctx, &pkey) != 1) + goto err; + + if (EVP_PKEY_get_raw_public_key(pkey, NULL, &key_len) != 1) + goto err; + if ((out_pub = enif_make_new_binary(env, key_len, &ret_pub)) == NULL) + goto err; + if (EVP_PKEY_get_raw_public_key(pkey, out_pub, &key_len) != 1) + goto err; + + if (EVP_PKEY_get_raw_private_key(pkey, NULL, &key_len) != 1) + goto err; + if ((out_priv = enif_make_new_binary(env, key_len, &ret_prv)) == NULL) + goto err; + if (EVP_PKEY_get_raw_private_key(pkey, out_priv, &key_len) != 1) + goto err; + + ret = enif_make_tuple2(env, ret_pub, ret_prv); + goto done; + + bad_arg: + ret = enif_make_badarg(env); + goto done; + + err: + ret = atom_error; + + done: + if (pkey) + EVP_PKEY_free(pkey); + if (ctx) + EVP_PKEY_CTX_free(ctx); + return ret; + +#else + return atom_notsup; +#endif +} + diff --git a/lib/crypto/c_src/evp.h b/lib/crypto/c_src/evp.h new file mode 100644 index 0000000000..d767260262 --- /dev/null +++ b/lib/crypto/c_src/evp.h @@ -0,0 +1,29 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_EVP_H__ +#define E_EVP_H__ 1 + +#include "common.h" + +ERL_NIF_TERM evp_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM evp_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_EVP_H__ */ diff --git a/lib/crypto/c_src/evp_compat.h b/lib/crypto/c_src/evp_compat.h new file mode 100644 index 0000000000..dc94a61d8e --- /dev/null +++ b/lib/crypto/c_src/evp_compat.h @@ -0,0 +1,210 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_EVP_COMPAT_H__ +#define E_EVP_COMPAT_H__ 1 + +/* + * In OpenSSL 1.1.0, most structs are opaque. That means that + * the structs cannot be allocated as automatic variables on the + * C stack (because the size is unknown) and that it is necessary + * to use access functions. + * + * For backward compatibility to previous versions of OpenSSL, define + * on our versions of the new functions defined in 1.1.0 here, so that + * we don't have to sprinkle ifdefs throughout the code. + */ + +static INLINE HMAC_CTX *HMAC_CTX_new(void); +static INLINE void HMAC_CTX_free(HMAC_CTX *ctx); + +static INLINE HMAC_CTX *HMAC_CTX_new() +{ + HMAC_CTX *ctx; + + if ((ctx = CRYPTO_malloc(sizeof(HMAC_CTX), __FILE__, __LINE__)) == NULL) + return NULL; + + HMAC_CTX_init(ctx); + return ctx; +} + +static INLINE void HMAC_CTX_free(HMAC_CTX *ctx) +{ + if (ctx == NULL) + return; + + HMAC_CTX_cleanup(ctx); + CRYPTO_free(ctx); +} + +/* Renamed in 1.1.0 */ +#define EVP_MD_CTX_new() EVP_MD_CTX_create() +#define EVP_MD_CTX_free(ctx) EVP_MD_CTX_destroy((ctx)) + +static INLINE void *BN_GENCB_get_arg(BN_GENCB *cb); + +static INLINE void *BN_GENCB_get_arg(BN_GENCB *cb) +{ + return cb->arg; +} + +static INLINE int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d); +static INLINE void RSA_get0_key(const RSA *r, const BIGNUM **n, const BIGNUM **e, const BIGNUM **d); +static INLINE int RSA_set0_factors(RSA *r, BIGNUM *p, BIGNUM *q); +static INLINE void RSA_get0_factors(const RSA *r, const BIGNUM **p, const BIGNUM **q); +static INLINE int RSA_set0_crt_params(RSA *r, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp); +static INLINE void RSA_get0_crt_params(const RSA *r, const BIGNUM **dmp1, const BIGNUM **dmq1, const BIGNUM **iqmp); + +static INLINE int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d) +{ + r->n = n; + r->e = e; + r->d = d; + return 1; +} + +static INLINE void RSA_get0_key(const RSA *r, const BIGNUM **n, const BIGNUM **e, const BIGNUM **d) +{ + *n = r->n; + *e = r->e; + *d = r->d; +} + +static INLINE int RSA_set0_factors(RSA *r, BIGNUM *p, BIGNUM *q) +{ + r->p = p; + r->q = q; + return 1; +} + +static INLINE void RSA_get0_factors(const RSA *r, const BIGNUM **p, const BIGNUM **q) +{ + *p = r->p; + *q = r->q; +} + +static INLINE int RSA_set0_crt_params(RSA *r, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp) +{ + r->dmp1 = dmp1; + r->dmq1 = dmq1; + r->iqmp = iqmp; + return 1; +} + +static INLINE void RSA_get0_crt_params(const RSA *r, const BIGNUM **dmp1, const BIGNUM **dmq1, const BIGNUM **iqmp) +{ + *dmp1 = r->dmp1; + *dmq1 = r->dmq1; + *iqmp = r->iqmp; +} + +static INLINE int DSA_set0_key(DSA *d, BIGNUM *pub_key, BIGNUM *priv_key); +static INLINE int DSA_set0_pqg(DSA *d, BIGNUM *p, BIGNUM *q, BIGNUM *g); +static INLINE void DSA_get0_pqg(const DSA *dsa, + const BIGNUM **p, const BIGNUM **q, const BIGNUM **g); +static INLINE void DSA_get0_key(const DSA *dsa, + const BIGNUM **pub_key, const BIGNUM **priv_key); + +static INLINE int DSA_set0_key(DSA *d, BIGNUM *pub_key, BIGNUM *priv_key) +{ + d->pub_key = pub_key; + d->priv_key = priv_key; + return 1; +} + +static INLINE int DSA_set0_pqg(DSA *d, BIGNUM *p, BIGNUM *q, BIGNUM *g) +{ + d->p = p; + d->q = q; + d->g = g; + return 1; +} + +static INLINE void +DSA_get0_pqg(const DSA *dsa, const BIGNUM **p, const BIGNUM **q, const BIGNUM **g) +{ + *p = dsa->p; + *q = dsa->q; + *g = dsa->g; +} + +static INLINE void +DSA_get0_key(const DSA *dsa, const BIGNUM **pub_key, const BIGNUM **priv_key) +{ + if (pub_key) + *pub_key = dsa->pub_key; + + if (priv_key) + *priv_key = dsa->priv_key; +} + + + +static INLINE int DH_set0_key(DH *dh, BIGNUM *pub_key, BIGNUM *priv_key); +static INLINE int DH_set0_pqg(DH *dh, BIGNUM *p, BIGNUM *q, BIGNUM *g); +static INLINE int DH_set_length(DH *dh, long length); +static INLINE void DH_get0_pqg(const DH *dh, + const BIGNUM **p, const BIGNUM **q, const BIGNUM **g); +static INLINE void DH_get0_key(const DH *dh, + const BIGNUM **pub_key, const BIGNUM **priv_key); + +static INLINE int DH_set0_key(DH *dh, BIGNUM *pub_key, BIGNUM *priv_key) +{ + dh->pub_key = pub_key; + dh->priv_key = priv_key; + return 1; +} + +static INLINE int DH_set0_pqg(DH *dh, BIGNUM *p, BIGNUM *q, BIGNUM *g) +{ + dh->p = p; + dh->q = q; + dh->g = g; + return 1; +} + +static INLINE int DH_set_length(DH *dh, long length) +{ + dh->length = length; + return 1; +} + + + +static INLINE void +DH_get0_pqg(const DH *dh, const BIGNUM **p, const BIGNUM **q, const BIGNUM **g) +{ + *p = dh->p; + *q = dh->q; + *g = dh->g; +} + +static INLINE void +DH_get0_key(const DH *dh, const BIGNUM **pub_key, const BIGNUM **priv_key) +{ + if (pub_key) + *pub_key = dh->pub_key; + + if (priv_key) + *priv_key = dh->priv_key; +} + +#endif /* E_EVP_COMPAT_H__ */ diff --git a/lib/crypto/c_src/fips.c b/lib/crypto/c_src/fips.c new file mode 100644 index 0000000000..b2d892d00b --- /dev/null +++ b/lib/crypto/c_src/fips.c @@ -0,0 +1,52 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "fips.h" + +ERL_NIF_TERM info_fips(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ +#ifdef FIPS_SUPPORT + return FIPS_mode() ? atom_enabled : atom_not_enabled; +#else + return atom_not_supported; +#endif +} + +ERL_NIF_TERM enable_fips_mode(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Boolean) */ + if (argv[0] == atom_true) { +#ifdef FIPS_SUPPORT + if (FIPS_mode_set(1)) { + return atom_true; + } +#endif + PRINTF_ERR0("CRYPTO: Could not setup FIPS mode"); + return atom_false; + } else if (argv[0] == atom_false) { +#ifdef FIPS_SUPPORT + if (!FIPS_mode_set(0)) { + return atom_false; + } +#endif + return atom_true; + } else { + return enif_make_badarg(env); + } +} diff --git a/lib/crypto/c_src/fips.h b/lib/crypto/c_src/fips.h new file mode 100644 index 0000000000..9a436bd202 --- /dev/null +++ b/lib/crypto/c_src/fips.h @@ -0,0 +1,29 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_FIPS_H__ +#define E_FIPS_H__ 1 + +#include "common.h" + +ERL_NIF_TERM info_fips(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM enable_fips_mode(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_FIPS_H__ */ diff --git a/lib/crypto/c_src/hash.c b/lib/crypto/c_src/hash.c new file mode 100644 index 0000000000..9b79258585 --- /dev/null +++ b/lib/crypto/c_src/hash.c @@ -0,0 +1,549 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "hash.h" +#include "digest.h" + +#ifdef HAVE_MD5 +# define MD5_CTX_LEN (sizeof(MD5_CTX)) +#endif +#ifdef HAVE_MD4 +# define MD4_CTX_LEN (sizeof(MD4_CTX)) +#endif +#ifdef HAVE_RIPEMD160 +# define RIPEMD160_CTX_LEN (sizeof(RIPEMD160_CTX)) +#endif + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) +struct evp_md_ctx { + EVP_MD_CTX* ctx; +}; + +/* Define resource types for OpenSSL context structures. */ +static ErlNifResourceType* evp_md_ctx_rtype; + +static void evp_md_ctx_dtor(ErlNifEnv* env, struct evp_md_ctx *ctx) { + if (ctx == NULL) + return; + + if (ctx->ctx) + EVP_MD_CTX_free(ctx->ctx); +} +#endif + +int init_hash_ctx(ErlNifEnv* env) { +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) + evp_md_ctx_rtype = enif_open_resource_type(env, NULL, "EVP_MD_CTX", + (ErlNifResourceDtor*) evp_md_ctx_dtor, + ERL_NIF_RT_CREATE|ERL_NIF_RT_TAKEOVER, + NULL); + if (evp_md_ctx_rtype == NULL) + goto err; +#endif + + return 1; + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) + err: + PRINTF_ERR0("CRYPTO: Could not open resource type 'EVP_MD_CTX'"); + return 0; +#endif +} + +ERL_NIF_TERM hash_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type) */ + struct digest_type_t *digp = NULL; + const EVP_MD *md; + ERL_NIF_TERM ret; + + ASSERT(argc == 1); + + if ((digp = get_digest_type(argv[0])) == NULL) + return enif_make_badarg(env); + + if ((md = digp->md.p) == NULL) + return atom_notsup; + + ret = enif_make_new_map(env); + + enif_make_map_put(env, ret, atom_type, + enif_make_int(env, EVP_MD_type(md)), &ret); + enif_make_map_put(env, ret, atom_size, + enif_make_int(env, EVP_MD_size(md)), &ret); + enif_make_map_put(env, ret, atom_block_size, + enif_make_int(env, EVP_MD_block_size(md)), &ret); + + return ret; +} + +ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type, Data) */ + struct digest_type_t *digp = NULL; + const EVP_MD *md; + ErlNifBinary data; + ERL_NIF_TERM ret; + unsigned ret_size; + unsigned char *outp; + + ASSERT(argc == 2); + + if ((digp = get_digest_type(argv[0])) == NULL) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[1], &data)) + goto bad_arg; + + if ((md = digp->md.p) == NULL) + goto err; + + ret_size = (unsigned)EVP_MD_size(md); + ASSERT(0 < ret_size && ret_size <= EVP_MAX_MD_SIZE); + + if ((outp = enif_make_new_binary(env, ret_size, &ret)) == NULL) + goto err; + if (EVP_Digest(data.data, data.size, outp, &ret_size, md, NULL) != 1) + goto err; + + ASSERT(ret_size == (unsigned)EVP_MD_size(md)); + + CONSUME_REDS(env, data); + return ret; + + bad_arg: + return enif_make_badarg(env); + + err: + return atom_notsup; +} + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) + +ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type) */ + struct digest_type_t *digp = NULL; + struct evp_md_ctx *ctx = NULL; + ERL_NIF_TERM ret; + + ASSERT(argc == 1); + + if ((digp = get_digest_type(argv[0])) == NULL) + goto bad_arg; + if (digp->md.p == NULL) + goto err; + + if ((ctx = enif_alloc_resource(evp_md_ctx_rtype, sizeof(struct evp_md_ctx))) == NULL) + goto err; + if ((ctx->ctx = EVP_MD_CTX_new()) == NULL) + goto err; + if (EVP_DigestInit(ctx->ctx, digp->md.p) != 1) + goto err; + + ret = enif_make_resource(env, ctx); + goto done; + + bad_arg: + return enif_make_badarg(env); + + err: + ret = atom_notsup; + + done: + if (ctx) + enif_release_resource(ctx); + return ret; +} + +ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Context, Data) */ + struct evp_md_ctx *ctx, *new_ctx = NULL; + ErlNifBinary data; + ERL_NIF_TERM ret; + + ASSERT(argc == 2); + + if (!enif_get_resource(env, argv[0], evp_md_ctx_rtype, (void**)&ctx)) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[1], &data)) + goto bad_arg; + + if ((new_ctx = enif_alloc_resource(evp_md_ctx_rtype, sizeof(struct evp_md_ctx))) == NULL) + goto err; + if ((new_ctx->ctx = EVP_MD_CTX_new()) == NULL) + goto err; + if (EVP_MD_CTX_copy(new_ctx->ctx, ctx->ctx) != 1) + goto err; + if (EVP_DigestUpdate(new_ctx->ctx, data.data, data.size) != 1) + goto err; + + ret = enif_make_resource(env, new_ctx); + CONSUME_REDS(env, data); + goto done; + + bad_arg: + return enif_make_badarg(env); + + err: + ret = atom_notsup; + + done: + if (new_ctx) + enif_release_resource(new_ctx); + return ret; +} + +ERL_NIF_TERM hash_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Context) */ + struct evp_md_ctx *ctx; + EVP_MD_CTX *new_ctx; + ERL_NIF_TERM ret; + unsigned ret_size; + unsigned char *outp; + + ASSERT(argc == 1); + + if (!enif_get_resource(env, argv[0], evp_md_ctx_rtype, (void**)&ctx)) + goto bad_arg; + + ret_size = (unsigned)EVP_MD_CTX_size(ctx->ctx); + ASSERT(0 < ret_size && ret_size <= EVP_MAX_MD_SIZE); + + if ((new_ctx = EVP_MD_CTX_new()) == NULL) + goto err; + if (EVP_MD_CTX_copy(new_ctx, ctx->ctx) != 1) + goto err; + if ((outp = enif_make_new_binary(env, ret_size, &ret)) == NULL) + goto err; + if (EVP_DigestFinal(new_ctx, outp, &ret_size) != 1) + goto err; + + ASSERT(ret_size == (unsigned)EVP_MD_CTX_size(ctx->ctx)); + goto done; + + bad_arg: + return enif_make_badarg(env); + + err: + ret = atom_notsup; + + done: + if (new_ctx) + EVP_MD_CTX_free(new_ctx); + return ret; +} + +#else /* if OPENSSL_VERSION_NUMBER < 1.0 */ + +ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type) */ + typedef int (*init_fun)(unsigned char*); + struct digest_type_t *digp = NULL; + ERL_NIF_TERM ctx; + size_t ctx_size = 0; + init_fun ctx_init = 0; + unsigned char *outp; + + ASSERT(argc == 1); + + if ((digp = get_digest_type(argv[0])) == NULL) + goto bad_arg; + if (digp->md.p == NULL) + goto err; + + switch (EVP_MD_type(digp->md.p)) + { +#ifdef HAVE_MD4 + case NID_md4: + ctx_size = MD4_CTX_LEN; + ctx_init = (init_fun)(&MD4_Init); + break; +#endif +#ifdef HAVE_MD5 + case NID_md5: + ctx_size = MD5_CTX_LEN; + ctx_init = (init_fun)(&MD5_Init); + break; +#endif +#ifdef HAVE_RIPEMD160 + case NID_ripemd160: + ctx_size = RIPEMD160_CTX_LEN; + ctx_init = (init_fun)(&RIPEMD160_Init); + break; +#endif + case NID_sha1: + ctx_size = sizeof(SHA_CTX); + ctx_init = (init_fun)(&SHA1_Init); + break; +#ifdef HAVE_SHA224 + case NID_sha224: + ctx_size = sizeof(SHA256_CTX); + ctx_init = (init_fun)(&SHA224_Init); + break; +#endif +#ifdef HAVE_SHA256 + case NID_sha256: + ctx_size = sizeof(SHA256_CTX); + ctx_init = (init_fun)(&SHA256_Init); + break; +#endif +#ifdef HAVE_SHA384 + case NID_sha384: + ctx_size = sizeof(SHA512_CTX); + ctx_init = (init_fun)(&SHA384_Init); + break; +#endif +#ifdef HAVE_SHA512 + case NID_sha512: + ctx_size = sizeof(SHA512_CTX); + ctx_init = (init_fun)(&SHA512_Init); + break; +#endif + default: + goto err; + } + ASSERT(ctx_size); + ASSERT(ctx_init); + + if ((outp = enif_make_new_binary(env, ctx_size, &ctx)) == NULL) + goto err; + + if (ctx_init(outp) != 1) + goto err; + + return enif_make_tuple2(env, argv[0], ctx); + + bad_arg: + return enif_make_badarg(env); + + err: + return atom_notsup; +} + +ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* ({Type, Context}, Data) */ + typedef int (*update_fun)(unsigned char*, const unsigned char*, size_t); + ERL_NIF_TERM new_ctx; + ErlNifBinary ctx, data; + const ERL_NIF_TERM *tuple; + int arity; + struct digest_type_t *digp = NULL; + unsigned char *ctx_buff; + size_t ctx_size = 0; + update_fun ctx_update = 0; + + ASSERT(argc == 2); + + if (!enif_get_tuple(env, argv[0], &arity, &tuple)) + goto bad_arg; + if (arity != 2) + goto bad_arg; + if ((digp = get_digest_type(tuple[0])) == NULL) + goto bad_arg; + if (!enif_inspect_binary(env, tuple[1], &ctx)) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[1], &data)) + goto bad_arg; + + if (digp->md.p == NULL) + goto err; + + switch (EVP_MD_type(digp->md.p)) + { +#ifdef HAVE_MD4 + case NID_md4: + ctx_size = MD4_CTX_LEN; + ctx_update = (update_fun)(&MD4_Update); + break; +#endif +#ifdef HAVE_MD5 + case NID_md5: + ctx_size = MD5_CTX_LEN; + ctx_update = (update_fun)(&MD5_Update); + break; +#endif +#ifdef HAVE_RIPEMD160 + case NID_ripemd160: + ctx_size = RIPEMD160_CTX_LEN; + ctx_update = (update_fun)(&RIPEMD160_Update); + break; +#endif + case NID_sha1: + ctx_size = sizeof(SHA_CTX); + ctx_update = (update_fun)(&SHA1_Update); + break; +#ifdef HAVE_SHA224 + case NID_sha224: + ctx_size = sizeof(SHA256_CTX); + ctx_update = (update_fun)(&SHA224_Update); + break; +#endif +#ifdef HAVE_SHA256 + case NID_sha256: + ctx_size = sizeof(SHA256_CTX); + ctx_update = (update_fun)(&SHA256_Update); + break; +#endif +#ifdef HAVE_SHA384 + case NID_sha384: + ctx_size = sizeof(SHA512_CTX); + ctx_update = (update_fun)(&SHA384_Update); + break; +#endif +#ifdef HAVE_SHA512 + case NID_sha512: + ctx_size = sizeof(SHA512_CTX); + ctx_update = (update_fun)(&SHA512_Update); + break; +#endif + default: + goto err; + } + ASSERT(ctx_size); + ASSERT(ctx_update); + + if (ctx.size != ctx_size) + goto bad_arg; + + if ((ctx_buff = enif_make_new_binary(env, ctx_size, &new_ctx)) == NULL) + goto err; + memcpy(ctx_buff, ctx.data, ctx_size); + + if (ctx_update(ctx_buff, data.data, data.size) != 1) + goto err; + + CONSUME_REDS(env, data); + return enif_make_tuple2(env, tuple[0], new_ctx); + + bad_arg: + return enif_make_badarg(env); + + err: + return atom_notsup; +} + +ERL_NIF_TERM hash_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* ({Type, Context}) */ + typedef int (*final_fun)(unsigned char*, void*); + ERL_NIF_TERM ret; + ErlNifBinary ctx; + const ERL_NIF_TERM *tuple; + int arity; + struct digest_type_t *digp = NULL; + const EVP_MD *md; + void *new_ctx = NULL; + size_t ctx_size = 0; + final_fun ctx_final = 0; + unsigned char *outp; + + ASSERT(argc == 1); + + if (!enif_get_tuple(env, argv[0], &arity, &tuple)) + goto bad_arg; + if (arity != 2) + goto bad_arg; + if ((digp = get_digest_type(tuple[0])) == NULL) + goto bad_arg; + if (!enif_inspect_binary(env, tuple[1], &ctx)) + goto bad_arg; + + if ((md = digp->md.p) == NULL) + goto err; + + switch (EVP_MD_type(md)) + { +#ifdef HAVE_MD4 + case NID_md4: + ctx_size = MD4_CTX_LEN; + ctx_final = (final_fun)(&MD4_Final); + break; +#endif +#ifdef HAVE_MD5 + case NID_md5: + ctx_size = MD5_CTX_LEN; + ctx_final = (final_fun)(&MD5_Final); + break; +#endif +#ifdef HAVE_RIPEMD160 + case NID_ripemd160: + ctx_size = RIPEMD160_CTX_LEN; + ctx_final = (final_fun)(&RIPEMD160_Final); + break; +#endif + case NID_sha1: + ctx_size = sizeof(SHA_CTX); + ctx_final = (final_fun)(&SHA1_Final); + break; +#ifdef HAVE_SHA224 + case NID_sha224: + ctx_size = sizeof(SHA256_CTX); + ctx_final = (final_fun)(&SHA224_Final); + break; +#endif +#ifdef HAVE_SHA256 + case NID_sha256: + ctx_size = sizeof(SHA256_CTX); + ctx_final = (final_fun)(&SHA256_Final); + break; +#endif +#ifdef HAVE_SHA384 + case NID_sha384: + ctx_size = sizeof(SHA512_CTX); + ctx_final = (final_fun)(&SHA384_Final); + break; +#endif +#ifdef HAVE_SHA512 + case NID_sha512: + ctx_size = sizeof(SHA512_CTX); + ctx_final = (final_fun)(&SHA512_Final); + break; +#endif + default: + goto err; + } + ASSERT(ctx_size); + ASSERT(ctx_final); + + if (ctx.size != ctx_size) + goto bad_arg; + + if ((new_ctx = enif_alloc(ctx_size)) == NULL) + goto err; + + memcpy(new_ctx, ctx.data, ctx_size); + + if ((outp = enif_make_new_binary(env, (size_t)EVP_MD_size(md), &ret)) == NULL) + goto err; + + if (ctx_final(outp, new_ctx) != 1) + goto err; + + goto done; + + bad_arg: + return enif_make_badarg(env); + + err: + ret = atom_notsup; + + done: + if (new_ctx) + enif_free(new_ctx); + return ret; +} + +#endif /* OPENSSL_VERSION_NUMBER < 1.0 */ diff --git a/lib/crypto/c_src/hash.h b/lib/crypto/c_src/hash.h new file mode 100644 index 0000000000..92a25cedb7 --- /dev/null +++ b/lib/crypto/c_src/hash.h @@ -0,0 +1,34 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_HASH_H__ +#define E_HASH_H__ 1 + +#include "common.h" + +int init_hash_ctx(ErlNifEnv *env); + +ERL_NIF_TERM hash_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM hash_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_HASH_H__ */ diff --git a/lib/crypto/c_src/hmac.c b/lib/crypto/c_src/hmac.c new file mode 100644 index 0000000000..ff7005d75e --- /dev/null +++ b/lib/crypto/c_src/hmac.c @@ -0,0 +1,270 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "hmac.h" +#include "digest.h" + +struct hmac_context +{ + ErlNifMutex* mtx; + int alive; + HMAC_CTX* ctx; +}; + +static ErlNifResourceType* hmac_context_rtype; + +static void hmac_context_dtor(ErlNifEnv* env, struct hmac_context*); + +int init_hmac_ctx(ErlNifEnv *env) { + hmac_context_rtype = enif_open_resource_type(env, NULL, "hmac_context", + (ErlNifResourceDtor*) hmac_context_dtor, + ERL_NIF_RT_CREATE|ERL_NIF_RT_TAKEOVER, + NULL); + if (hmac_context_rtype == NULL) + goto err; + + return 1; + + err: + PRINTF_ERR0("CRYPTO: Could not open resource type 'hmac_context'"); + return 0; +} + +ERL_NIF_TERM hmac_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type, Key, Data) or (Type, Key, Data, MacSize) */ + struct digest_type_t *digp = NULL; + ErlNifBinary key, data; + unsigned char buff[EVP_MAX_MD_SIZE]; + unsigned size = 0, req_size = 0; + ERL_NIF_TERM ret; + unsigned char *outp; + + ASSERT(argc == 3 || argc == 4); + + if ((digp = get_digest_type(argv[0])) == NULL) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[1], &key)) + goto bad_arg; + if (key.size > INT_MAX) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[2], &data)) + goto bad_arg; + if (argc == 4) { + if (!enif_get_uint(env, argv[3], &req_size)) + goto bad_arg; + } + + if (digp->md.p == NULL) + goto err; + if (HMAC(digp->md.p, + key.data, (int)key.size, + data.data, data.size, + buff, &size) == NULL) + goto err; + + ASSERT(0 < size && size <= EVP_MAX_MD_SIZE); + CONSUME_REDS(env, data); + + if (argc == 4) { + if (req_size > size) + goto bad_arg; + + size = req_size; + } + + if ((outp = enif_make_new_binary(env, size, &ret)) == NULL) + goto err; + + memcpy(outp, buff, size); + return ret; + + bad_arg: + return enif_make_badarg(env); + + err: + return atom_notsup; +} + +static void hmac_context_dtor(ErlNifEnv* env, struct hmac_context *obj) +{ + if (obj == NULL) + return; + + if (obj->alive) { + if (obj->ctx) + HMAC_CTX_free(obj->ctx); + obj->alive = 0; + } + + if (obj->mtx != NULL) + enif_mutex_destroy(obj->mtx); +} + +ERL_NIF_TERM hmac_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Type, Key) */ + struct digest_type_t *digp = NULL; + ErlNifBinary key; + ERL_NIF_TERM ret; + struct hmac_context *obj = NULL; + + ASSERT(argc == 2); + + if ((digp = get_digest_type(argv[0])) == NULL) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[1], &key)) + goto bad_arg; + if (key.size > INT_MAX) + goto bad_arg; + + if (digp->md.p == NULL) + goto err; + + if ((obj = enif_alloc_resource(hmac_context_rtype, sizeof(struct hmac_context))) == NULL) + goto err; + obj->ctx = NULL; + obj->mtx = NULL; + obj->alive = 0; + + if ((obj->ctx = HMAC_CTX_new()) == NULL) + goto err; + obj->alive = 1; + if ((obj->mtx = enif_mutex_create("crypto.hmac")) == NULL) + goto err; + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) + // Check the return value of HMAC_Init: it may fail in FIPS mode + // for disabled algorithms + if (!HMAC_Init_ex(obj->ctx, key.data, (int)key.size, digp->md.p, NULL)) + goto err; +#else + // In ancient versions of OpenSSL, this was a void function. + HMAC_Init_ex(obj->ctx, key.data, (int)key.size, digp->md.p, NULL); +#endif + + ret = enif_make_resource(env, obj); + goto done; + + bad_arg: + return enif_make_badarg(env); + + err: + ret = atom_notsup; + + done: + if (obj) + enif_release_resource(obj); + return ret; +} + +ERL_NIF_TERM hmac_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Context, Data) */ + ERL_NIF_TERM ret; + ErlNifBinary data; + struct hmac_context *obj = NULL; + + ASSERT(argc == 2); + + if (!enif_get_resource(env, argv[0], (ErlNifResourceType*)hmac_context_rtype, (void**)&obj)) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[1], &data)) + goto bad_arg; + + enif_mutex_lock(obj->mtx); + if (!obj->alive) + goto err; + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) + if (!HMAC_Update(obj->ctx, data.data, data.size)) + goto err; +#else + // In ancient versions of OpenSSL, this was a void function. + HMAC_Update(obj->ctx, data.data, data.size); +#endif + + CONSUME_REDS(env,data); + ret = argv[0]; + goto done; + + bad_arg: + return enif_make_badarg(env); + + err: + ret = enif_make_badarg(env); + + done: + enif_mutex_unlock(obj->mtx); + return ret; +} + +ERL_NIF_TERM hmac_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Context) or (Context, HashLen) */ + ERL_NIF_TERM ret; + struct hmac_context* obj; + unsigned char mac_buf[EVP_MAX_MD_SIZE]; + unsigned char * mac_bin; + unsigned int req_len = 0; + unsigned int mac_len; + + ASSERT(argc == 1 || argc == 2); + + if (!enif_get_resource(env, argv[0], (ErlNifResourceType*)hmac_context_rtype, (void**)&obj)) + goto bad_arg; + if (argc == 2) { + if (!enif_get_uint(env, argv[1], &req_len)) + goto bad_arg; + } + + enif_mutex_lock(obj->mtx); + if (!obj->alive) + goto err; + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) + if (!HMAC_Final(obj->ctx, mac_buf, &mac_len)) + goto err; +#else + // In ancient versions of OpenSSL, this was a void function. + HMAC_Final(obj->ctx, mac_buf, &mac_len); +#endif + + if (obj->ctx) + HMAC_CTX_free(obj->ctx); + obj->alive = 0; + + if (argc == 2 && req_len < mac_len) { + /* Only truncate to req_len bytes if asked. */ + mac_len = req_len; + } + if ((mac_bin = enif_make_new_binary(env, mac_len, &ret)) == NULL) + goto err; + + memcpy(mac_bin, mac_buf, mac_len); + goto done; + + bad_arg: + return enif_make_badarg(env); + + err: + ret = enif_make_badarg(env); + + done: + enif_mutex_unlock(obj->mtx); + return ret; +} + diff --git a/lib/crypto/c_src/hmac.h b/lib/crypto/c_src/hmac.h new file mode 100644 index 0000000000..1f0e0ca632 --- /dev/null +++ b/lib/crypto/c_src/hmac.h @@ -0,0 +1,33 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_HMAC_H__ +#define E_HMAC_H__ 1 + +#include "common.h" + +int init_hmac_ctx(ErlNifEnv *env); + +ERL_NIF_TERM hmac_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM hmac_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM hmac_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM hmac_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_HMAC_H__ */ diff --git a/lib/crypto/c_src/info.c b/lib/crypto/c_src/info.c new file mode 100644 index 0000000000..42f477fead --- /dev/null +++ b/lib/crypto/c_src/info.c @@ -0,0 +1,107 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "info.h" + +#ifdef HAVE_DYNAMIC_CRYPTO_LIB + +# if defined(DEBUG) +char *crypto_callback_name = "crypto_callback.debug"; +# elif defined(VALGRIND) +char *crypto_callback_name = "crypto_callback.valgrind"; +# else +char *crypto_callback_name = "crypto_callback"; +# endif + +int change_basename(ErlNifBinary* bin, char* buf, size_t bufsz, const char* newfile) +{ + size_t i; + size_t newlen; + + for (i = bin->size; i > 0; i--) { + if (bin->data[i-1] == '/') + break; + } + + newlen = strlen(newfile); + if (i > SIZE_MAX - newlen) + goto err; + + if (i + newlen >= bufsz) + goto err; + + memcpy(buf, bin->data, i); + strcpy(buf+i, newfile); + + return 1; + + err: + return 0; +} + +void error_handler(void* null, const char* errstr) +{ + PRINTF_ERR1("CRYPTO LOADING ERROR: '%s'", errstr); +} +#endif /* HAVE_DYNAMIC_CRYPTO_LIB */ + +ERL_NIF_TERM info_lib(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{/* () */ + /* [{<<"OpenSSL">>,9470143,<<"OpenSSL 0.9.8k 25 Mar 2009">>}] */ + + ERL_NIF_TERM name_term, ver_term; + static const char libname[] = "OpenSSL"; + size_t name_sz; + const char* ver; + size_t ver_sz; + int ver_num; + unsigned char *out_name, *out_ver; + + ASSERT(argc == 0); + + name_sz = strlen(libname); + ver = SSLeay_version(SSLEAY_VERSION); + ver_sz = strlen(ver); + ver_num = OPENSSL_VERSION_NUMBER; + + /* R16: + * Ignore library version number from SSLeay() and instead show header + * version. Otherwise user might try to call a function that is implemented + * by a newer library but not supported by the headers used at compile time. + * Example: DES_ede3_cfb_encrypt in 0.9.7i but not in 0.9.7d. + * + * Version string is still from library though. + */ + + if ((out_name = enif_make_new_binary(env, name_sz, &name_term)) == NULL) + goto err; + if ((out_ver = enif_make_new_binary(env, ver_sz, &ver_term)) == NULL) + goto err; + + memcpy(out_name, libname, name_sz); + memcpy(out_ver, ver, ver_sz); + + return enif_make_list1(env, enif_make_tuple3(env, name_term, + enif_make_int(env, ver_num), + ver_term)); + + err: + return enif_make_badarg(env); +} diff --git a/lib/crypto/c_src/info.h b/lib/crypto/c_src/info.h new file mode 100644 index 0000000000..67690625c9 --- /dev/null +++ b/lib/crypto/c_src/info.h @@ -0,0 +1,35 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_INFO_H__ +#define E_INFO_H__ 1 + +#include "common.h" + +#ifdef HAVE_DYNAMIC_CRYPTO_LIB +extern char *crypto_callback_name; + +int change_basename(ErlNifBinary* bin, char* buf, size_t bufsz, const char* newfile); +void error_handler(void* null, const char* errstr); +#endif + +ERL_NIF_TERM info_lib(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_INFO_H__ */ diff --git a/lib/crypto/c_src/math.c b/lib/crypto/c_src/math.c new file mode 100644 index 0000000000..85494bbc93 --- /dev/null +++ b/lib/crypto/c_src/math.c @@ -0,0 +1,53 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "math.h" + +ERL_NIF_TERM do_exor(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Data1, Data2) */ + ErlNifBinary d1, d2; + unsigned char* ret_ptr; + size_t i; + ERL_NIF_TERM ret; + + ASSERT(argc == 2); + + if (!enif_inspect_iolist_as_binary(env, argv[0], &d1)) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, argv[1], &d2)) + goto bad_arg; + if (d1.size != d2.size) + goto bad_arg; + + if ((ret_ptr = enif_make_new_binary(env, d1.size, &ret)) == NULL) + goto err; + + for (i=0; i<d1.size; i++) { + ret_ptr[i] = d1.data[i] ^ d2.data[i]; + } + + CONSUME_REDS(env,d1); + return ret; + + bad_arg: + err: + return enif_make_badarg(env); +} + diff --git a/lib/crypto/c_src/math.h b/lib/crypto/c_src/math.h new file mode 100644 index 0000000000..b8d68ea654 --- /dev/null +++ b/lib/crypto/c_src/math.h @@ -0,0 +1,28 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_MATH_H__ +#define E_MATH_H__ 1 + +#include "common.h" + +ERL_NIF_TERM do_exor(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_MATH_H__ */ diff --git a/lib/crypto/c_src/openssl_config.h b/lib/crypto/c_src/openssl_config.h new file mode 100644 index 0000000000..339eb5b8f4 --- /dev/null +++ b/lib/crypto/c_src/openssl_config.h @@ -0,0 +1,405 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_OPENSSL_CONFIG_H__ +#define E_OPENSSL_CONFIG_H__ 1 + +#define OPENSSL_THREAD_DEFINES +#include <openssl/opensslconf.h> + +#include <openssl/crypto.h> +#include <openssl/des.h> + +/* #include <openssl/idea.h> This is not supported on the openssl OTP requires */ +#include <openssl/dsa.h> +#include <openssl/rsa.h> +#include <openssl/aes.h> +#include <openssl/md5.h> +#include <openssl/md4.h> +#include <openssl/sha.h> +#include <openssl/ripemd.h> +#include <openssl/bn.h> +#include <openssl/objects.h> +#ifndef OPENSSL_NO_RC4 + #include <openssl/rc4.h> +#endif /* OPENSSL_NO_RC4 */ +#ifndef OPENSSL_NO_RC2 + #include <openssl/rc2.h> +#endif +#include <openssl/blowfish.h> +#include <openssl/rand.h> +#include <openssl/evp.h> +#include <openssl/hmac.h> +#include <openssl/err.h> + +/* Helper macro to construct a OPENSSL_VERSION_NUMBER. + * See openssl/opensslv.h + */ +#define PACKED_OPENSSL_VERSION(MAJ, MIN, FIX, P) \ + ((((((((MAJ << 8) | MIN) << 8 ) | FIX) << 8) | (P-'a'+1)) << 4) | 0xf) + +#define PACKED_OPENSSL_VERSION_PLAIN(MAJ, MIN, FIX) \ + PACKED_OPENSSL_VERSION(MAJ,MIN,FIX,('a'-1)) + + +/* LibreSSL was cloned from OpenSSL 1.0.1g and claims to be API and BPI compatible + * with 1.0.1. + * + * LibreSSL has the same names on include files and symbols as OpenSSL, but defines + * the OPENSSL_VERSION_NUMBER to be >= 2.0.0 + * + * Therefor works tests like this as intendend: + * OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) + * (The test is for example "2.4.2" >= "1.0.0" although the test + * with the cloned OpenSSL test would be "1.0.1" >= "1.0.0") + * + * But tests like this gives wrong result: + * OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) + * (The test is false since "2.4.2" < "1.1.0". It should have been + * true because the LibreSSL API version is "1.0.1") + * + */ + +#ifdef LIBRESSL_VERSION_NUMBER +/* A macro to test on in this file */ +#define HAS_LIBRESSL +#endif + +#ifdef HAS_LIBRESSL +/* LibreSSL dislikes FIPS */ +# ifdef FIPS_SUPPORT +# undef FIPS_SUPPORT +# endif + +/* LibreSSL has never supported the custom mem functions */ +#ifndef HAS_LIBRESSL +# define HAS_CRYPTO_MEM_FUNCTIONS +#endif + +# if LIBRESSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(2,7,0) +/* LibreSSL wants the 1.0.1 API */ +# define NEED_EVP_COMPATIBILITY_FUNCTIONS +# endif +#endif + + +#if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION_PLAIN(1,1,0) +# define NEED_EVP_COMPATIBILITY_FUNCTIONS +#endif + + +#ifndef HAS_LIBRESSL +# if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) +# define HAS_EVP_PKEY_CTX +# define HAVE_EVP_CIPHER_CTX_COPY +# endif +#endif + + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) +#include <openssl/modes.h> +#endif + +#include "crypto_callback.h" + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(0,9,8) \ + && !defined(OPENSSL_NO_SHA224) && defined(NID_sha224) \ + && !defined(OPENSSL_NO_SHA256) /* disabled like this in my sha.h (?) */ +# define HAVE_SHA224 +#endif +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(0,9,8) \ + && !defined(OPENSSL_NO_SHA256) && defined(NID_sha256) +# define HAVE_SHA256 +#endif +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(0,9,8) \ + && !defined(OPENSSL_NO_SHA384) && defined(NID_sha384)\ + && !defined(OPENSSL_NO_SHA512) /* disabled like this in my sha.h (?) */ +# define HAVE_SHA384 +#endif +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(0,9,8) \ + && !defined(OPENSSL_NO_SHA512) && defined(NID_sha512) +# define HAVE_SHA512 +#endif +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(0,9,7,'e') +# define HAVE_DES_ede3_cfb_encrypt +#endif + +// SHA3: +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,1,1) +// An error in beta releases of 1.1.1 fixed in production release +# ifdef NID_sha3_224 +# define HAVE_SHA3_224 +# endif +# ifdef NID_sha3_256 +# define HAVE_SHA3_256 +# endif +#endif +# ifdef NID_sha3_384 +# define HAVE_SHA3_384 +# endif +# ifdef NID_sha3_512 +# define HAVE_SHA3_512 +# endif + +// BLAKE2: +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,1,1) \ + && !defined(HAS_LIBRESSL) \ + && !defined(OPENSSL_NO_BLAKE2) +# define HAVE_BLAKE2 +#endif + +#ifndef OPENSSL_NO_BF +# define HAVE_BF +#endif + +#ifndef OPENSSL_NO_DES +# define HAVE_DES +#endif + +#ifndef OPENSSL_NO_DH +# define HAVE_DH +#endif + +#ifndef OPENSSL_NO_DSA +# define HAVE_DSA +#endif + +#ifndef OPENSSL_NO_MD4 +# define HAVE_MD4 +#endif + +#ifndef OPENSSL_NO_MD5 +# define HAVE_MD5 +#endif + +#ifndef OPENSSL_NO_RC2 +# define HAVE_RC2 +#endif + +#ifndef OPENSSL_NO_RC4 +# define HAVE_RC4 +#endif + +#ifndef OPENSSL_NO_RMD160 +/* Note RMD160 vs RIPEMD160 */ +# define HAVE_RIPEMD160 +#endif + + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(0,9,8,'o') \ + && !defined(OPENSSL_NO_EC) \ + && !defined(OPENSSL_NO_ECDH) \ + && !defined(OPENSSL_NO_ECDSA) +# define HAVE_EC +#endif + +// (test for >= 1.1.1pre8) +#if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1) -7) \ + && !defined(HAS_LIBRESSL) \ + && defined(HAVE_EC) +# define HAVE_ED_CURVE_DH +# if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)) +# define HAVE_EDDSA +# endif +#endif + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(0,9,8,'c') +# define HAVE_AES_IGE +#endif + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,1) +# define HAVE_EVP_AES_CTR +# define HAVE_AEAD +# define HAVE_GCM +# define HAVE_CCM +# ifndef OPENSSL_NO_CMAC +# define HAVE_CMAC +# endif +# if defined(RSA_PKCS1_OAEP_PADDING) +# define HAVE_RSA_OAEP_PADDING +# endif +# define HAVE_RSA_MGF1_MD +# if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION(1,0,1,'d') +# define HAVE_GCM_EVP_DECRYPT_BUG +# endif +#endif + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,1,0) +# ifndef HAS_LIBRESSL +# if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305) +# define HAVE_CHACHA20_POLY1305 +# endif +# define HAVE_RSA_OAEP_MD +# endif +#endif + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(1,1,0,'d') +# ifndef HAS_LIBRESSL +# ifndef OPENSSL_NO_CHACHA +# define HAVE_CHACHA20 +# endif +# endif +#endif + +// OPENSSL_VERSION_NUMBER >= 1.1.1-pre8 +#if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)-7) +# ifndef HAS_LIBRESSL +# if !defined(OPENSSL_NO_POLY1305) +# define HAVE_POLY1305 +# endif +# endif +#endif + +#if OPENSSL_VERSION_NUMBER <= PACKED_OPENSSL_VERSION(0,9,8,'l') +# define HAVE_ECB_IVEC_BUG +#endif + +#ifndef HAS_LIBRESSL +# ifdef RSA_SSLV23_PADDING +# define HAVE_RSA_SSLV23_PADDING +# endif +#endif + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) +# ifdef RSA_PKCS1_PSS_PADDING +# define HAVE_RSA_PKCS1_PSS_PADDING +# endif +#endif + +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(0,9,8,'h') \ + && defined(HAVE_EC) +/* If OPENSSL_NO_EC is set, there will be an error in ec.h included from engine.h + So if EC is disabled, you can't use Engine either.... +*/ +# define HAS_ENGINE_SUPPORT +#endif + + +#if defined(HAS_ENGINE_SUPPORT) +# include <openssl/engine.h> +#endif + +#if defined(HAVE_CMAC) +#include <openssl/cmac.h> +#endif + +#if defined(HAVE_EC) +#include <openssl/ec.h> +#include <openssl/ecdh.h> +#include <openssl/ecdsa.h> +#endif + +#ifdef VALGRIND + # include <valgrind/memcheck.h> + +/* libcrypto mixes supplied buffer contents into its entropy pool, + which makes valgrind complain about the use of uninitialized data. + We use this valgrind "request" to make sure that no such seemingly + undefined data is returned. +*/ + # define ERL_VALGRIND_MAKE_MEM_DEFINED(ptr,size) \ + VALGRIND_MAKE_MEM_DEFINED(ptr,size) + + # define ERL_VALGRIND_ASSERT_MEM_DEFINED(Ptr,Size) \ + do { \ + int __erl_valgrind_mem_defined = VALGRIND_CHECK_MEM_IS_DEFINED((Ptr),(Size)); \ + if (__erl_valgrind_mem_defined != 0) { \ + fprintf(stderr,"\r\n####### VALGRIND_ASSSERT(%p,%ld) failed at %s:%d\r\n", \ + (Ptr),(long)(Size), __FILE__, __LINE__); \ + abort(); \ + } \ + } while (0) + +#else + # define ERL_VALGRIND_MAKE_MEM_DEFINED(ptr,size) + # define ERL_VALGRIND_ASSERT_MEM_DEFINED(ptr,size) +#endif + +#ifdef DEBUG + # define ASSERT(e) \ + ((void) ((e) ? 1 : (fprintf(stderr,"Assert '%s' failed at %s:%d\n",\ + #e, __FILE__, __LINE__), abort(), 0))) +#else + # define ASSERT(e) ((void) 1) +#endif + +#ifdef __GNUC__ + # define INLINE __inline__ +#elif defined(__WIN32__) + # define INLINE __forceinline +#else + # define INLINE +#endif + + +#define get_int32(s) ((((unsigned char*) (s))[0] << 24) | \ + (((unsigned char*) (s))[1] << 16) | \ + (((unsigned char*) (s))[2] << 8) | \ + (((unsigned char*) (s))[3])) + +#define put_uint32(s,i) \ +{ (s)[0] = (unsigned char)(((i) >> 24) & 0xff);\ + (s)[1] = (unsigned char)(((i) >> 16) & 0xff);\ + (s)[2] = (unsigned char)(((i) >> 8) & 0xff);\ + (s)[3] = (unsigned char)((i) & 0xff);\ +} + +/* This shall correspond to the similar macro in crypto.erl */ +/* Current value is: erlang:system_info(context_reductions) * 10 */ +#define MAX_BYTES_TO_NIF 20000 + +#define CONSUME_REDS(NifEnv, Ibin) \ +do { \ + size_t _cost = (Ibin).size; \ + if (_cost > SIZE_MAX / 100) \ + _cost = 100; \ + else \ + _cost = (_cost * 100) / MAX_BYTES_TO_NIF; \ + \ + if (_cost) { \ + (void) enif_consume_timeslice((NifEnv), \ + (_cost > 100) ? 100 : (int)_cost); \ + } \ + } while (0) + +#ifdef NEED_EVP_COMPATIBILITY_FUNCTIONS +# include "evp_compat.h" +#else +# define HAVE_OPAQUE_BN_GENCB +#endif + +#if 0 +# define PRINTF_ERR0(FMT) enif_fprintf(stderr, FMT "\n") +# define PRINTF_ERR1(FMT, A1) enif_fprintf(stderr, FMT "\n", A1) +# define PRINTF_ERR2(FMT, A1, A2) enif_fprintf(stderr, FMT "\n", A1, A2) +#else +# define PRINTF_ERR0(FMT) +# define PRINTF_ERR1(FMT,A1) +# define PRINTF_ERR2(FMT,A1,A2) +#endif + +#ifdef FIPS_SUPPORT +/* In FIPS mode non-FIPS algorithms are disabled and return badarg. */ +#define CHECK_NO_FIPS_MODE() { if (FIPS_mode()) return atom_notsup; } +#else +#define CHECK_NO_FIPS_MODE() +#endif + +#endif /* E_OPENSSL_CONFIG_H__ */ diff --git a/lib/crypto/c_src/otp_test_engine.c b/lib/crypto/c_src/otp_test_engine.c index f452fb2d0c..c3bd9dfb55 100644 --- a/lib/crypto/c_src/otp_test_engine.c +++ b/lib/crypto/c_src/otp_test_engine.c @@ -21,8 +21,11 @@ #ifdef _WIN32 #define OPENSSL_OPT_WINDLL #endif + #include <stdio.h> #include <string.h> +#include <limits.h> +#include <stdint.h> #include <openssl/md5.h> #include <openssl/rsa.h> @@ -87,13 +90,12 @@ static int test_init(ENGINE *e) { printf("OTP Test Engine Initializatzion!\r\n"); #if defined(FAKE_RSA_IMPL) - if ( !RSA_meth_set_finish(test_rsa_method, test_rsa_free) - || !RSA_meth_set_sign(test_rsa_method, test_rsa_sign) - || !RSA_meth_set_verify(test_rsa_method, test_rsa_verify) - ) { - fprintf(stderr, "Setup RSA_METHOD failed\r\n"); - return 0; - } + if (!RSA_meth_set_finish(test_rsa_method, test_rsa_free)) + goto err; + if (!RSA_meth_set_sign(test_rsa_method, test_rsa_sign)) + goto err; + if (!RSA_meth_set_verify(test_rsa_method, test_rsa_verify)) + goto err; #endif /* if defined(FAKE_RSA_IMPL) */ /* Load all digest and cipher algorithms. Needed for password protected private keys */ @@ -101,6 +103,12 @@ static int test_init(ENGINE *e) { OpenSSL_add_all_digests(); return 111; + +#if defined(FAKE_RSA_IMPL) +err: + fprintf(stderr, "Setup RSA_METHOD failed\r\n"); + return 0; +#endif } static void add_test_data(unsigned char *md, unsigned int len) @@ -152,15 +160,15 @@ static int test_engine_md5_update(EVP_MD_CTX *ctx,const void *data, size_t count static int test_engine_md5_final(EVP_MD_CTX *ctx,unsigned char *md) { #ifdef OLD - int ret; + fprintf(stderr, "MD5 final size of EVP_MD: %lu\r\n", (unsigned long)sizeof(EVP_MD)); + if (!MD5_Final(md, data(ctx))) + goto err; - fprintf(stderr, "MD5 final size of EVP_MD: %lu\r\n", sizeof(EVP_MD)); - ret = MD5_Final(md, data(ctx)); + add_test_data(md, MD5_DIGEST_LENGTH); + return 1; - if (ret > 0) { - add_test_data(md, MD5_DIGEST_LENGTH); - } - return ret; + err: + return 0; #else fprintf(stderr, "MD5 final\r\n"); add_test_data(md, MD5_DIGEST_LENGTH); @@ -190,7 +198,6 @@ static int test_digest_ids[] = {NID_md5}; static int test_engine_digest_selector(ENGINE *e, const EVP_MD **digest, const int **nids, int nid) { - int ok = 1; if (!digest) { *nids = test_digest_ids; fprintf(stderr, "Digest is empty! Nid:%d\r\n", nid); @@ -201,64 +208,82 @@ static int test_engine_digest_selector(ENGINE *e, const EVP_MD **digest, #ifdef OLD *digest = &test_engine_md5_method; #else - EVP_MD *md = EVP_MD_meth_new(NID_md5, NID_undef); - if (!md || - !EVP_MD_meth_set_result_size(md, MD5_DIGEST_LENGTH) || - !EVP_MD_meth_set_flags(md, 0) || - !EVP_MD_meth_set_init(md, test_engine_md5_init) || - !EVP_MD_meth_set_update(md, test_engine_md5_update) || - !EVP_MD_meth_set_final(md, test_engine_md5_final) || - !EVP_MD_meth_set_copy(md, NULL) || - !EVP_MD_meth_set_cleanup(md, NULL) || - !EVP_MD_meth_set_input_blocksize(md, MD5_CBLOCK) || - !EVP_MD_meth_set_app_datasize(md, sizeof(EVP_MD *) + sizeof(MD5_CTX)) || - !EVP_MD_meth_set_ctrl(md, NULL)) - { - ok = 0; - *digest = NULL; - } else - { - *digest = md; - } + EVP_MD *md; + + if ((md = EVP_MD_meth_new(NID_md5, NID_undef)) == NULL) + goto err; + if (EVP_MD_meth_set_result_size(md, MD5_DIGEST_LENGTH) != 1) + goto err; + if (EVP_MD_meth_set_flags(md, 0) != 1) + goto err; + if (EVP_MD_meth_set_init(md, test_engine_md5_init) != 1) + goto err; + if (EVP_MD_meth_set_update(md, test_engine_md5_update) != 1) + goto err; + if (EVP_MD_meth_set_final(md, test_engine_md5_final) != 1) + goto err; + if (EVP_MD_meth_set_copy(md, NULL) != 1) + goto err; + if (EVP_MD_meth_set_cleanup(md, NULL) != 1) + goto err; + if (EVP_MD_meth_set_input_blocksize(md, MD5_CBLOCK) != 1) + goto err; + if (EVP_MD_meth_set_app_datasize(md, sizeof(EVP_MD *) + sizeof(MD5_CTX)) != 1) + goto err; + if (EVP_MD_meth_set_ctrl(md, NULL) != 1) + goto err; + + *digest = md; #endif } else { - ok = 0; - *digest = NULL; + goto err; } - return ok; + return 1; + + err: + *digest = NULL; + return 0; } static int bind_helper(ENGINE * e, const char *id) { #if defined(FAKE_RSA_IMPL) - test_rsa_method = RSA_meth_new("OTP test RSA method", 0); - if (test_rsa_method == NULL) { + if ((test_rsa_method = RSA_meth_new("OTP test RSA method", 0)) == NULL) { fprintf(stderr, "RSA_meth_new failed\r\n"); - return 0; + goto err; } #endif /* if defined(FAKE_RSA_IMPL) */ - if (!ENGINE_set_id(e, test_engine_id) - || !ENGINE_set_name(e, test_engine_name) - || !ENGINE_set_init_function(e, test_init) - || !ENGINE_set_digests(e, &test_engine_digest_selector) - /* For testing of key storage in an Engine: */ - || !ENGINE_set_load_privkey_function(e, &test_privkey_load) - || !ENGINE_set_load_pubkey_function(e, &test_pubkey_load) - ) - return 0; + if (!ENGINE_set_id(e, test_engine_id)) + goto err; + if (!ENGINE_set_name(e, test_engine_name)) + goto err; + if (!ENGINE_set_init_function(e, test_init)) + goto err; + if (!ENGINE_set_digests(e, &test_engine_digest_selector)) + goto err; + /* For testing of key storage in an Engine: */ + if (!ENGINE_set_load_privkey_function(e, &test_privkey_load)) + goto err; + if (!ENGINE_set_load_pubkey_function(e, &test_pubkey_load)) + goto err; #if defined(FAKE_RSA_IMPL) - if ( !ENGINE_set_RSA(e, test_rsa_method) ) { - RSA_meth_free(test_rsa_method); - test_rsa_method = NULL; - return 0; - } + if (!ENGINE_set_RSA(e, test_rsa_method)) + goto err; #endif /* if defined(FAKE_RSA_IMPL) */ return 1; + + err: +#if defined(FAKE_RSA_IMPL) + if (test_rsa_method) + RSA_meth_free(test_rsa_method); + test_rsa_method = NULL; +#endif + return 0; } IMPLEMENT_DYNAMIC_CHECK_FN(); @@ -304,7 +329,7 @@ EVP_PKEY* test_key_load(ENGINE *eng, const char *id, UI_METHOD *ui_method, void fprintf(stderr, "Contents of file \"%s\":\r\n",id); f = fopen(id, "r"); { /* Print the contents of the key file */ - char c; + int c; while (!feof(f)) { switch (c=fgetc(f)) { case '\n': @@ -324,23 +349,28 @@ EVP_PKEY* test_key_load(ENGINE *eng, const char *id, UI_METHOD *ui_method, void int pem_passwd_cb_fun(char *buf, int size, int rwflag, void *password) { - int i; + size_t i; + + if (size < 0) + return 0; fprintf(stderr, "In pem_passwd_cb_fun\r\n"); if (!password) return 0; i = strlen(password); - if (i < size) { - /* whole pwd (incl terminating 0) fits */ - fprintf(stderr, "Got FULL pwd %d(%d) chars\r\n", i, size); - memcpy(buf, (char*)password, i+1); - return i+1; - } else { - fprintf(stderr, "Got TO LONG pwd %d(%d) chars\r\n", i, size); - /* meaningless with a truncated password */ - return 0; - } + if (i >= (size_t)size || i > INT_MAX - 1) + goto err; + + /* whole pwd (incl terminating 0) fits */ + fprintf(stderr, "Got FULL pwd %zu(%d) chars\r\n", i, size); + memcpy(buf, (char*)password, i+1); + return (int)i+1; + + err: + fprintf(stderr, "Got TO LONG pwd %zu(%d) chars\r\n", i, size); + /* meaningless with a truncated password */ + return 0; } #endif @@ -349,7 +379,7 @@ int pem_passwd_cb_fun(char *buf, int size, int rwflag, void *password) /* RSA sign. This returns a fixed string so the test case can test that it was called instead of the cryptolib default RSA sign */ -unsigned char fake_flag[] = {255,3,124,180,35,10,180,151,101,247,62,59,80,122,220, +static unsigned char fake_flag[] = {255,3,124,180,35,10,180,151,101,247,62,59,80,122,220, 142,24,180,191,34,51,150,112,27,43,142,195,60,245,213,80,179}; int test_rsa_sign(int dtype, @@ -360,11 +390,10 @@ int test_rsa_sign(int dtype, /* The key */ const RSA *rsa) { - int slen; fprintf(stderr, "test_rsa_sign (dtype=%i) called m_len=%u *siglen=%u\r\n", dtype, m_len, *siglen); if (!sigret) { fprintf(stderr, "sigret = NULL\r\n"); - return -1; + goto err; } /* {int i; @@ -376,14 +405,20 @@ int test_rsa_sign(int dtype, if ((sizeof(fake_flag) == m_len) && memcmp(m,fake_flag,m_len) == 0) { + int slen; + printf("To be faked\r\n"); /* To be faked */ - slen = RSA_size(rsa); - add_test_data(sigret, slen); /* The signature is 0,1,2...255,0,1... */ - *siglen = slen; /* Must set this. Why? */ + if ((slen = RSA_size(rsa)) < 0) + goto err; + add_test_data(sigret, (unsigned int)slen); /* The signature is 0,1,2...255,0,1... */ + *siglen = (unsigned int)slen; /* Must set this. Why? */ return 1; /* 1 = success */ } return 0; + + err: + return -1; } int test_rsa_verify(int dtype, @@ -398,8 +433,13 @@ int test_rsa_verify(int dtype, if ((sizeof(fake_flag) == m_len) && memcmp(m,fake_flag,m_len) == 0) { + int size; + + if ((size = RSA_size(rsa)) < 0) + return 0; + printf("To be faked\r\n"); - return (siglen == RSA_size(rsa)) + return (siglen == (unsigned int)size) && chk_test_data(sigret, siglen); } return 0; diff --git a/lib/crypto/c_src/pkey.c b/lib/crypto/c_src/pkey.c new file mode 100644 index 0000000000..a1e2677b34 --- /dev/null +++ b/lib/crypto/c_src/pkey.c @@ -0,0 +1,1486 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "pkey.h" +#include "bn.h" +#include "digest.h" +#include "dss.h" +#include "ec.h" +#include "eddsa.h" +#include "engine.h" +#include "rsa.h" + +#define PKEY_BADARG -1 +#define PKEY_NOTSUP 0 +#define PKEY_OK 1 + +typedef struct PKeyCryptOptions { + const EVP_MD *rsa_mgf1_md; + ErlNifBinary rsa_oaep_label; + const EVP_MD *rsa_oaep_md; + int rsa_padding; + const EVP_MD *signature_md; +} PKeyCryptOptions; + +typedef struct PKeySignOptions { + const EVP_MD *rsa_mgf1_md; + int rsa_padding; + int rsa_pss_saltlen; +} PKeySignOptions; + + +static int get_pkey_digest_type(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM type, + const EVP_MD **md); +static int get_pkey_sign_digest(ErlNifEnv *env, ERL_NIF_TERM algorithm, + ERL_NIF_TERM type, ERL_NIF_TERM data, + unsigned char *md_value, const EVP_MD **mdp, + unsigned char **tbsp, size_t *tbslenp); +static int get_pkey_sign_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM options, + const EVP_MD *md, PKeySignOptions *opt); +static int get_pkey_private_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM key, EVP_PKEY **pkey); +static int get_pkey_public_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM key, + EVP_PKEY **pkey); +static int get_pkey_crypt_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM options, + PKeyCryptOptions *opt); +static size_t size_of_RSA(EVP_PKEY *pkey); + + +static int get_pkey_digest_type(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM type, + const EVP_MD **md) +{ + struct digest_type_t *digp = NULL; + *md = NULL; + + if (type == atom_none && algorithm == atom_rsa) + return PKEY_OK; + if (algorithm == atom_eddsa) { +#ifdef HAVE_EDDSA + if (!FIPS_mode()) return PKEY_OK; +#else + return PKEY_NOTSUP; +#endif + } + if ((digp = get_digest_type(type)) == NULL) + return PKEY_BADARG; + if (digp->md.p == NULL) + return PKEY_NOTSUP; + + *md = digp->md.p; + return PKEY_OK; +} + +static int get_pkey_sign_digest(ErlNifEnv *env, ERL_NIF_TERM algorithm, + ERL_NIF_TERM type, ERL_NIF_TERM data, + unsigned char *md_value, const EVP_MD **mdp, + unsigned char **tbsp, size_t *tbslenp) +{ + int i, ret; + const ERL_NIF_TERM *tpl_terms; + int tpl_arity; + ErlNifBinary tbs_bin; + EVP_MD_CTX *mdctx = NULL; + const EVP_MD *md; + unsigned char *tbs; + size_t tbslen; + unsigned int tbsleni; + + md = *mdp; + tbs = *tbsp; + tbslen = *tbslenp; + + if ((i = get_pkey_digest_type(env, algorithm, type, &md)) != PKEY_OK) + return i; + + if (enif_get_tuple(env, data, &tpl_arity, &tpl_terms)) { + if (tpl_arity != 2) + goto bad_arg; + if (tpl_terms[0] != atom_digest) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, tpl_terms[1], &tbs_bin)) + goto bad_arg; + if (tbs_bin.size > INT_MAX) + goto bad_arg; + if (md != NULL) { + if ((int)tbs_bin.size != EVP_MD_size(md)) + goto bad_arg; + } + + /* We have a digest (= hashed text) in tbs_bin */ + tbs = tbs_bin.data; + tbslen = tbs_bin.size; + } else if (md == NULL) { + if (!enif_inspect_iolist_as_binary(env, data, &tbs_bin)) + goto bad_arg; + + /* md == NULL, that is no hashing because DigestType argument was atom_none */ + tbs = tbs_bin.data; + tbslen = tbs_bin.size; + } else { + if (!enif_inspect_iolist_as_binary(env, data, &tbs_bin)) + goto bad_arg; + + /* We have the cleartext in tbs_bin and the hash algo info in md */ + tbs = md_value; + + if ((mdctx = EVP_MD_CTX_create()) == NULL) + goto err; + + /* Looks well, now hash the plain text into a digest according to md */ + if (EVP_DigestInit_ex(mdctx, md, NULL) != 1) + goto err; + if (EVP_DigestUpdate(mdctx, tbs_bin.data, tbs_bin.size) != 1) + goto err; + if (EVP_DigestFinal_ex(mdctx, tbs, &tbsleni) != 1) + goto err; + + tbslen = (size_t)tbsleni; + } + + *mdp = md; + *tbsp = tbs; + *tbslenp = tbslen; + + ret = PKEY_OK; + goto done; + + bad_arg: + err: + ret = PKEY_BADARG; + + done: + if (mdctx) + EVP_MD_CTX_destroy(mdctx); + return ret; +} + +static int get_pkey_sign_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM options, + const EVP_MD *md, PKeySignOptions *opt) +{ + ERL_NIF_TERM head, tail; + const ERL_NIF_TERM *tpl_terms; + int tpl_arity; + const EVP_MD *opt_md; + + if (!enif_is_list(env, options)) + goto bad_arg; + + /* defaults */ + if (algorithm == atom_rsa) { + opt->rsa_mgf1_md = NULL; + opt->rsa_padding = RSA_PKCS1_PADDING; + opt->rsa_pss_saltlen = -2; + } + + if (enif_is_empty_list(env, options)) + return PKEY_OK; + + if (algorithm != atom_rsa) + goto bad_arg; + + tail = options; + while (enif_get_list_cell(env, tail, &head, &tail)) { + if (!enif_get_tuple(env, head, &tpl_arity, &tpl_terms)) + goto bad_arg; + if (tpl_arity != 2) + goto bad_arg; + + if (tpl_terms[0] == atom_rsa_mgf1_md && enif_is_atom(env, tpl_terms[1])) { + int result; + + result = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); + if (result != PKEY_OK) + return result; + + opt->rsa_mgf1_md = opt_md; + + } else if (tpl_terms[0] == atom_rsa_padding) { + if (tpl_terms[1] == atom_rsa_pkcs1_padding) { + opt->rsa_padding = RSA_PKCS1_PADDING; + + } else if (tpl_terms[1] == atom_rsa_pkcs1_pss_padding) { +#ifdef HAVE_RSA_PKCS1_PSS_PADDING + opt->rsa_padding = RSA_PKCS1_PSS_PADDING; + if (opt->rsa_mgf1_md == NULL) + opt->rsa_mgf1_md = md; +#else + return PKEY_NOTSUP; +#endif + + } else if (tpl_terms[1] == atom_rsa_x931_padding) { + opt->rsa_padding = RSA_X931_PADDING; + + } else if (tpl_terms[1] == atom_rsa_no_padding) { + opt->rsa_padding = RSA_NO_PADDING; + + } else { + goto bad_arg; + } + + } else if (tpl_terms[0] == atom_rsa_pss_saltlen) { + if (!enif_get_int(env, tpl_terms[1], &(opt->rsa_pss_saltlen))) + goto bad_arg; + if (opt->rsa_pss_saltlen < -2) + goto bad_arg; + + } else { + goto bad_arg; + } + } + + return PKEY_OK; + + bad_arg: + return PKEY_BADARG; +} + +static int get_pkey_private_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM key, EVP_PKEY **pkey) +{ + EVP_PKEY *result = NULL; + RSA *rsa = NULL; +#ifdef HAVE_DSA + DSA *dsa = NULL; +#endif +#if defined(HAVE_EC) + EC_KEY *ec = NULL; +#endif + char *id = NULL; + char *password = NULL; + + if (enif_is_map(env, key)) { +#ifdef HAS_ENGINE_SUPPORT + /* Use key stored in engine */ + ENGINE *e; + + if (!get_engine_and_key_id(env, key, &id, &e)) + goto err; + + password = get_key_password(env, key); + result = ENGINE_load_private_key(e, id, NULL, password); + +#else + return PKEY_BADARG; +#endif + } else if (algorithm == atom_rsa) { + if ((rsa = RSA_new()) == NULL) + goto err; + + if (!get_rsa_private_key(env, key, rsa)) + goto err; + if ((result = EVP_PKEY_new()) == NULL) + goto err; + if (EVP_PKEY_assign_RSA(result, rsa) != 1) + goto err; + /* On success, result owns rsa */ + rsa = NULL; + + } else if (algorithm == atom_ecdsa) { +#if defined(HAVE_EC) + const ERL_NIF_TERM *tpl_terms; + int tpl_arity; + + if (!enif_get_tuple(env, key, &tpl_arity, &tpl_terms)) + goto err; + if (tpl_arity != 2) + goto err; + if (!enif_is_tuple(env, tpl_terms[0])) + goto err; + if (!enif_is_binary(env, tpl_terms[1])) + goto err; + if (!get_ec_key(env, tpl_terms[0], tpl_terms[1], atom_undefined, &ec)) + goto err; + + if ((result = EVP_PKEY_new()) == NULL) + goto err; + if (EVP_PKEY_assign_EC_KEY(result, ec) != 1) + goto err; + /* On success, result owns ec */ + ec = NULL; + +#else + return PKEY_NOTSUP; +#endif + } else if (algorithm == atom_eddsa) { +#ifdef HAVE_EDDSA + if (!FIPS_mode()) + { + if (!get_eddsa_key(env, 0, key, &result)) + goto err; + else + goto done; // Not nice.... + } +#else + return PKEY_NOTSUP; +#endif + } else if (algorithm == atom_dss) { +#ifdef HAVE_DSA + if ((dsa = DSA_new()) == NULL) + goto err; + if (!get_dss_private_key(env, key, dsa)) + goto err; + + if ((result = EVP_PKEY_new()) == NULL) + goto err; + if (EVP_PKEY_assign_DSA(result, dsa) != 1) + goto err; + /* On success, result owns dsa */ + dsa = NULL; + + } else { +#endif + return PKEY_BADARG; + } + goto done; + + err: + if (result) + EVP_PKEY_free(result); + result = NULL; + + done: + if (password) + enif_free(password); + if (id) + enif_free(id); + if (rsa) + RSA_free(rsa); +#ifdef HAVE_DSA + if (dsa) + DSA_free(dsa); +#endif +#ifdef HAVE_EC + if (ec) + EC_KEY_free(ec); +#endif + + if (result == NULL) { + return PKEY_BADARG; + } else { + *pkey = result; + return PKEY_OK; + } +} + +static int get_pkey_public_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM key, + EVP_PKEY **pkey) +{ + EVP_PKEY *result = NULL; + RSA *rsa = NULL; +#ifdef HAVE_DSA + DSA *dsa = NULL; +#endif +#if defined(HAVE_EC) + EC_KEY *ec = NULL; +#endif + char *id = NULL; + char *password = NULL; + + if (enif_is_map(env, key)) { +#ifdef HAS_ENGINE_SUPPORT + /* Use key stored in engine */ + ENGINE *e; + + if (!get_engine_and_key_id(env, key, &id, &e)) + goto err; + + password = get_key_password(env, key); + result = ENGINE_load_public_key(e, id, NULL, password); + +#else + return PKEY_BADARG; +#endif + } else if (algorithm == atom_rsa) { + if ((rsa = RSA_new()) == NULL) + goto err; + + if (!get_rsa_public_key(env, key, rsa)) + goto err; + + if ((result = EVP_PKEY_new()) == NULL) + goto err; + if (EVP_PKEY_assign_RSA(result, rsa) != 1) + goto err; + /* On success, result owns rsa */ + rsa = NULL; + + } else if (algorithm == atom_ecdsa) { +#if defined(HAVE_EC) + const ERL_NIF_TERM *tpl_terms; + int tpl_arity; + + if (!enif_get_tuple(env, key, &tpl_arity, &tpl_terms)) + goto err; + if (tpl_arity != 2) + goto err; + if (!enif_is_tuple(env, tpl_terms[0])) + goto err; + if (!enif_is_binary(env, tpl_terms[1])) + goto err; + if (!get_ec_key(env, tpl_terms[0], atom_undefined, tpl_terms[1], &ec)) + goto err; + + if ((result = EVP_PKEY_new()) == NULL) + goto err; + + if (EVP_PKEY_assign_EC_KEY(result, ec) != 1) + goto err; + /* On success, result owns ec */ + ec = NULL; + +#else + return PKEY_NOTSUP; +#endif + } else if (algorithm == atom_eddsa) { +#ifdef HAVE_EDDSA + if (!FIPS_mode()) { + if (!get_eddsa_key(env, 1, key, &result)) + goto err; + } +#else + return PKEY_NOTSUP; +#endif + } else if (algorithm == atom_dss) { +#ifdef HAVE_DSA + if ((dsa = DSA_new()) == NULL) + goto err; + + if (!get_dss_public_key(env, key, dsa)) + goto err; + + if ((result = EVP_PKEY_new()) == NULL) + goto err; + if (EVP_PKEY_assign_DSA(result, dsa) != 1) + goto err; + /* On success, result owns dsa */ + dsa = NULL; +#else + return PKEY_NOTSUP; +#endif + } else { + return PKEY_BADARG; + } + + goto done; + + err: + if (result) + EVP_PKEY_free(result); + result = NULL; + + done: + if (password) + enif_free(password); + if (id) + enif_free(id); + if (rsa) + RSA_free(rsa); +#ifdef HAVE_DSA + if (dsa) + DSA_free(dsa); +#endif +#ifdef HAVE_EC + if (ec) + EC_KEY_free(ec); +#endif + + if (result == NULL) { + return PKEY_BADARG; + } else { + *pkey = result; + return PKEY_OK; + } +} + +ERL_NIF_TERM pkey_sign_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{/* (Algorithm, Type, Data|{digest,Digest}, Key|#{}, Options) */ + int i; + int sig_bin_alloc = 0; + ERL_NIF_TERM ret; + const EVP_MD *md = NULL; + unsigned char md_value[EVP_MAX_MD_SIZE]; + EVP_PKEY *pkey = NULL; +#ifdef HAVE_EDDSA + EVP_MD_CTX *mdctx = NULL; +#endif +#ifdef HAS_EVP_PKEY_CTX + EVP_PKEY_CTX *ctx = NULL; + size_t siglen; +#else + int len; + unsigned int siglen; +#endif + PKeySignOptions sig_opt; + ErlNifBinary sig_bin; /* signature */ + unsigned char *tbs; /* data to be signed */ + size_t tbslen; + RSA *rsa = NULL; +#ifdef HAVE_DSA + DSA *dsa = NULL; +#endif +#if defined(HAVE_EC) + EC_KEY *ec = NULL; +#endif +/*char buf[1024]; +enif_get_atom(env,argv[0],buf,1024,ERL_NIF_LATIN1); printf("algo=%s ",buf); +enif_get_atom(env,argv[1],buf,1024,ERL_NIF_LATIN1); printf("hash=%s ",buf); +*/ + +#ifndef HAS_ENGINE_SUPPORT + if (enif_is_map(env, argv[3])) + return atom_notsup; +#endif + + i = get_pkey_sign_digest(env, argv[0], argv[1], argv[2], md_value, &md, &tbs, &tbslen); + switch (i) { + case PKEY_OK: + break; + case PKEY_NOTSUP: + goto notsup; + default: + goto bad_arg; + } + + i = get_pkey_sign_options(env, argv[0], argv[4], md, &sig_opt); + switch (i) { + case PKEY_OK: + break; + case PKEY_NOTSUP: + goto notsup; + default: + goto bad_arg; + } + + if (get_pkey_private_key(env, argv[0], argv[3], &pkey) != PKEY_OK) + goto bad_arg; + +#ifdef HAS_EVP_PKEY_CTX + if ((ctx = EVP_PKEY_CTX_new(pkey, NULL)) == NULL) + goto err; + + if (argv[0] != atom_eddsa) { + if (EVP_PKEY_sign_init(ctx) != 1) + goto err; + if (md != NULL) { + if (EVP_PKEY_CTX_set_signature_md(ctx, md) != 1) + goto err; + } + } + + if (argv[0] == atom_rsa) { + if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) != 1) + goto err; +# ifdef HAVE_RSA_PKCS1_PSS_PADDING + if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) { + if (sig_opt.rsa_mgf1_md != NULL) { +# ifdef HAVE_RSA_MGF1_MD + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) != 1) + goto err; +# else + goto notsup; +# endif + } + if (sig_opt.rsa_pss_saltlen > -2) { + if (EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) != 1) + goto err; + } + } +#endif + } + + if (argv[0] == atom_eddsa) { +#ifdef HAVE_EDDSA + if (!FIPS_mode()) { + if ((mdctx = EVP_MD_CTX_new()) == NULL) + goto err; + + if (EVP_DigestSignInit(mdctx, NULL, NULL, NULL, pkey) != 1) + goto err; + if (EVP_DigestSign(mdctx, NULL, &siglen, tbs, tbslen) != 1) + goto err; + if (!enif_alloc_binary(siglen, &sig_bin)) + goto err; + sig_bin_alloc = 1; + + if (EVP_DigestSign(mdctx, sig_bin.data, &siglen, tbs, tbslen) != 1) + goto bad_key; + } + else +#endif + goto notsup; + } else { + if (EVP_PKEY_sign(ctx, NULL, &siglen, tbs, tbslen) != 1) + goto err; + if (!enif_alloc_binary(siglen, &sig_bin)) + goto err; + sig_bin_alloc = 1; + + if (md != NULL) { + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); + } + if (EVP_PKEY_sign(ctx, sig_bin.data, &siglen, tbs, tbslen) != 1) + goto bad_key; + } +#else +/*printf("Old interface\r\n"); + */ + if (argv[0] == atom_rsa) { + if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) + goto err; + if ((len = RSA_size(rsa)) < 0) + goto err; + if (!enif_alloc_binary((size_t)len, &sig_bin)) + goto err; + sig_bin_alloc = 1; + + if ((len = EVP_MD_size(md)) < 0) + goto err; + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); + + if (RSA_sign(md->type, tbs, (unsigned int)len, sig_bin.data, &siglen, rsa) != 1) + goto bad_key; + } else if (argv[0] == atom_dss) { + if ((dsa = EVP_PKEY_get1_DSA(pkey)) == NULL) + goto err; + if ((len = DSA_size(dsa)) < 0) + goto err; + if (!enif_alloc_binary((size_t)len, &sig_bin)) + goto err; + sig_bin_alloc = 1; + + if ((len = EVP_MD_size(md)) < 0) + goto err; + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); + + if (DSA_sign(md->type, tbs, len, sig_bin.data, &siglen, dsa) != 1) + goto bad_key; + } else if (argv[0] == atom_ecdsa) { +#if defined(HAVE_EC) + if ((ec = EVP_PKEY_get1_EC_KEY(pkey)) == NULL) + goto err; + if ((len = ECDSA_size(ec)) < 0) + goto err; + if (!enif_alloc_binary((size_t)len, &sig_bin)) + goto err; + sig_bin_alloc = 1; + + len = EVP_MD_size(md); + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); + + if (ECDSA_sign(md->type, tbs, len, sig_bin.data, &siglen, ec) != 1) + goto bad_key; +#else + goto notsup; +#endif + } else { + goto bad_arg; + } +#endif + + ERL_VALGRIND_MAKE_MEM_DEFINED(sig_bin.data, siglen); + if (siglen != sig_bin.size) { + if (!enif_realloc_binary(&sig_bin, siglen)) + goto err; + ERL_VALGRIND_ASSERT_MEM_DEFINED(sig_bin.data, siglen); + } + ret = enif_make_binary(env, &sig_bin); + sig_bin_alloc = 0; + goto done; + + bad_key: + ret = atom_error; + goto done; + + notsup: + ret = atom_notsup; + goto done; + + bad_arg: + err: + ret = enif_make_badarg(env); + goto done; + + done: + if (sig_bin_alloc) + enif_release_binary(&sig_bin); + if (rsa) + RSA_free(rsa); +#ifdef HAVE_DSA + if (dsa) + DSA_free(dsa); +#endif +#ifdef HAVE_EC + if (ec) + EC_KEY_free(ec); +#endif +#ifdef HAS_EVP_PKEY_CTX + if (ctx) + EVP_PKEY_CTX_free(ctx); +#endif + if (pkey) + EVP_PKEY_free(pkey); + +#ifdef HAVE_EDDSA + if (mdctx) + EVP_MD_CTX_free(mdctx); +#endif + + return ret; +} + +ERL_NIF_TERM pkey_verify_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{/* (Algorithm, Type, Data|{digest,Digest}, Signature, Key, Options) */ + int i; + int result; + const EVP_MD *md = NULL; + unsigned char md_value[EVP_MAX_MD_SIZE]; + EVP_PKEY *pkey = NULL; +#ifdef HAS_EVP_PKEY_CTX + EVP_PKEY_CTX *ctx = NULL; +#else +#endif + PKeySignOptions sig_opt; + ErlNifBinary sig_bin; /* signature */ + unsigned char *tbs; /* data to be signed */ + size_t tbslen; + ERL_NIF_TERM ret; + RSA *rsa = NULL; +#ifdef HAVE_DSA + DSA *dsa = NULL; +#endif +#ifdef HAVE_EC + EC_KEY *ec = NULL; +#endif +#ifdef HAVE_EDDSA + EVP_MD_CTX *mdctx = NULL; +#endif + +#ifndef HAS_ENGINE_SUPPORT + if (enif_is_map(env, argv[4])) + return atom_notsup; +#endif + + if (!enif_inspect_binary(env, argv[3], &sig_bin)) + return enif_make_badarg(env); + + i = get_pkey_sign_digest(env, argv[0], argv[1], argv[2], md_value, &md, &tbs, &tbslen); + switch (i) { + case PKEY_OK: + break; + case PKEY_NOTSUP: + goto notsup; + default: + goto bad_arg; + } + + i = get_pkey_sign_options(env, argv[0], argv[5], md, &sig_opt); + switch (i) { + case PKEY_OK: + break; + case PKEY_NOTSUP: + goto notsup; + default: + goto bad_arg; + } + + if (get_pkey_public_key(env, argv[0], argv[4], &pkey) != PKEY_OK) { + goto bad_arg; + } + +#ifdef HAS_EVP_PKEY_CTX +/* printf("EVP interface\r\n"); + */ + if ((ctx = EVP_PKEY_CTX_new(pkey, NULL)) == NULL) + goto err; + + if (argv[0] != atom_eddsa) { + if (EVP_PKEY_verify_init(ctx) != 1) + goto err; + if (md != NULL) { + if (EVP_PKEY_CTX_set_signature_md(ctx, md) != 1) + goto err; + } + } + + if (argv[0] == atom_rsa) { + if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) != 1) + goto err; + if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) { + if (sig_opt.rsa_mgf1_md != NULL) { +# ifdef HAVE_RSA_MGF1_MD + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) != 1) + goto err; +# else + goto notsup; +# endif + } + if (sig_opt.rsa_pss_saltlen > -2) { + if (EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) != 1) + goto err; + } + } + } + + if (argv[0] == atom_eddsa) { +#ifdef HAVE_EDDSA + if (!FIPS_mode()) { + if ((mdctx = EVP_MD_CTX_new()) == NULL) + goto err; + + if (EVP_DigestVerifyInit(mdctx, NULL, NULL, NULL, pkey) != 1) + goto err; + + result = EVP_DigestVerify(mdctx, sig_bin.data, sig_bin.size, tbs, tbslen); + } + else +#endif + goto notsup; + } else { + if (md != NULL) { + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); + } + result = EVP_PKEY_verify(ctx, sig_bin.data, sig_bin.size, tbs, tbslen); + } +#else +/*printf("Old interface\r\n"); +*/ + if (tbslen > INT_MAX) + goto bad_arg; + if (sig_bin.size > INT_MAX) + goto bad_arg; + if (argv[0] == atom_rsa) { + if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) + goto err; + result = RSA_verify(md->type, tbs, (unsigned int)tbslen, sig_bin.data, (unsigned int)sig_bin.size, rsa); + } else if (argv[0] == atom_dss) { + if ((dsa = EVP_PKEY_get1_DSA(pkey)) == NULL) + goto err; + result = DSA_verify(0, tbs, (int)tbslen, sig_bin.data, (int)sig_bin.size, dsa); + } else if (argv[0] == atom_ecdsa) { +#if defined(HAVE_EC) + if ((ec = EVP_PKEY_get1_EC_KEY(pkey)) == NULL) + goto err; + result = ECDSA_verify(EVP_MD_type(md), tbs, (int)tbslen, sig_bin.data, (int)sig_bin.size, ec); +#else + goto notsup; +#endif + } else { + goto bad_arg; + } +#endif + + ret = (result == 1 ? atom_true : atom_false); + goto done; + + bad_arg: + err: + ret = enif_make_badarg(env); + goto done; + + notsup: + ret = atom_notsup; + + done: +#ifdef HAS_EVP_PKEY_CTX + if (ctx) + EVP_PKEY_CTX_free(ctx); +#endif +#ifdef HAVE_EDDSA + if (mdctx) + EVP_MD_CTX_free(mdctx); +#endif + if (pkey) + EVP_PKEY_free(pkey); + if (rsa) + RSA_free(rsa); +#ifdef HAVE_DSA + if (dsa) + DSA_free(dsa); +#endif +#ifdef HAVE_EC + if (ec) + EC_KEY_free(ec); +#endif + + return ret; +} + +static int get_pkey_crypt_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM options, + PKeyCryptOptions *opt) +{ + ERL_NIF_TERM head, tail; + const ERL_NIF_TERM *tpl_terms; + int tpl_arity; + const EVP_MD *opt_md; + + if (!enif_is_list(env, options)) + goto bad_arg; + + /* defaults */ + if (algorithm == atom_rsa) { + opt->rsa_mgf1_md = NULL; + opt->rsa_oaep_label.data = NULL; + opt->rsa_oaep_label.size = 0; + opt->rsa_oaep_md = NULL; + opt->rsa_padding = RSA_PKCS1_PADDING; + opt->signature_md = NULL; + } + + if (enif_is_empty_list(env, options)) + return PKEY_OK; + + if (algorithm != atom_rsa) + goto bad_arg; + + tail = options; + while (enif_get_list_cell(env, tail, &head, &tail)) { + if (!enif_get_tuple(env, head, &tpl_arity, &tpl_terms)) + goto bad_arg; + if (tpl_arity != 2) + goto bad_arg; + + if (tpl_terms[0] == atom_rsa_padding + || tpl_terms[0] == atom_rsa_pad /* Compatibility */ + ) { + if (tpl_terms[1] == atom_rsa_pkcs1_padding) { + opt->rsa_padding = RSA_PKCS1_PADDING; + +#ifdef HAVE_RSA_OAEP_PADDING + } else if (tpl_terms[1] == atom_rsa_pkcs1_oaep_padding) { + opt->rsa_padding = RSA_PKCS1_OAEP_PADDING; +#endif + +#ifdef HAVE_RSA_SSLV23_PADDING + } else if (tpl_terms[1] == atom_rsa_sslv23_padding) { + opt->rsa_padding = RSA_SSLV23_PADDING; +#endif + + } else if (tpl_terms[1] == atom_rsa_x931_padding) { + opt->rsa_padding = RSA_X931_PADDING; + + } else if (tpl_terms[1] == atom_rsa_no_padding) { + opt->rsa_padding = RSA_NO_PADDING; + + } else { + goto bad_arg; + } + + } else if (tpl_terms[0] == atom_signature_md && enif_is_atom(env, tpl_terms[1])) { + int i; + i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); + if (i != PKEY_OK) { + return i; + } + opt->signature_md = opt_md; + + } else if (tpl_terms[0] == atom_rsa_mgf1_md && enif_is_atom(env, tpl_terms[1])) { + int i; +#ifndef HAVE_RSA_MGF1_MD + if (tpl_terms[1] != atom_sha) + return PKEY_NOTSUP; +#endif + i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); + if (i != PKEY_OK) { + return i; + } + opt->rsa_mgf1_md = opt_md; + + } else if (tpl_terms[0] == atom_rsa_oaep_label + && enif_inspect_binary(env, tpl_terms[1], &(opt->rsa_oaep_label))) { +#ifdef HAVE_RSA_OAEP_MD + continue; +#else + return PKEY_NOTSUP; +#endif + + } else if (tpl_terms[0] == atom_rsa_oaep_md && enif_is_atom(env, tpl_terms[1])) { + int i; +#ifndef HAVE_RSA_OAEP_MD + if (tpl_terms[1] != atom_sha) + return PKEY_NOTSUP; +#endif + i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); + if (i != PKEY_OK) { + return i; + } + opt->rsa_oaep_md = opt_md; + + } else { + goto bad_arg; + } + } + + return PKEY_OK; + + bad_arg: + return PKEY_BADARG; +} + +static size_t size_of_RSA(EVP_PKEY *pkey) { + int ret = 0; + RSA *rsa = NULL; + + if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) + goto err; + ret = RSA_size(rsa); + + err: + if (rsa) + RSA_free(rsa); + + return (ret < 0) ? 0 : (size_t)ret; +} + +ERL_NIF_TERM pkey_crypt_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{/* (Algorithm, Data, PublKey=[E,N]|[E,N,D]|[E,N,D,P1,P2,E1,E2,C], Options, IsPrivate, IsEncrypt) */ + ERL_NIF_TERM ret; + int i; + int result = 0; + int tmp_bin_alloc = 0; + int out_bin_alloc = 0; + EVP_PKEY *pkey = NULL; +#ifdef HAS_EVP_PKEY_CTX + EVP_PKEY_CTX *ctx = NULL; +#else + int len; + RSA *rsa = NULL; +#endif + PKeyCryptOptions crypt_opt; + ErlNifBinary in_bin, out_bin, tmp_bin; + size_t outlen; +#ifdef HAVE_RSA_SSLV23_PADDING + size_t tmplen; +#endif + int is_private, is_encrypt; + int algo_init = 0; + unsigned char *label_copy = NULL; + + ASSERT(argc == 6); + + is_private = (argv[4] == atom_true); + is_encrypt = (argv[5] == atom_true); + +/* char algo[1024]; */ + +#ifndef HAS_ENGINE_SUPPORT + if (enif_is_map(env, argv[2])) + return atom_notsup; +#endif + + if (!enif_inspect_binary(env, argv[1], &in_bin)) + goto bad_arg; + + i = get_pkey_crypt_options(env, argv[0], argv[3], &crypt_opt); + switch (i) { + case PKEY_OK: + break; + case PKEY_NOTSUP: + goto notsup; + default: + goto bad_arg; + } + + if (is_private) { + if (get_pkey_private_key(env, argv[0], argv[2], &pkey) != PKEY_OK) + goto bad_arg; + } else { + if (get_pkey_public_key(env, argv[0], argv[2], &pkey) != PKEY_OK) + goto bad_arg; + } + +#ifdef HAS_EVP_PKEY_CTX + if ((ctx = EVP_PKEY_CTX_new(pkey, NULL)) == NULL) + goto err; + +/* enif_get_atom(env,argv[0],algo,1024,ERL_NIF_LATIN1); */ + + if (is_private) { + if (is_encrypt) { + /* private encrypt */ + if ((algo_init = EVP_PKEY_sign_init(ctx)) != 1) + goto bad_arg; + } else { + /* private decrypt */ + if ((algo_init = EVP_PKEY_decrypt_init(ctx)) != 1) + goto bad_arg; + } + } else { + if (is_encrypt) { + /* public encrypt */ + if ((algo_init = EVP_PKEY_encrypt_init(ctx)) != 1) + goto bad_arg; + } else { + /* public decrypt */ + if ((algo_init = EVP_PKEY_verify_recover_init(ctx)) != 1) + goto bad_arg; + } + } + + if (argv[0] == atom_rsa) { + if (crypt_opt.signature_md != NULL) { + if (EVP_PKEY_CTX_set_signature_md(ctx, crypt_opt.signature_md) != 1) + goto bad_arg; + } + +#ifdef HAVE_RSA_SSLV23_PADDING + if (crypt_opt.rsa_padding == RSA_SSLV23_PADDING) { + if (is_encrypt) { + tmplen = size_of_RSA(pkey); + if (tmplen < 1 || tmplen > INT_MAX) + goto err; + if (!enif_alloc_binary(tmplen, &tmp_bin)) + goto err; + tmp_bin_alloc = 1; + if (in_bin.size > INT_MAX) + goto err; + if (!RSA_padding_add_SSLv23(tmp_bin.data, (int)tmplen, in_bin.data, (int)in_bin.size)) + goto err; + in_bin = tmp_bin; + } + if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_NO_PADDING) != 1) + goto err; + } else +#endif + { + if (EVP_PKEY_CTX_set_rsa_padding(ctx, crypt_opt.rsa_padding) != 1) + goto err; + } + +#ifdef HAVE_RSA_OAEP_MD + if (crypt_opt.rsa_padding == RSA_PKCS1_OAEP_PADDING) { + if (crypt_opt.rsa_oaep_md != NULL) { + if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, crypt_opt.rsa_oaep_md) != 1) + goto err; + } + + if (crypt_opt.rsa_mgf1_md != NULL) { + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, crypt_opt.rsa_mgf1_md) != 1) + goto err; + } + + if (crypt_opt.rsa_oaep_label.data != NULL && crypt_opt.rsa_oaep_label.size > 0) { + if (crypt_opt.rsa_oaep_label.size > INT_MAX) + goto err; + if ((label_copy = OPENSSL_malloc(crypt_opt.rsa_oaep_label.size)) == NULL) + goto err; + + memcpy((void *)(label_copy), (const void *)(crypt_opt.rsa_oaep_label.data), + crypt_opt.rsa_oaep_label.size); + + if (EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, label_copy, + (int)crypt_opt.rsa_oaep_label.size) != 1) + goto err; + /* On success, label_copy is owned by ctx */ + label_copy = NULL; + } + } +#endif + } + + if (is_private) { + if (is_encrypt) { + /* private_encrypt */ + result = EVP_PKEY_sign(ctx, NULL, &outlen, in_bin.data, in_bin.size); + } else { + /* private_decrypt */ + result = EVP_PKEY_decrypt(ctx, NULL, &outlen, in_bin.data, in_bin.size); + } + } else { + if (is_encrypt) { + /* public_encrypt */ + result = EVP_PKEY_encrypt(ctx, NULL, &outlen, in_bin.data, in_bin.size); + } else { + /* public_decrypt */ + result = EVP_PKEY_verify_recover(ctx, NULL, &outlen, in_bin.data, in_bin.size); + } + } + /* fprintf(stderr,"i = %d %s:%d\r\n", i, __FILE__, __LINE__); */ + + if (result != 1) + goto err; + + if (!enif_alloc_binary(outlen, &out_bin)) + goto err; + out_bin_alloc = 1; + + if (is_private) { + if (is_encrypt) { + /* private_encrypt */ + result = EVP_PKEY_sign(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); + } else { + /* private_decrypt */ + result = EVP_PKEY_decrypt(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); + } + } else { + if (is_encrypt) { + /* public_encrypt */ + result = EVP_PKEY_encrypt(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); + } else { + /* public_decrypt */ + result = EVP_PKEY_verify_recover(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); + } + } + +#else + /* Non-EVP cryptolib. Only support RSA */ + + if (argv[0] != atom_rsa) { + algo_init = -2; /* exitcode: notsup */ + goto bad_arg; + } + + if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) + goto err; + if ((len = RSA_size(rsa)) < 0) + goto err; + if (!enif_alloc_binary((size_t)len, &out_bin)) + goto err; + out_bin_alloc = 1; + + if (in_bin.size > INT_MAX) + goto err; + if (is_private) { + if (is_encrypt) { + /* non-evp rsa private encrypt */ + ERL_VALGRIND_ASSERT_MEM_DEFINED(in_bin.data,in_bin.size); + result = RSA_private_encrypt((int)in_bin.size, in_bin.data, + out_bin.data, rsa, crypt_opt.rsa_padding); + if (result > 0) { + ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, result); + } + } else { + /* non-evp rsa private decrypt */ + result = RSA_private_decrypt((int)in_bin.size, in_bin.data, + out_bin.data, rsa, crypt_opt.rsa_padding); + if (result > 0) { + ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, result); + if (!enif_realloc_binary(&out_bin, (size_t)result)) + goto err; + } + } + } else { + if (is_encrypt) { + /* non-evp rsa public encrypt */ + ERL_VALGRIND_ASSERT_MEM_DEFINED(in_bin.data,in_bin.size); + result = RSA_public_encrypt((int)in_bin.size, in_bin.data, + out_bin.data, rsa, crypt_opt.rsa_padding); + if (result > 0) { + ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, result); + } + } else { + /* non-evp rsa public decrypt */ + result = RSA_public_decrypt((int)in_bin.size, in_bin.data, + out_bin.data, rsa, crypt_opt.rsa_padding); + if (result > 0) { + ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, result); + if (!enif_realloc_binary(&out_bin, (size_t)result)) + goto err; + } + } + } + + outlen = (size_t)result; +#endif + + if ((result > 0) && argv[0] == atom_rsa && !is_encrypt) { +#ifdef HAVE_RSA_SSLV23_PADDING + if (crypt_opt.rsa_padding == RSA_SSLV23_PADDING) { + unsigned char *p; + + tmplen = size_of_RSA(pkey); + if (tmplen < 1 || tmplen > INT_MAX) + goto err; + if (!enif_alloc_binary(tmplen, &tmp_bin)) + goto err; + tmp_bin_alloc = 1; + if (out_bin.size > INT_MAX) + goto err; + + p = out_bin.data; + p++; + + result = RSA_padding_check_SSLv23(tmp_bin.data, (int)tmplen, p, (int)out_bin.size - 1, (int)tmplen); + if (result >= 0) { + outlen = (size_t)result; + in_bin = out_bin; + out_bin = tmp_bin; + tmp_bin = in_bin; + result = 1; + } + } +#endif + } + + if (result > 0) { + ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, outlen); + if (outlen != out_bin.size) { + if (!enif_realloc_binary(&out_bin, outlen)) + goto err; + ERL_VALGRIND_ASSERT_MEM_DEFINED(out_bin.data, outlen); + } + ret = enif_make_binary(env, &out_bin); + out_bin_alloc = 0; + } else { + ret = atom_error; + } + goto done; + + notsup: + ret = atom_notsup; + goto done; + + bad_arg: + err: + if (algo_init == -2) + ret = atom_notsup; + else + ret = enif_make_badarg(env); + + done: + if (out_bin_alloc) + enif_release_binary(&out_bin); + if (tmp_bin_alloc) + enif_release_binary(&tmp_bin); + +#ifdef HAS_EVP_PKEY_CTX + if (ctx) + EVP_PKEY_CTX_free(ctx); +#else + if (rsa) + RSA_free(rsa); +#endif + if (pkey) + EVP_PKEY_free(pkey); + + if (label_copy) + OPENSSL_free(label_copy); + + return ret; +} + +ERL_NIF_TERM privkey_to_pubkey_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ /* (Algorithm, PrivKey | KeyMap) */ + ERL_NIF_TERM ret; + EVP_PKEY *pkey = NULL; + RSA *rsa = NULL; +#ifdef HAVE_DSA + DSA *dsa = NULL; +#endif + ERL_NIF_TERM result[8]; + + ASSERT(argc == 2); + + if (get_pkey_private_key(env, argv[0], argv[1], &pkey) != PKEY_OK) + goto bad_arg; + + if (argv[0] == atom_rsa) { + const BIGNUM *n = NULL, *e = NULL, *d = NULL; + + if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) + goto err; + + RSA_get0_key(rsa, &n, &e, &d); + + // Exponent E + if ((result[0] = bin_from_bn(env, e)) == atom_error) + goto err; + // Modulus N = p*q + if ((result[1] = bin_from_bn(env, n)) == atom_error) + goto err; + + ret = enif_make_list_from_array(env, result, 2); + +#ifdef HAVE_DSA + } else if (argv[0] == atom_dss) { + const BIGNUM *p = NULL, *q = NULL, *g = NULL, *pub_key = NULL; + + if ((dsa = EVP_PKEY_get1_DSA(pkey)) == NULL) + goto err; + + DSA_get0_pqg(dsa, &p, &q, &g); + DSA_get0_key(dsa, &pub_key, NULL); + + if ((result[0] = bin_from_bn(env, p)) == atom_error) + goto err; + if ((result[1] = bin_from_bn(env, q)) == atom_error) + goto err; + if ((result[2] = bin_from_bn(env, g)) == atom_error) + goto err; + if ((result[3] = bin_from_bn(env, pub_key)) == atom_error) + goto err; + + ret = enif_make_list_from_array(env, result, 4); +#endif + } else if (argv[0] == atom_ecdsa) { +#if defined(HAVE_EC) + /* not yet implemented + EC_KEY *ec = EVP_PKEY_get1_EC_KEY(pkey); + if (ec) { + / * Example of result: + { + Curve = {Field, Prime, Point, Order, CoFactor} = + { + Field = {prime_field,<<255,...,255>>}, + Prime = {<<255,...,252>>, + <<90,...,75>>, + <<196,...,144>> + }, + Point = <<4,...,245>>, + Order = <<255,...,81>>, + CoFactor = <<1>> + }, + Key = <<151,...,62>> + } + or + { + Curve = + {characteristic_two_field, + M, + Basis = {tpbasis, _} + | {ppbasis, k1, k2, k3} + }, + Key + } + * / + EVP_PKEY_free(pkey); + return enif_make_list_from_array(env, ..., ...); + */ +#endif + goto bad_arg; + } else { + goto bad_arg; + } + + goto done; + + bad_arg: + err: + ret = enif_make_badarg(env); + + done: + if (rsa) + RSA_free(rsa); +#ifdef HAVE_DSA + if (dsa) + DSA_free(dsa); +#endif + if (pkey) + EVP_PKEY_free(pkey); + + return ret; +} diff --git a/lib/crypto/c_src/pkey.h b/lib/crypto/c_src/pkey.h new file mode 100644 index 0000000000..f647a4a160 --- /dev/null +++ b/lib/crypto/c_src/pkey.h @@ -0,0 +1,31 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_PKEY_H__ +#define E_PKEY_H__ 1 + +#include "common.h" + +ERL_NIF_TERM pkey_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM pkey_verify_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM pkey_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM privkey_to_pubkey_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_PKEY_H__ */ diff --git a/lib/crypto/c_src/poly1305.c b/lib/crypto/c_src/poly1305.c new file mode 100644 index 0000000000..76579c0a29 --- /dev/null +++ b/lib/crypto/c_src/poly1305.c @@ -0,0 +1,90 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "poly1305.h" + +/* For OpenSSL >= 1.1.1 the hmac_nif and cmac_nif could be integrated into poly1305 (with 'type' as parameter) */ +ERL_NIF_TERM poly1305_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Key, Text) */ +#ifdef HAVE_POLY1305 + ErlNifBinary key_bin, text, ret_bin; + ERL_NIF_TERM ret; + EVP_PKEY *key = NULL; + EVP_MD_CTX *mctx = NULL; + EVP_PKEY_CTX *pctx = NULL; + const EVP_MD *md = NULL; + size_t size; + int ret_bin_alloc = 0; + + ASSERT(argc == 2); + + if (!enif_inspect_binary(env, argv[0], &key_bin)) + goto bad_arg; + if (key_bin.size != 32) + goto bad_arg; + if (!enif_inspect_binary(env, argv[1], &text)) + goto bad_arg; + + if ((key = EVP_PKEY_new_raw_private_key(EVP_PKEY_POLY1305, /*engine*/ NULL, key_bin.data, key_bin.size)) == NULL) + goto err; + + if ((mctx = EVP_MD_CTX_new()) == NULL) + goto err; + if (EVP_DigestSignInit(mctx, &pctx, md, /*engine*/ NULL, key) != 1) + goto err; + if (EVP_DigestSignUpdate(mctx, text.data, text.size) != 1) + goto err; + + if (EVP_DigestSignFinal(mctx, NULL, &size) != 1) + goto err; + if (!enif_alloc_binary(size, &ret_bin)) + goto err; + ret_bin_alloc = 1; + if (EVP_DigestSignFinal(mctx, ret_bin.data, &size) != 1) + goto err; + + if (size != ret_bin.size) { + if (!enif_realloc_binary(&ret_bin, size)) + goto err; + } + + ret = enif_make_binary(env, &ret_bin); + ret_bin_alloc = 0; + goto done; + + bad_arg: + return enif_make_badarg(env); + + err: + if (ret_bin_alloc) + enif_release_binary(&ret_bin); + ret = atom_error; + + done: + if (mctx) + EVP_MD_CTX_free(mctx); + if (key) + EVP_PKEY_free(key); + return ret; + +#else + return enif_raise_exception(env, atom_notsup); +#endif +} diff --git a/lib/crypto/c_src/poly1305.h b/lib/crypto/c_src/poly1305.h new file mode 100644 index 0000000000..4bf45e6218 --- /dev/null +++ b/lib/crypto/c_src/poly1305.h @@ -0,0 +1,28 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_POLY1305_H__ +#define E_POLY1305_H__ 1 + +#include "common.h" + +ERL_NIF_TERM poly1305_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_POLY1305_H__ */ diff --git a/lib/crypto/c_src/rand.c b/lib/crypto/c_src/rand.c new file mode 100644 index 0000000000..3812ae0991 --- /dev/null +++ b/lib/crypto/c_src/rand.c @@ -0,0 +1,149 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "rand.h" +#include "bn.h" + +ERL_NIF_TERM strong_rand_bytes_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Bytes) */ + unsigned bytes; + unsigned char* data; + ERL_NIF_TERM ret; + + ASSERT(argc == 1); + + if (!enif_get_uint(env, argv[0], &bytes)) + goto bad_arg; + if (bytes > INT_MAX) + goto bad_arg; + + if ((data = enif_make_new_binary(env, bytes, &ret)) == NULL) + goto err; + if (RAND_bytes(data, (int)bytes) != 1) + goto err; + + ERL_VALGRIND_MAKE_MEM_DEFINED(data, bytes); + return ret; + + bad_arg: + return enif_make_badarg(env); + + err: + return atom_false; +} + +ERL_NIF_TERM strong_rand_range_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Range) */ + BIGNUM *bn_range = NULL, *bn_rand = NULL; + ERL_NIF_TERM ret; + + ASSERT(argc == 1); + + if (!get_bn_from_bin(env, argv[0], &bn_range)) + goto bad_arg; + + if ((bn_rand = BN_new()) == NULL) + goto err; + if (!BN_rand_range(bn_rand, bn_range)) + goto err; + + if ((ret = bin_from_bn(env, bn_rand)) == atom_error) + goto err; + goto done; + + bad_arg: + return enif_make_badarg(env); + + err: + ret = atom_false; + + done: + if (bn_rand) + BN_free(bn_rand); + if (bn_range) + BN_free(bn_range); + return ret; +} + +ERL_NIF_TERM rand_uniform_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Lo,Hi) */ + BIGNUM *bn_from = NULL, *bn_to = NULL, *bn_rand = NULL; + unsigned char* data; + int dlen; + ERL_NIF_TERM ret; + + ASSERT(argc == 2); + + if (!get_bn_from_mpint(env, argv[0], &bn_from)) + goto bad_arg; + if (!get_bn_from_mpint(env, argv[1], &bn_rand)) + goto bad_arg; + + if ((bn_to = BN_new()) == NULL) + goto err; + + if (!BN_sub(bn_to, bn_rand, bn_from)) + goto err; + if (!BN_pseudo_rand_range(bn_rand, bn_to)) + goto err; + if (!BN_add(bn_rand, bn_rand, bn_from)) + goto err; + + if ((dlen = BN_num_bytes(bn_rand)) < 0) + goto err; + if ((data = enif_make_new_binary(env, (size_t)dlen+4, &ret)) == NULL) + goto err; + + put_uint32(data, (unsigned int)dlen); + BN_bn2bin(bn_rand, data+4); + ERL_VALGRIND_MAKE_MEM_DEFINED(data+4, dlen); + goto done; + + bad_arg: + err: + ret = enif_make_badarg(env); + + done: + if (bn_rand) + BN_free(bn_rand); + if (bn_from) + BN_free(bn_from); + if (bn_to) + BN_free(bn_to); + return ret; +} + +ERL_NIF_TERM rand_seed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Seed) */ + ErlNifBinary seed_bin; + + ASSERT(argc == 1); + + if (!enif_inspect_binary(env, argv[0], &seed_bin)) + goto bad_arg; + if (seed_bin.size > INT_MAX) + goto bad_arg; + + RAND_seed(seed_bin.data, (int)seed_bin.size); + return atom_ok; + + bad_arg: + return enif_make_badarg(env); +} diff --git a/lib/crypto/c_src/rand.h b/lib/crypto/c_src/rand.h new file mode 100644 index 0000000000..9c23d343ec --- /dev/null +++ b/lib/crypto/c_src/rand.h @@ -0,0 +1,31 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_RAND_H__ +#define E_RAND_H__ 1 + +#include "common.h" + +ERL_NIF_TERM strong_rand_bytes_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM strong_rand_range_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM rand_uniform_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM rand_seed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_RAND_H__ */ diff --git a/lib/crypto/c_src/rsa.c b/lib/crypto/c_src/rsa.c new file mode 100644 index 0000000000..e9f29aa496 --- /dev/null +++ b/lib/crypto/c_src/rsa.c @@ -0,0 +1,282 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "rsa.h" +#include "bn.h" + +static ERL_NIF_TERM rsa_generate_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM put_rsa_private_key(ErlNifEnv* env, const RSA *rsa); +static int check_erlang_interrupt(int maj, int min, BN_GENCB *ctxt); + +int get_rsa_private_key(ErlNifEnv* env, ERL_NIF_TERM key, RSA *rsa) +{ + /* key=[E,N,D]|[E,N,D,P1,P2,E1,E2,C] */ + ERL_NIF_TERM head, tail; + BIGNUM *e = NULL, *n = NULL, *d = NULL; + BIGNUM *p = NULL, *q = NULL; + BIGNUM *dmp1 = NULL, *dmq1 = NULL, *iqmp = NULL; + + if (!enif_get_list_cell(env, key, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &e)) + goto bad_arg; + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &n)) + goto bad_arg; + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &d)) + goto bad_arg; + + if (!RSA_set0_key(rsa, n, e, d)) + goto err; + /* rsa now owns n, e, and d */ + n = NULL; + e = NULL; + d = NULL; + + if (enif_is_empty_list(env, tail)) + return 1; + + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &p)) + goto bad_arg; + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &q)) + goto bad_arg; + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &dmp1)) + goto bad_arg; + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &dmq1)) + goto bad_arg; + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &iqmp)) + goto bad_arg; + if (!enif_is_empty_list(env, tail)) + goto bad_arg; + + if (!RSA_set0_factors(rsa, p, q)) + goto err; + /* rsa now owns p and q */ + p = NULL; + q = NULL; + + if (!RSA_set0_crt_params(rsa, dmp1, dmq1, iqmp)) + goto err; + /* rsa now owns dmp1, dmq1, and iqmp */ + dmp1 = NULL; + dmq1 = NULL; + iqmp = NULL; + + return 1; + + bad_arg: + err: + if (e) + BN_free(e); + if (n) + BN_free(n); + if (d) + BN_free(d); + if (p) + BN_free(p); + if (q) + BN_free(q); + if (dmp1) + BN_free(dmp1); + if (dmq1) + BN_free(dmq1); + if (iqmp) + BN_free(iqmp); + + return 0; +} + +int get_rsa_public_key(ErlNifEnv* env, ERL_NIF_TERM key, RSA *rsa) +{ + /* key=[E,N] */ + ERL_NIF_TERM head, tail; + BIGNUM *e = NULL, *n = NULL; + + if (!enif_get_list_cell(env, key, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &e)) + goto bad_arg; + if (!enif_get_list_cell(env, tail, &head, &tail)) + goto bad_arg; + if (!get_bn_from_bin(env, head, &n)) + goto bad_arg; + if (!enif_is_empty_list(env, tail)) + goto bad_arg; + + if (!RSA_set0_key(rsa, n, e, NULL)) + goto err; + /* rsa now owns n and e */ + n = NULL; + e = NULL; + + return 1; + + bad_arg: + err: + if (e) + BN_free(e); + if (n) + BN_free(n); + + return 0; +} + +/* Creates a term which can be parsed by get_rsa_private_key(). This is a list of plain integer binaries (not mpints). */ +static ERL_NIF_TERM put_rsa_private_key(ErlNifEnv* env, const RSA *rsa) +{ + ERL_NIF_TERM result[8]; + const BIGNUM *n = NULL, *e = NULL, *d = NULL, *p = NULL, *q = NULL, *dmp1 = NULL, *dmq1 = NULL, *iqmp = NULL; + + /* Return at least [E,N,D] */ + RSA_get0_key(rsa, &n, &e, &d); + + if ((result[0] = bin_from_bn(env, e)) == atom_error) // Exponent E + goto err; + if ((result[1] = bin_from_bn(env, n)) == atom_error) // Modulus N = p*q + goto err; + if ((result[2] = bin_from_bn(env, d)) == atom_error) // Exponent D + goto err; + + /* Check whether the optional additional parameters are available */ + RSA_get0_factors(rsa, &p, &q); + RSA_get0_crt_params(rsa, &dmp1, &dmq1, &iqmp); + + if (p && q && dmp1 && dmq1 && iqmp) { + if ((result[3] = bin_from_bn(env, p)) == atom_error) // Factor p + goto err; + if ((result[4] = bin_from_bn(env, q)) == atom_error) // Factor q + goto err; + if ((result[5] = bin_from_bn(env, dmp1)) == atom_error) // D mod (p-1) + goto err; + if ((result[6] = bin_from_bn(env, dmq1)) == atom_error) // D mod (q-1) + goto err; + if ((result[7] = bin_from_bn(env, iqmp)) == atom_error) // (1/q) mod p + goto err; + + return enif_make_list_from_array(env, result, 8); + } else { + return enif_make_list_from_array(env, result, 3); + } + + err: + return enif_make_badarg(env); +} + +static int check_erlang_interrupt(int maj, int min, BN_GENCB *ctxt) +{ + ErlNifEnv *env = BN_GENCB_get_arg(ctxt); + + if (!enif_is_current_process_alive(env)) { + return 0; + } else { + return 1; + } +} + +static ERL_NIF_TERM rsa_generate_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (ModulusSize, PublicExponent) */ + ERL_NIF_TERM ret; + int modulus_bits; + BIGNUM *pub_exp = NULL, *three = NULL; + RSA *rsa = NULL; + BN_GENCB *intr_cb = NULL; +#ifndef HAVE_OPAQUE_BN_GENCB + BN_GENCB intr_cb_buf; +#endif + + ASSERT(argc == 2); + + if (!enif_get_int(env, argv[0], &modulus_bits)) + goto bad_arg; + if (modulus_bits < 256) + goto bad_arg; + if (!get_bn_from_bin(env, argv[1], &pub_exp)) + goto bad_arg; + + /* Make sure the public exponent is large enough (at least 3). + * Without this, RSA_generate_key_ex() can run forever. */ + if ((three = BN_new()) == NULL) + goto err; + if (!BN_set_word(three, 3)) + goto err; + if (BN_cmp(pub_exp, three) < 0) + goto err; + + /* For large keys, prime generation can take many seconds. Set up + * the callback which we use to test whether the process has been + * interrupted. */ +#ifdef HAVE_OPAQUE_BN_GENCB + if ((intr_cb = BN_GENCB_new()) == NULL) + goto err; +#else + intr_cb = &intr_cb_buf; +#endif + BN_GENCB_set(intr_cb, check_erlang_interrupt, env); + + if ((rsa = RSA_new()) == NULL) + goto err; + + if (!RSA_generate_key_ex(rsa, modulus_bits, pub_exp, intr_cb)) + goto err; + + ret = put_rsa_private_key(env, rsa); + goto done; + + bad_arg: + return enif_make_badarg(env); + + err: + ret = atom_error; + + done: + if (pub_exp) + BN_free(pub_exp); + if (three) + BN_free(three); +#ifdef HAVE_OPAQUE_BN_GENCB + if (intr_cb) + BN_GENCB_free(intr_cb); +#endif + if (rsa) + RSA_free(rsa); + return ret; +} + +ERL_NIF_TERM rsa_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + /* RSA key generation can take a long time (>1 sec for a large + * modulus), so schedule it as a CPU-bound operation. */ + return enif_schedule_nif(env, "rsa_generate_key", + ERL_NIF_DIRTY_JOB_CPU_BOUND, + rsa_generate_key, argc, argv); +} diff --git a/lib/crypto/c_src/rsa.h b/lib/crypto/c_src/rsa.h new file mode 100644 index 0000000000..69c02aa2cb --- /dev/null +++ b/lib/crypto/c_src/rsa.h @@ -0,0 +1,31 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_RSA_H__ +#define E_RSA_H__ 1 + +#include "common.h" + +int get_rsa_public_key(ErlNifEnv* env, ERL_NIF_TERM key, RSA *rsa); +int get_rsa_private_key(ErlNifEnv* env, ERL_NIF_TERM key, RSA *rsa); + +ERL_NIF_TERM rsa_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_RSA_H__ */ diff --git a/lib/crypto/c_src/srp.c b/lib/crypto/c_src/srp.c new file mode 100644 index 0000000000..2979048006 --- /dev/null +++ b/lib/crypto/c_src/srp.c @@ -0,0 +1,307 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "srp.h" +#include "bn.h" + +ERL_NIF_TERM srp_value_B_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Multiplier, Verifier, Generator, Exponent, Prime) */ + BIGNUM *bn_verifier = NULL; + BIGNUM *bn_exponent = NULL, *bn_generator = NULL, *bn_prime = NULL, *bn_multiplier = NULL, *bn_result = NULL; + BN_CTX *bn_ctx = NULL; + unsigned char* ptr; + int dlen; + ERL_NIF_TERM ret; + + CHECK_NO_FIPS_MODE(); + + ASSERT(argc == 5); + + if (!get_bn_from_bin(env, argv[0], &bn_multiplier)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[1], &bn_verifier)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[2], &bn_generator)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[3], &bn_exponent)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[4], &bn_prime)) + goto bad_arg; + + if ((bn_result = BN_new()) == NULL) + goto err; + if ((bn_ctx = BN_CTX_new()) == NULL) + goto err; + + /* B = k*v + g^b % N */ + + /* k * v */ + if (!BN_mod_mul(bn_multiplier, bn_multiplier, bn_verifier, bn_prime, bn_ctx)) + goto err; + + /* g^b % N */ + if (!BN_mod_exp(bn_result, bn_generator, bn_exponent, bn_prime, bn_ctx)) + goto err; + + /* k*v + g^b % N */ + if (!BN_mod_add(bn_result, bn_result, bn_multiplier, bn_prime, bn_ctx)) + goto err; + + /* check that B % N != 0, reuse bn_multiplier */ + if (!BN_nnmod(bn_multiplier, bn_result, bn_prime, bn_ctx)) + goto err; + + if (BN_is_zero(bn_multiplier)) + goto err; + + if ((dlen = BN_num_bytes(bn_result)) < 0) + goto err; + if ((ptr = enif_make_new_binary(env, (size_t)dlen, &ret)) == NULL) + goto err; + + if (BN_bn2bin(bn_result, ptr) < 0) + goto err; + + goto done; + + bad_arg: + ret = enif_make_badarg(env); + goto done; + + err: + ret = atom_error; + + done: + if (bn_multiplier) + BN_free(bn_multiplier); + if (bn_verifier) + BN_free(bn_verifier); + if (bn_generator) + BN_free(bn_generator); + if (bn_exponent) + BN_free(bn_exponent); + if (bn_prime) + BN_free(bn_prime); + if (bn_result) + BN_free(bn_result); + if (bn_ctx) + BN_CTX_free(bn_ctx); + + return ret; +} + +ERL_NIF_TERM srp_user_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (a, u, B, Multiplier, Prime, Exponent, Generator) */ +/* + <premaster secret> = (B - (k * g^x)) ^ (a + (u * x)) % N +*/ + BIGNUM *bn_exponent = NULL, *bn_a = NULL; + BIGNUM *bn_u = NULL, *bn_multiplier = NULL, *bn_exp2 = NULL; + BIGNUM *bn_base = NULL, *bn_prime = NULL, *bn_generator = NULL; + BIGNUM *bn_B = NULL, *bn_result = NULL; + BN_CTX *bn_ctx = NULL; + unsigned char *ptr; + int dlen; + ERL_NIF_TERM ret; + + CHECK_NO_FIPS_MODE(); + + ASSERT(argc == 7); + + if (!get_bn_from_bin(env, argv[0], &bn_a)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[1], &bn_u)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[2], &bn_B)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[3], &bn_multiplier)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[4], &bn_generator)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[5], &bn_exponent)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[6], &bn_prime)) + goto bad_arg; + + if ((bn_ctx = BN_CTX_new()) == NULL) + goto err; + if ((bn_result = BN_new()) == NULL) + goto err; + + /* check that B % N != 0 */ + if (!BN_nnmod(bn_result, bn_B, bn_prime, bn_ctx)) + goto err; + if (BN_is_zero(bn_result)) + goto err; + + /* (B - (k * g^x)) */ + if ((bn_base = BN_new()) == NULL) + goto err; + if (!BN_mod_exp(bn_result, bn_generator, bn_exponent, bn_prime, bn_ctx)) + goto err; + if (!BN_mod_mul(bn_result, bn_multiplier, bn_result, bn_prime, bn_ctx)) + goto err; + if (!BN_mod_sub(bn_base, bn_B, bn_result, bn_prime, bn_ctx)) + goto err; + + /* a + (u * x) */ + if ((bn_exp2 = BN_new()) == NULL) + goto err; + if (!BN_mul(bn_result, bn_u, bn_exponent, bn_ctx)) + goto err; + if (!BN_add(bn_exp2, bn_a, bn_result)) + goto err; + + /* (B - (k * g^x)) ^ (a + (u * x)) % N */ + if (!BN_mod_exp(bn_result, bn_base, bn_exp2, bn_prime, bn_ctx)) + goto err; + + if ((dlen = BN_num_bytes(bn_result)) < 0) + goto err; + if ((ptr = enif_make_new_binary(env, (size_t)dlen, &ret)) == NULL) + goto err; + + if (BN_bn2bin(bn_result, ptr) < 0) + goto err; + + goto done; + + bad_arg: + ret = enif_make_badarg(env); + goto done; + + err: + ret = atom_error; + + done: + if (bn_a) + BN_free(bn_a); + if (bn_u) + BN_free(bn_u); + if (bn_B) + BN_free(bn_B); + if (bn_multiplier) + BN_free(bn_multiplier); + if (bn_generator) + BN_free(bn_generator); + if (bn_exponent) + BN_free(bn_exponent); + if (bn_prime) + BN_free(bn_prime); + if (bn_ctx) + BN_CTX_free(bn_ctx); + if (bn_result) + BN_free(bn_result); + if (bn_base) + BN_free(bn_base); + if (bn_exp2) + BN_free(bn_exp2); + + return ret; +} + +ERL_NIF_TERM srp_host_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{/* (Verifier, b, u, A, Prime) */ +/* + <premaster secret> = (A * v^u) ^ b % N +*/ + BIGNUM *bn_b = NULL, *bn_verifier = NULL; + BIGNUM *bn_prime = NULL, *bn_A = NULL, *bn_u = NULL, *bn_base = NULL, *bn_result = NULL; + BN_CTX *bn_ctx = NULL; + unsigned char *ptr; + int dlen; + ERL_NIF_TERM ret; + + CHECK_NO_FIPS_MODE(); + + ASSERT(argc == 5); + + if (!get_bn_from_bin(env, argv[0], &bn_verifier)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[1], &bn_b)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[2], &bn_u)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[3], &bn_A)) + goto bad_arg; + if (!get_bn_from_bin(env, argv[4], &bn_prime)) + goto bad_arg; + + if ((bn_ctx = BN_CTX_new()) == NULL) + goto err; + if ((bn_result = BN_new()) == NULL) + goto err; + + /* check that A % N != 0 */ + if (!BN_nnmod(bn_result, bn_A, bn_prime, bn_ctx)) + goto err; + if (BN_is_zero(bn_result)) + goto err; + + /* (A * v^u) */ + if ((bn_base = BN_new()) == NULL) + goto err; + if (!BN_mod_exp(bn_base, bn_verifier, bn_u, bn_prime, bn_ctx)) + goto err; + if (!BN_mod_mul(bn_base, bn_A, bn_base, bn_prime, bn_ctx)) + goto err; + + /* (A * v^u) ^ b % N */ + if (!BN_mod_exp(bn_result, bn_base, bn_b, bn_prime, bn_ctx)) + goto err; + + if ((dlen = BN_num_bytes(bn_result)) < 0) + goto err; + if ((ptr = enif_make_new_binary(env, (size_t)dlen, &ret)) == NULL) + goto err; + + if (BN_bn2bin(bn_result, ptr) < 0) + goto err; + + goto done; + + bad_arg: + ret = enif_make_badarg(env); + goto done; + + err: + ret = atom_error; + + done: + if (bn_verifier) + BN_free(bn_verifier); + if (bn_b) + BN_free(bn_b); + if (bn_u) + BN_free(bn_u); + if (bn_A) + BN_free(bn_A); + if (bn_prime) + BN_free(bn_prime); + if (bn_ctx) + BN_CTX_free(bn_ctx); + if (bn_result) + BN_free(bn_result); + if (bn_base) + BN_free(bn_base); + + return ret; +} + diff --git a/lib/crypto/c_src/srp.h b/lib/crypto/c_src/srp.h new file mode 100644 index 0000000000..c356690470 --- /dev/null +++ b/lib/crypto/c_src/srp.h @@ -0,0 +1,30 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef E_SRP_H__ +#define E_SRP_H__ 1 + +#include "common.h" + +ERL_NIF_TERM srp_value_B_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM srp_user_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM srp_host_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#endif /* E_SRP_H__ */ diff --git a/lib/crypto/configure.in b/lib/crypto/configure.in new file mode 100644 index 0000000000..a3b6673f29 --- /dev/null +++ b/lib/crypto/configure.in @@ -0,0 +1,775 @@ +dnl Process this file with autoconf to produce a configure script. -*-m4-*- +dnl +dnl %CopyrightBegin% +dnl +dnl Copyright Ericsson AB 2018. All Rights Reserved. +dnl +dnl Licensed under the Apache License, Version 2.0 (the "License"); +dnl you may not use this file except in compliance with the License. +dnl You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. +dnl +dnl %CopyrightEnd% +dnl + +dnl define([AC_CACHE_LOAD], )dnl +dnl define([AC_CACHE_SAVE], )dnl + + +AC_INIT(vsn.mk) + +if test -z "$ERL_TOP" || test ! -d "$ERL_TOP" ; then + AC_CONFIG_AUX_DIRS(autoconf) +else + erl_top=${ERL_TOP} + AC_CONFIG_AUX_DIRS($erl_top/erts/autoconf) +fi + +if test "X$host" != "Xfree_source" -a "X$host" != "Xwin32"; then + AC_CANONICAL_HOST +else + host_os=win32 +fi + +LM_PRECIOUS_VARS + +if test "$cross_compiling" = "yes"; then + CROSS_COMPILING=yes +else + CROSS_COMPILING=no +fi +AC_SUBST(CROSS_COMPILING) + +ERL_XCOMP_SYSROOT_INIT + +AC_PROG_CC +LM_WINDOWS_ENVIRONMENT + +ERL_DED + +dnl +dnl SSL, SSH and CRYPTO need the OpenSSL libraries +dnl +dnl Check flags --with-ssl, --without-ssl --with-ssl=PATH. +dnl If no option is given or --with-ssl is set without a path then we +dnl search for OpenSSL libraries and header files in the standard locations. +dnl If set to --without-ssl we disable the use of SSL, SSH and CRYPTO. +dnl If set to --with-ssl=PATH we use that path as the prefix, i.e. we +dnl use "PATH/include" and "PATH/lib". + +AC_CHECK_SIZEOF(void *) + +std_ssl_locations="/usr/local /usr/sfw /usr /opt/local /usr/pkg /usr/local/openssl /usr/lib/openssl /usr/openssl /usr/local/ssl /usr/lib/ssl /usr/ssl /" + +AC_ARG_WITH(ssl-zlib, +AS_HELP_STRING([--with-ssl-zlib=PATH], + [specify location of ZLib to be used by OpenSSL]) +AS_HELP_STRING([--with-ssl-zlib], + [link SSL with Zlib (default if found)]) +AS_HELP_STRING([--without-ssl-zlib], + [don't link SSL with ZLib])) + +if test "x$with_ssl_zlib" = "xno"; then + SSL_LINK_WITH_ZLIB=no + STATIC_ZLIB_LIBS= +elif test "x$with_ssl_zlib" = "xyes" || test "x$with_ssl_zlib" = "x"; then + if test $erl_xcomp_without_sysroot = yes; then + AC_MSG_WARN([Cannot search for zlib; missing cross system root (erl_xcomp_sysroot).]) + SSL_LINK_WITH_ZLIB=no + STATIC_ZLIB_LIBS= + elif test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes"; then + SSL_LINK_WITH_ZLIB=no + STATIC_ZLIB_LIBS= + else + SSL_LINK_WITH_ZLIB=no + STATIC_ZLIB_LIBS= + AC_MSG_CHECKING(for static ZLib to be used by SSL in standard locations) + for rdir in $std_ssl_locations; do + dir="$erl_xcomp_sysroot$rdir" + if test "x$ac_cv_sizeof_void_p" = "x8"; then + if test -f "$dir/lib64/libz.a"; then + SSL_LINK_WITH_ZLIB=yes + STATIC_ZLIB_LIBS="$dir/lib64/libz.a" + break + elif test -f "$dir/lib/64/libz.a"; then + SSL_LINK_WITH_ZLIB=yes + STATIC_ZLIB_LIBS="$dir/lib/64/libz.a" + break + fi + fi + if test -f "$dir/lib/libz.a"; then + SSL_LINK_WITH_ZLIB=yes + STATIC_ZLIB_LIBS="$dir/lib/libz.a" + break + fi + done + if test "x$SSL_LINK_WITH_ZLIB" = "xno"; then + AC_MSG_RESULT([no]) + else + AC_MSG_RESULT([$STATIC_ZLIB_LIBS]) + fi + fi +else + SSL_LINK_WITH_ZLIB=no + STATIC_ZLIB_LIBS= + if test -f "$with_ssl_zlib/libz.a"; then + SSL_LINK_WITH_ZLIB=yes + STATIC_ZLIB_LIBS=$with_ssl_zlib/libz.a + elif test -f "$with_ssl_zlib/lib/libz.a"; then + SSL_LINK_WITH_ZLIB=yes + STATIC_ZLIB_LIBS=$with_ssl_zlib/lib/libz.a + fi + if test "x$ac_cv_sizeof_void_p" = "x8"; then + if test -f "$with_ssl_zlib/lib64/libz.a"; then + SSL_LINK_WITH_ZLIB=yes + STATIC_ZLIB_LIBS=$with_ssl_zlib/lib64/libz.a + elif test -f "$with_ssl_zlib/lib/64/libz.a"; then + SSL_LINK_WITH_ZLIB=yes + STATIC_ZLIB_LIBS=$with_ssl_zlib/lib/64/libz.a + fi + fi + if test "x$SSL_LINK_WITH_ZLIB" = "xno"; then + AC_MSG_ERROR(Invalid path to option --with-ssl-zlib=PATH) + fi +fi + + +AC_ARG_WITH(ssl, +AS_HELP_STRING([--with-ssl=PATH], [specify location of OpenSSL include and lib]) +AS_HELP_STRING([--with-ssl], [use SSL (default)]) +AS_HELP_STRING([--without-ssl], [don't use SSL])) + +AC_ARG_WITH(ssl-incl, +AS_HELP_STRING([--with-ssl-incl=PATH], [location of OpenSSL include dir, if different than specified by --with-ssl=PATH]), +[ +case X$with_ssl in + X | Xyes | Xno) AC_MSG_ERROR([--with-ssl-incl=PATH set without --with-ssl=PATH]);; +esac +], +[with_ssl_incl=$with_ssl]) #default + +AC_ARG_WITH(ssl-rpath, +AS_HELP_STRING([--with-ssl-rpath=yes|no|PATHS], + [runtime library path for OpenSSL. Default is "yes", which equates to a + number of standard locations. If "no", then no runtime + library paths will be used. Anything else should be a + comma separated list of paths.]), +[ +case X$with_ssl in + Xno) AC_MSG_ERROR([--with-ssl-rpath set without --with-ssl]);; +esac +], +[with_ssl_rpath=yes]) #default + + +AC_ARG_ENABLE(dynamic-ssl-lib, +AS_HELP_STRING([--disable-dynamic-ssl-lib], + [disable using dynamic openssl libraries]), +[ case "$enableval" in + no) enable_dynamic_ssl=no ;; + *) enable_dynamic_ssl=yes ;; + esac ], enable_dynamic_ssl=yes) + +#---------------------------------------------------------------------- +# We actually might do the SSL tests twice due to late discovery of +# kerberos problems with static linking, in case we redo it all trying +# dynamic SSL libraries instead. +#---------------------------------------------------------------------- + +ssl_done=no + +while test "x$ssl_done" != "xyes"; do + +ssl_done=yes # Default only one run + +# Remove all SKIP files from previous runs +for a in ssl crypto ssh; do + rm -f "$ERL_TOP/lib/$a/SKIP" +done + +SSL_DYNAMIC_ONLY=$enable_dynamic_ssl +SSL_STATIC_ONLY=no + +case "$erl_xcomp_without_sysroot-$with_ssl" in + yes-* | no-no) + SSL_APP= + CRYPTO_APP= + SSH_APP= + if test "$with_ssl" = "no"; then + skip="User gave --without-ssl option" + else + skip="Cannot search for ssl; missing cross system root (erl_xcomp_sysroot)." + fi + for a in ssl crypto ssh; do + echo "$skip" > $ERL_TOP/lib/$a/SKIP + done + ;; + no-yes | no- ) + # On windows, we could try to find the installation + # of Shining Light OpenSSL, which can be found by poking in + # the uninstall section in the registry, it's worth a try... + extra_dir="" + if test "x$MIXED_CYGWIN" = "xyes"; then + AC_CHECK_PROG(REGTOOL, regtool, regtool, false) + if test "$ac_cv_prog_REGTOOL" != false; then + wrp="/machine/software/microsoft/windows/currentversion/" + if test "x$ac_cv_sizeof_void_p" = "x8"; then + urp="uninstall/openssl (64-bit)_is1/inno setup: app path" + regtool_subsystem=-w + else + urp="uninstall/openssl (32-bit)_is1/inno setup: app path" + regtool_subsystem=-W + fi + rp="$wrp$urp" + if regtool -q $regtool_subsystem get "$rp" > /dev/null; then + true + else + # Fallback to unspecified wordlength + urp="uninstall/openssl_is1/inno setup: app path" + rp="$wrp$urp" + fi + if regtool -q $regtool_subsystem get "$rp" > /dev/null; then + ssl_install_dir=`regtool -q $regtool_subsystem get "$rp"` + # Try hard to get rid of spaces... + if cygpath -d "$ssl_install_dir" > /dev/null 2>&1; then + ssl_install_dir=`cygpath -d "$ssl_install_dir"` + fi + extra_dir=`cygpath $ssl_install_dir` + fi + fi + elif test "x$MIXED_MSYS" = "xyes"; then + AC_CHECK_PROG(REGTOOL, reg_query.sh, reg_query.sh, false) + if test "$ac_cv_prog_REGTOOL" != false; then + if test "x$ac_cv_sizeof_void_p" = "x8"; then + rp="HKLM/SOFTWARE/Microsoft/Windows/CurrentVersion/Uninstall/OpenSSL (64-bit)_is1" + else + rp="HKLM/SOFTWARE/Microsoft/Windows/CurrentVersion/Uninstall/OpenSSL_is1" + fi + key="Inno Setup: App Path" + if "$ac_cv_prog_REGTOOL" "$rp" "$key" > /dev/null; then + ssl_install_dir=`"$ac_cv_prog_REGTOOL" "$rp" "$key"` + extra_dir=`win2msys_path.sh "$ssl_install_dir"` + fi + fi + fi + # We search for OpenSSL in the common OS standard locations. + SSL_APP=ssl + CRYPTO_APP=crypto + SSH_APP=ssh + + SSL_CRYPTO_LIBNAME=crypto + SSL_SSL_LIBNAME=ssl + + if test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes"; then + if test "x$ac_cv_sizeof_void_p" = "x8"; then + std_win_ssl_locations="/cygdrive/c/OpenSSL-Win64 /c/OpenSSL-Win64 /opt/local64/pgm/OpenSSL" + else + std_win_ssl_locations="/cygdrive/c/OpenSSL-Win32 /c/OpenSSL-Win32 /cygdrive/c/OpenSSL /c/OpenSSL /opt/local/pgm/OpenSSL" + fi + else + std_win_ssl_locations="" + fi + + + AC_MSG_CHECKING(for OpenSSL >= 0.9.8c in standard locations) + for rdir in $extra_dir $std_win_ssl_locations $std_ssl_locations; do + dir="$erl_xcomp_sysroot$rdir" + if test -f "$erl_xcomp_isysroot$rdir/include/openssl/opensslv.h"; then + is_real_ssl=yes + SSL_INCDIR="$dir" + if test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes"; then + if test -f "$dir/lib/VC/libeay32.lib"; then + SSL_RUNTIME_LIBDIR="$rdir/lib/VC" + SSL_LIBDIR="$dir/lib/VC" + SSL_CRYPTO_LIBNAME=libeay32 + SSL_SSL_LIBNAME=ssleay32 + elif test -f "$dir/lib/VC/openssl.lib"; then + SSL_RUNTIME_LIBDIR="$rdir/lib/VC" + SSL_LIBDIR="$dir/lib/VC" + elif test -f $dir/lib/VC/libeay32MD.lib; then + SSL_CRYPTO_LIBNAME=libeay32MD + SSL_SSL_LIBNAME=ssleay32MD + if test "x$enable_dynamic_ssl" = "xno" && \ + test -f $dir/lib/VC/static/libeay32MD.lib; then + SSL_RUNTIME_LIBDIR="$rdir/lib/VC/static" + SSL_LIBDIR="$dir/lib/VC/static" + else + SSL_RUNTIME_LIBDIR="$rdir/lib/VC" + SSL_LIBDIR="$dir/lib/VC" + fi + elif test -f "$dir/lib/libeay32.lib"; then + SSL_RUNTIME_LIBDIR="$rdir/lib" + SSL_LIBDIR="$dir/lib" + SSL_CRYPTO_LIBNAME=libeay32 + SSL_SSL_LIBNAME=ssleay32 + elif test -f "$dir/lib/openssl.lib"; then + SSL_RUNTIME_LIBDIR="$rdir/lib" + SSL_LIBDIR="$dir/lib" + else + is_real_ssl=no + fi + elif test -f "$dir/lib/powerpc/libsslcrypto.a"; then + SSL_CRYPTO_LIBNAME=sslcrypto + SSL_LIBDIR="$dir/lib/powerpc/" + SSL_RUNTIME_LIBDIR="$rdir/lib/powerpc/" + else + if test "x$ac_cv_sizeof_void_p" = "x8"; then + if test -f "$dir/lib64/libcrypto.a"; then + SSL_RUNTIME_LIBDIR="$rdir/lib64" + SSL_LIBDIR="$dir/lib64" + elif test -f "$dir/lib/64/libcrypto.a"; then + SSL_RUNTIME_LIBDIR="$rdir/lib/64" + SSL_LIBDIR="$dir/lib/64" + elif test -f "$dir/lib64/libcrypto.so"; then + SSL_RUNTIME_LIBDIR="$rdir/lib64" + SSL_LIBDIR="$dir/lib64" + elif test -f "$dir/lib/64/libcrypto.so"; then + SSL_RUNTIME_LIBDIR="$rdir/lib/64" + SSL_LIBDIR="$dir/lib/64" + else + SSL_RUNTIME_LIBDIR="$rdir/lib" + SSL_LIBDIR="$dir/lib" + fi + else + SSL_RUNTIME_LIBDIR="$rdir/lib" + SSL_LIBDIR="$dir/lib" + fi + fi + if test '!' -f "$SSL_LIBDIR/lib${SSL_CRYPTO_LIBNAME}.a"; then + SSL_DYNAMIC_ONLY=yes + elif test '!' -f "$SSL_LIBDIR/lib${SSL_CRYPTO_LIBNAME}.so" -a '!' -f "$SSL_LIBDIR/lib${SSL_CRYPTO_LIBNAME}.dylib"; then + SSL_STATIC_ONLY=yes + fi + SSL_BINDIR="$rdir/bin" + if test "x$is_real_ssl" = "xyes" ; then + SSL_INCLUDE="-I$dir/include" + old_CPPFLAGS=$CPPFLAGS + CPPFLAGS=$SSL_INCLUDE + AC_EGREP_CPP(^yes$,[ +#include <openssl/opensslv.h> +#if OPENSSL_VERSION_NUMBER >= 0x0090803fL +yes +#endif + ],[ + ssl_found=yes + ],[ + SSL_APP= + ssl_found=no + ]) + CPPFLAGS=$old_CPPFLAGS + if test "x$ssl_found" = "xyes"; then + if test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes"; then + ssl_linkable=yes + elif test "x${SSL_CRYPTO_LIBNAME}" = "xsslcrypto"; then + # This should only be triggered seen OSE + ssl_linkable=yes + else + saveCFLAGS="$CFLAGS" + saveLDFLAGS="$LDFLAGS" + saveLIBS="$LIBS" + CFLAGS="$CFLAGS $SSL_INCLUDE" + if test "x$SSL_STATIC_ONLY" = "xyes"; then + LIBS="${SSL_LIBDIR}/lib${SSL_CRYPTO_LIBNAME}.a" + else + LDFLAGS="$LDFLAGS -L$SSL_LIBDIR" + LIBS="$LIBS -l${SSL_CRYPTO_LIBNAME}" + fi + AC_TRY_LINK([ + #include <stdio.h> + #include <openssl/hmac.h>], + [ + HMAC(0, 0, 0, 0, 0, 0, 0); + ], + [ssl_linkable=yes], + [ssl_linkable=no]) + CFLAGS="$saveCFLAGS" + LDFLAGS="$saveLDFLAGS" + LIBS="$saveLIBS" + fi + fi + if test "x$ssl_found" = "xyes" && test "x$ssl_linkable" = "xyes"; then + AC_MSG_RESULT([$dir]) + break; + fi + fi + fi + done + + if test "x$ssl_found" != "xyes" ; then + dnl + dnl If no SSL found above, check whether we are running on OpenBSD. + dnl + case $host_os in + openbsd*) + if test -f "$erl_xcomp_isysroot/usr/include/openssl/opensslv.h"; then + # Trust OpenBSD to have everything the in the correct locations. + ssl_found=yes + ssl_linkable=yes + SSL_INCDIR="$erl_xcomp_sysroot/usr" + AC_MSG_RESULT([$SSL_INCDIR]) + SSL_RUNTIME_LIB="/usr/lib" + SSL_LIB="$erl_xcomp_sysroot/usr/lib" + SSL_BINDIR="/usr/sbin" + dnl OpenBSD requires us to link with -L and -l + SSL_DYNAMIC_ONLY="yes" + fi + ;; + esac + fi +dnl Now, certain linuxes have a 64bit libcrypto +dnl that cannot build shared libraries (i.e. not PIC) +dnl One could argue that this is wrong, but +dnl so it is - be adoptable + if test "$ssl_found" = "yes" && test "$ssl_linkable" = "yes" && test "$SSL_DYNAMIC_ONLY" != "yes"; then + case $host_os in + linux*) + saveCFLAGS="$CFLAGS" + saveLDFLAGS="$LDFLAGS" + saveLIBS="$LIBS" + CFLAGS="$DED_CFLAGS $SSL_INCLUDE" + LDFLAGS="$DED_LDFLAGS" + LIBS="$SSL_LIBDIR/libcrypto.a $STATIC_ZLIB_LIBS" + AC_TRY_LINK([ + #include <stdio.h> + #include <openssl/hmac.h>], + [ + HMAC(0, 0, 0, 0, 0, 0, 0); + ], + [ssl_dyn_linkable=yes], + [ssl_dyn_linkable=no]) + CFLAGS="$saveCFLAGS" + LDFLAGS="$saveLDFLAGS" + LIBS="$saveLIBS" + if test "x$ssl_dyn_linkable" != "xyes"; then + SSL_DYNAMIC_ONLY=yes + AC_MSG_WARN([SSL will be linked against dynamic lib as static lib is not purely relocatable]) + fi + ;; + esac + fi + + + + + if test "x$ssl_found" != "xyes" || test "x$ssl_linkable" != "xyes"; then + if test "x$ssl_found" = "xyes"; then + AC_MSG_RESULT([found; but not usable]) + else + AC_MSG_RESULT([no]) + fi + SSL_APP= + CRYPTO_APP= + SSH_APP= + AC_MSG_WARN([No (usable) OpenSSL found, skipping ssl, ssh and crypto applications]) + + for a in ssl crypto ssh; do + echo "No usable OpenSSL found" > $ERL_TOP/lib/$a/SKIP + done + fi + ;; + *) + # Option given with PATH to package + if test ! -d "$with_ssl" ; then + AC_MSG_ERROR(Invalid path to option --with-ssl=PATH) + fi + if test ! -d "$with_ssl_incl" ; then + AC_MSG_ERROR(Invalid path to option --with-ssl-incl=PATH) + fi + SSL_INCDIR="$with_ssl_incl" + SSL_CRYPTO_LIBNAME=crypto + SSL_SSL_LIBNAME=ssl + if test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes" && test -d "$with_ssl/lib/VC"; then + if test -f "$with_ssl/lib/VC/libeay32.lib"; then + SSL_LIBDIR="$with_ssl/lib/VC" + SSL_CRYPTO_LIBNAME=libeay32 + SSL_SSL_LIBNAME=ssleay32 + elif test -f "$with_ssl/lib/VC/openssl.lib"; then + SSL_LIBDIR="$with_ssl/lib/VC" + elif test -f $with_ssl/lib/VC/libeay32MD.lib; then + SSL_CRYPTO_LIBNAME=libeay32MD + SSL_SSL_LIBNAME=ssleay32MD + if test "x$enable_dynamic_ssl" = "xno" && \ + test -f $with_ssl/lib/VC/static/libeay32MD.lib; then + SSL_LIBDIR="$with_ssl/lib/VC/static" + else + SSL_LIBDIR="$with_ssl/lib/VC" + fi + elif test -f "$with_ssl/lib/libeay32.lib"; then + SSL_LIBDIR="$with_ssl/lib" + SSL_CRYPTO_LIBNAME=libeay32 + SSL_SSL_LIBNAME=ssleay32 + else + # This probably wont work, but that's what the user said, so... + SSL_LIBDIR="$with_ssl/lib" + fi + elif test -f "$dir/lib/powerpc/libsslcrypto.a"; then + SSL_CRYPTO_LIBNAME=sslcrypto + SSL_LIBDIR="$with_ssl/lib/powerpc/" + elif test "x$ac_cv_sizeof_void_p" = "x8"; then + if test -f "$with_ssl/lib64/libcrypto.a"; then + SSL_LIBDIR="$with_ssl/lib64" + elif test -f "$with_ssl/lib/64/libcrypto.a"; then + SSL_LIBDIR="$with_ssl/lib/64" + elif test -f "$with_ssl/lib64/libcrypto.so"; then + SSL_LIBDIR="$with_ssl/lib64" + elif test -f "$with_ssl/lib/64/libcrypto.so"; then + SSL_LIBDIR="$with_ssl/lib/64" + else + SSL_LIBDIR="$with_ssl/lib" + fi + else + SSL_LIBDIR="$with_ssl/lib" + fi + if test '!' -f "${SSL_LIBDIR}/lib${SSL_CRYPTO_LIBNAME}.a"; then + SSL_DYNAMIC_ONLY=yes + elif test '!' -f ${SSL_LIBDIR}/lib${SSL_CRYPTO_LIBNAME}.so -a '!' -f "$SSL_LIBDIR/lib${SSL_CRYPTO_LIBNAME}.dylib"; then + SSL_STATIC_ONLY=yes + fi + SSL_INCLUDE="-I$with_ssl_incl/include" + SSL_APP=ssl + CRYPTO_APP=crypto + SSH_APP=ssh + if test "$cross_compiling" = "yes"; then + SSL_RUNTIME_LIBDIR=`echo "$SSL_LIBDIR" | sed -n "s|^$erl_xcomp_sysroot\(/*\)\(.*\)\$|/\2|p"` + else + SSL_RUNTIME_LIBDIR="$SSL_LIBDIR" + fi +esac + +if test "x$SSL_APP" != "x" ; then + dnl We found openssl, now check if we use kerberos 5 support + dnl FIXME: Do we still support platforms that have Kerberos? + AC_MSG_CHECKING(for OpenSSL kerberos 5 support) + old_CPPFLAGS=$CPPFLAGS + CPPFLAGS=$SSL_INCLUDE + AC_EGREP_CPP(^yes$,[ +#include <openssl/opensslv.h> +#include <openssl/opensslconf.h> +#if OPENSSL_VERSION_NUMBER < 0x1010000fL && !defined(OPENSSL_NO_KRB5) +yes +#endif + ],[ + AC_MSG_RESULT([yes]) + ssl_krb5_enabled=yes + if test "x$SSL_DYNAMIC_ONLY" != "xyes"; then + if test -f "$SSL_LIBDIR/libkrb5.a"; then + SSL_LINK_WITH_KERBEROS=yes + STATIC_KERBEROS_LIBS="$SSL_LIBDIR/libkrb5.a" + if test -f "$SSL_LIBDIR/libkrb5support.a"; then + STATIC_KERBEROS_LIBS="$STATIC_KERBEROS_LIBS $SSL_LIBDIR/libkrb5support.a" + fi + if test -f "$SSL_LIBDIR/libk5crypto.a"; then + STATIC_KERBEROS_LIBS="$STATIC_KERBEROS_LIBS $SSL_LIBDIR/libk5crypto.a" + fi + if test -f "$SSL_LIBDIR/libresolv.a"; then + STATIC_KERBEROS_LIBS="$STATIC_KERBEROS_LIBS $SSL_LIBDIR/libresolv.a" + fi + if test -f "$SSL_LIBDIR/libcom_err.a"; then + STATIC_KERBEROS_LIBS="$STATIC_KERBEROS_LIBS $SSL_LIBDIR/libcom_err.a" + fi + else + AC_MSG_WARN([Kerberos needed but no kerberos static libraries found]) + AC_MSG_WARN([Rescanning for dynamic SSL libraries]) + enable_dynamic_ssl=yes + ssl_done=no + SSL_LINK_WITH_KERBEROS=no + STATIC_KERBEROS_LIBS="" + ssl_krb5_enabled=no + SSL_WITH_KERBEROS=no + fi + else + SSL_LINK_WITH_KERBEROS=no + STATIC_KERBEROS_LIBS="" + fi + ],[ + AC_MSG_RESULT([no]) + ssl_krb5_enabled=no + SSL_WITH_KERBEROS=no + ]) + CPPFLAGS=$old_CPPFLAGS + SSL_KRB5_INCLUDE= + if test "x$ssl_krb5_enabled" = "xyes" ; then + AC_MSG_CHECKING(for krb5.h in standard locations) + for dir in $extra_dir "$SSL_INCDIR/include" "$SSL_INCDIR/include/openssl" \ + "$SSL_INCDIR/include/kerberos" \ + "$erl_xcomp_isysroot/cygdrive/c/kerberos/include" \ + "$erl_xcomp_isysroot/usr/local/kerberos/include" \ + "$erl_xcomp_isysroot/usr/kerberos/include" \ + "$erl_xcomp_isysroot/usr/include" + do + if test -f "$dir/krb5.h" ; then + SSL_KRB5_INCLUDE="$dir" + break + fi + done + if test "x$SSL_KRB5_INCLUDE" = "x" ; then + AC_MSG_RESULT([not found]) + SSL_APP= + CRYPTO_APP= + SSH_APP= + AC_MSG_WARN([OpenSSL is configured for kerberos but no krb5.h found]) + for a in ssl crypto ssh ; do + echo "OpenSSL is configured for kerberos but no krb5.h found" > $ERL_TOP/lib/$a/SKIP + done + else + AC_MSG_RESULT([found in $SSL_KRB5_INCLUDE]) + SSL_INCLUDE="$SSL_INCLUDE -I$SSL_KRB5_INCLUDE" + fi + fi +fi + +done # while test ssl_done != yes + +SSL_DED_LD_RUNTIME_LIBRARY_PATH= +ded_ld_rflg="$DED_LD_FLAG_RUNTIME_LIBRARY_PATH" + + +case "$with_ssl_rpath" in + +yes) # Use standard lib locations for ssl runtime library path + + if test "$SSL_APP" != "" && test "$SSL_DYNAMIC_ONLY" = "yes" && test "$ded_ld_rflg" != ""; then + + AC_MSG_CHECKING(for ssl runtime library path to use) + + libdirs="/lib" + + if test "$ac_cv_sizeof_void_p" = "8"; then + dir_lib64=no + dir_lib_64=no + + case "$SSL_RUNTIME_LIBDIR" in + */lib/64 | */lib/64/ ) dir_lib_64=yes;; + */lib64 | */lib64/ ) dir_lib64=yes;; + *) ;; + esac + + for dir in $std_ssl_locations; do + test $dir_lib_64 = no && + test -d "$erl_xcomp_sysroot$dir/lib/64" && + dir_lib_64=yes + test $dir_lib64 = no && + test -d "$erl_xcomp_sysroot$dir/lib64" && + dir_lib64=yes + done + + test $dir_lib_64 = yes && libdirs="/lib/64 $libdirs" + test $dir_lib64 = yes && libdirs="/lib64 $libdirs" + fi + + for type in std x_std curr; do + + ded_ld_rpath="$ded_ld_rflg$SSL_RUNTIME_LIBDIR" + rpath="$SSL_RUNTIME_LIBDIR" + + if test $type != curr; then + for ldir in $libdirs; do + for dir in $std_ssl_locations; do + test "$SSL_LIBDIR" != "$dir$ldir" || continue + test $type != x_std || test -d "$dir$ldir" || continue + if test "$dir" = "/"; then + libdir="$ldir" + else + libdir="$dir$ldir" + fi + ded_ld_rpath="$ded_ld_rpath $ded_ld_rflg$libdir" + rpath="$rpath:$libdir" + done + done + fi + + saveCFLAGS="$CFLAGS" + saveLDFLAGS="$LDFLAGS" + saveLIBS="$LIBS" + CFLAGS="$CFLAGS $SSL_INCLUDE" + LDFLAGS="$LDFLAGS $ld_rpath -L$SSL_LIBDIR" + LIBS="-lcrypto" + AC_TRY_LINK([ + #include <stdio.h> + #include <openssl/hmac.h> + ], + [ + HMAC(0, 0, 0, 0, 0, 0, 0); + ], + [rpath_success=yes], + [rpath_success=no]) + CFLAGS="$saveCFLAGS" + LDFLAGS="$saveLDFLAGS" + LIBS="$saveLIBS" + + test "$rpath_success" = "yes" && break + done + + test "$rpath_success" = "yes" || { ded_ld_rpath=; rpath=; } + + SSL_DED_LD_RUNTIME_LIBRARY_PATH="$ded_ld_rpath" + + AC_MSG_RESULT([$rpath]) + test "$rpath" != "" || AC_MSG_WARN([Cannot set run path during linking]) + fi + ;; + +no) # Use no ssl runtime library path + SSL_DED_LD_RUNTIME_LIBRARY_PATH= + ;; + +*) # Use ssl runtime library paths set by --with-ssl-rpath (without any check) + ded_ld_rpath= + delimit= + for dir in `echo $with_ssl_rpath | sed "s/,/ /g"`; do + ded_ld_rpath="$ded_ld_rpath$delimit$ded_ld_rflg$dir" + delimit=" " + done + SSL_DED_LD_RUNTIME_LIBRARY_PATH="$ded_ld_rpath" + ;; + +esac + + +AC_ARG_ENABLE(fips, +AS_HELP_STRING([--enable-fips], [enable OpenSSL FIPS mode support]) +AS_HELP_STRING([--disable-fips], [disable OpenSSL FIPS mode support (default)]), +[ case "$enableval" in + yes) enable_fips_support=yes ;; + *) enable_fips_support=no ;; + esac ], enable_fips_support=no) + +if test "x$enable_fips_support" = "xyes" && test "$CRYPTO_APP" != ""; then + saveCFLAGS="$CFLAGS" + saveLDFLAGS="$LDFLAGS" + saveLIBS="$LIBS" + CFLAGS="$CFLAGS $SSL_INCLUDE" + LDFLAGS="$LDFLAGS $ded_ld_rpath -L$SSL_LIBDIR" + LIBS="-lcrypto" + AC_CHECK_FUNC([FIPS_mode_set], + [SSL_FLAGS="-DFIPS_SUPPORT"], + [SSL_FLAGS=]) + CFLAGS="$saveCFLAGS" + LDFLAGS="$saveLDFLAGS" + LIBS="$saveLIBS" +else + SSL_FLAGS= +fi + +AC_SUBST(SSL_INCLUDE) +AC_SUBST(SSL_INCDIR) +AC_SUBST(SSL_LIBDIR) +AC_SUBST(SSL_FLAGS) +AC_SUBST(SSL_CRYPTO_LIBNAME) +AC_SUBST(SSL_SSL_LIBNAME) +AC_SUBST(SSL_DED_LD_RUNTIME_LIBRARY_PATH) +AC_SUBST(SSL_DYNAMIC_ONLY) +AC_SUBST(SSL_LINK_WITH_KERBEROS) +AC_SUBST(STATIC_KERBEROS_LIBS) +AC_SUBST(SSL_LINK_WITH_ZLIB) +AC_SUBST(STATIC_ZLIB_LIBS) + +AC_OUTPUT(c_src/$host/Makefile:c_src/Makefile.in) + diff --git a/lib/crypto/doc/src/Makefile b/lib/crypto/doc/src/Makefile index cbcafb7375..8da494dad6 100644 --- a/lib/crypto/doc/src/Makefile +++ b/lib/crypto/doc/src/Makefile @@ -39,7 +39,8 @@ XML_REF3_FILES = crypto.xml XML_REF6_FILES = crypto_app.xml XML_PART_FILES = usersguide.xml -XML_CHAPTER_FILES = notes.xml licenses.xml fips.xml engine_load.xml engine_keys.xml algorithm_details.xml +XML_CHAPTER_FILES = notes.xml licenses.xml fips.xml engine_load.xml engine_keys.xml \ + algorithm_details.xml new_api.xml BOOK_FILES = book.xml diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 5c1909fc7f..d1d1252f29 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -1,4 +1,3 @@ -<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd"> <erlref> @@ -43,8 +42,12 @@ <item> <url href="https://www.nist.gov/publications/sha-3-standard-permutation-based-hash-and-extendable-output-functions?pub_id=919061"> SHA-3 Standard: Permutation-Based Hash and Extendable-Output Functions [FIPS PUB 202] - </url> + </url> </item> + <tag>BLAKE2</tag> + <item> + <url href="https://blake2.net/">BLAKE2 — fast secure hashing</url> + </item> <tag>MD5</tag> <item> <url href="http://www.ietf.org/rfc/rfc1321.txt">The MD5 Message Digest Algorithm [RFC 1321]</url> @@ -187,63 +190,114 @@ </description> <datatypes> - <datatype_title>Ciphers</datatype_title> + <datatype_title>Ciphers, new API</datatype_title> <datatype> - <name name="stream_cipher"/> + <name name="cipher"/> <desc> - <p>Stream ciphers for - <seealso marker="#stream_encrypt-2">stream_encrypt/2</seealso> and - <seealso marker="#stream_decrypt-2">stream_decrypt/2</seealso> . - </p> + </desc> + </datatype> + <datatype> + <name name="cipher_no_iv"/> + <desc> + </desc> + </datatype> + <datatype> + <name name="cipher_iv"/> + <desc> + </desc> + </datatype> + <datatype> + <name name="cipher_aead"/> + <desc> + <p>Ciphers known by the CRYPTO application when using the + <seealso marker="crypto:new_api#the-new-api">new API</seealso>.</p> + <p>Note that this list might be reduced if the underlying libcrypto does not support all of them.</p> </desc> </datatype> + <datatype_title>Ciphers, old API</datatype_title> <datatype> <name name="block_cipher_with_iv"/> + <desc> + </desc> + </datatype> + <datatype> + <name name="block_cipher_without_iv"/> + <desc> + </desc> + </datatype> + <datatype> + <name name="stream_cipher"/> + <desc> + </desc> + </datatype> + <datatype> + <name name="aead_cipher"/> + <desc> + </desc> + </datatype> + <datatype> <name name="cbc_cipher"/> + <desc> + </desc> + </datatype> + <datatype> <name name="cfb_cipher"/> <desc> - <p>Block ciphers with initialization vector for - <seealso marker="#block_encrypt-4">block_encrypt/4</seealso> and - <seealso marker="#block_decrypt-4">block_decrypt/4</seealso> . - </p> </desc> </datatype> - <datatype> - <name name="block_cipher_without_iv"/> + <name name="ctr_cipher"/> + <desc> + </desc> + </datatype> + <datatype> <name name="ecb_cipher"/> <desc> - <p>Block ciphers without initialization vector for - <seealso marker="#block_encrypt-3">block_encrypt/3</seealso> and - <seealso marker="#block_decrypt-3">block_decrypt/3</seealso> . - </p> + <p>Ciphers known by the CRYPTO application when using the + <seealso marker="crypto:new_api#the-old-api">old API</seealso>.</p> + <p>Note that this list might be reduced if the underlying libcrypto does not support all of them.</p> </desc> </datatype> <datatype> - <name name="aead_cipher"/> + <name name="retired_cbc_cipher_aliases"/> + <desc> + </desc> + </datatype> + <datatype> + <name name="retired_cfb_cipher_aliases"/> + <desc> + </desc> + </datatype> + <datatype> + <name name="retired_ctr_cipher_aliases"/> <desc> - <p>Ciphers with simultaneous MAC-calculation or MAC-checking. - <seealso marker="#block_encrypt-4">block_encrypt/4</seealso> and - <seealso marker="#block_decrypt-4">block_decrypt/4</seealso> . + </desc> + </datatype> + <datatype> + <name name="retired_ecb_cipher_aliases"/> + <desc> + <p>Alternative, old names of ciphers known by the CRYPTO application when using the + <seealso marker="crypto:new_api#the-old-api">old API</seealso>. + See <seealso marker="crypto:new_api#retired-cipher-names">Retired cipher names</seealso> for names to + use instead to be prepared for an easy convertion to the + <seealso marker="crypto:new_api#the-new-api">new API</seealso>. </p> + <p>Note that this list might be reduced if the underlying libcrypto does not support all of them.</p> </desc> </datatype> - <datatype_title>Digests</datatype_title> + <datatype_title>Digests and hash</datatype_title> <datatype> - <name name="sha1"/> - <name name="sha2"/> - <name name="sha3"/> + <name name="hash_algorithm"/> <desc> </desc> </datatype> <datatype> - <name name="compatibility_only_hash"/> + <name name="hmac_hash_algorithm"/> <desc> - <p>The <c>compatibility_only_hash()</c> algorithms are recommended only for compatibility with existing applications.</p> </desc> </datatype> @@ -265,6 +319,17 @@ </desc> </datatype> + <datatype> + <name name="sha1"/> + <name name="sha2"/> + <name name="sha3"/> + <name name="blake2"/> + <name name="compatibility_only_hash"/> + <desc> + <p>The <c>compatibility_only_hash()</c> algorithms are recommended only for compatibility with existing applications.</p> + </desc> + </datatype> + <datatype_title>Elliptic Curves</datatype_title> <datatype> <name name="ec_named_curve"/> @@ -513,96 +578,219 @@ <name name="stream_state"/> <name name="hmac_state"/> <name name="hash_state"/> + <name name="crypto_state"/> <desc> <p>Contexts with an internal state that should not be manipulated but passed between function calls. </p> </desc> </datatype> + <datatype_title>Error types</datatype_title> + + <datatype> + <name name="run_time_error"/> + <desc> + <p>The exception <c>error:badarg</c> signifies that one or more arguments are of wrong data type, + or are otherwise badly formed. + </p> + <p>The exception <c>error:notsup</c> signifies that the algorithm is known but is not supported + by current underlying libcrypto or explicitly disabled when building that. + </p> + <p>For a list of supported algorithms, see <seealso marker="#supports-0">supports/0</seealso>. + </p> + </desc> + </datatype> + + <datatype> + <name name="descriptive_error"/> + <desc> + <p>This is a more developed variant of the older + <seealso marker="#type-run_time_error">run_time_error()</seealso>. + </p> + <p>The exception is:</p> + <pre> + {Tag, {C_FileName,LineNumber}, Description} + + Tag = badarg | notsup | error + C_FileName = string() + LineNumber = integer() + Description = string() + </pre> + + <p>It is like the older type an exception of the <c>error</c> class. In addition they contain + a descriptive text in English. That text is targeted to a developer. Examples are "Bad key size" + or "Cipher id is not an atom". + </p> + <p>The exception tags are:</p> + <taglist> + <tag><c>badarg</c></tag> + <item><p>Signifies that one or more arguments are of wrong data type or are otherwise badly formed.</p> + </item> + + <tag><c>notsup</c></tag> + <item><p>Signifies that the algorithm is known but is not supported by current underlying libcrypto + or explicitly disabled when building that one.</p> + </item> + + <tag><c>error</c></tag> + <item><p>An error condition that should not occur, for example a memory allocation failed or + the underlying cryptolib returned an error code, for example "Can't initialize context, step 1". + Thoose text usually needs searching the C-code to be understood.</p> + </item> + </taglist> + <p>To catch the exception, use for example:</p> + <code> + try crypto:crypto_init(Ciph, Key, IV, true) + catch + error:{Tag, {C_FileName,LineNumber}, Description} -> + do_something(......) + ..... + end + </code> + </desc> + </datatype> + </datatypes> <!--================ FUNCTIONS ================--> + <section> + <title>New API</title> + </section> + <funcs> <func> - <name name="block_encrypt" arity="3" since="OTP 18.0"/> - <fsummary>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher</fsummary> + <name name="crypto_init" arity="3" since="OTP 22.0"/> + <fsummary>Initializes a series of encryptions or decryptions</fsummary> <desc> - <p>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher.</p> - <p>May raise exception <c>error:notsup</c> in case the chosen <c>Type</c> - is not supported by the underlying libcrypto implementation.</p> - <p>For keylengths and blocksizes see the - <seealso marker="crypto:algorithm_details#ciphers">User's Guide</seealso>. + <p>As <seealso marker="#crypto_init/4">crypto_init/4</seealso> but for ciphers without IVs.</p> + </desc> + </func> + + <func> + <name name="crypto_init" arity="4" since="OTP 22.0"/> + <fsummary>Initializes a series of encryptions or decryptions</fsummary> + <desc> + <p>Part of the <seealso marker="crypto:new_api#the-new-api">new API</seealso>. + Initializes a series of encryptions or decryptions and creates an internal state + with a reference that is returned. + The actual encryption or decryption is done by + <seealso marker="crypto#crypto_update/2">crypto_update/2</seealso>. + </p> + <p>For encryption, set the <c>EncryptFlag</c> to <c>true</c>. For decryption, set it to <c>false</c>. + </p> + <p>See <seealso marker="crypto:new_api#examples-of-crypto_init-4-and-crypto_update-2"> + examples in the User's Guide.</seealso> </p> </desc> </func> <func> - <name name="block_decrypt" arity="3" since="OTP 18.0"/> - <fsummary>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher</fsummary> + <name name="crypto_update" arity="2" since="OTP 22.0"/> + <fsummary>Do an actual crypto operation on a part of the full text</fsummary> <desc> - <p>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher.</p> - <p>May raise exception <c>error:notsup</c> in case the chosen <c>Type</c> - is not supported by the underlying libcrypto implementation.</p> - <p>For keylengths and blocksizes see the - <seealso marker="crypto:algorithm_details#ciphers">User's Guide</seealso>. + <p>Part of the <seealso marker="crypto:new_api#the-new-api">new API</seealso>. + It does an actual crypto operation on a part of the full text. If the part is less + than a number of full blocks, only the full blocks (possibly none) are encrypted + or decrypted and the remaining bytes are saved to the next <c>crypto_update</c> operation. + The <c>State</c> should be created with + <seealso marker="crypto#crypto_init/3">crypto_init/3</seealso> + or + <seealso marker="crypto#crypto_init/4">crypto_init/4</seealso>. + </p> + <p>See <seealso marker="crypto:new_api#examples-of-crypto_init-4-and-crypto_update-2"> + examples in the User's Guide.</seealso> </p> </desc> </func> <func> - <name since="OTP R16B01">block_encrypt(Type, Key, Ivec, PlainText) -> CipherText</name> - <name since="OTP R16B01">block_encrypt(AeadType, Key, Ivec, {AAD, PlainText}) -> {CipherText, CipherTag}</name> - <name since="OTP R16B01">block_encrypt(aes_gcm | aes_ccm, Key, Ivec, {AAD, PlainText, TagLength}) -> {CipherText, CipherTag}</name> - <fsummary>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher</fsummary> - <type> - <v>Type = <seealso marker="#type-block_cipher_with_iv">block_cipher_with_iv()</seealso></v> - <v>AeadType = <seealso marker="#type-aead_cipher">aead_cipher()</seealso></v> - <v>Key = <seealso marker="#type-key">key()</seealso> | <seealso marker="#type-des3_key">des3_key()</seealso></v> - <v>PlainText = iodata()</v> - <v>AAD = IVec = CipherText = CipherTag = binary()</v> - <v>TagLength = 1..16</v> - </type> + <name name="crypto_dyn_iv_init" arity="3" since="OTP 22.0"/> + <fsummary>Initializes a series of encryptions or decryptions where the IV is provided later</fsummary> <desc> - <p>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher. - <c>IVec</c> is an arbitrary initializing vector.</p> - <p>In AEAD (Authenticated Encryption with Associated Data) mode, encrypt - <c>PlainText</c>according to <c>Type</c> block cipher and calculate - <c>CipherTag</c> that also authenticates the <c>AAD</c> (Associated Authenticated Data).</p> - <p>May raise exception <c>error:notsup</c> in case the chosen <c>Type</c> - is not supported by the underlying libcrypto implementation.</p> - <p>For keylengths, iv-sizes and blocksizes see the - <seealso marker="crypto:algorithm_details#ciphers">User's Guide</seealso>. + <p>Part of the <seealso marker="crypto:new_api#the-new-api">new API</seealso>. + Initializes a series of encryptions or decryptions where the IV is provided later. + The actual encryption or decryption is done by + <seealso marker="crypto#crypto_dyn_iv_update/3">crypto_dyn_iv_update/3</seealso>. + </p> + <p>For encryption, set the <c>EncryptFlag</c> to <c>true</c>. For decryption, set it to <c>false</c>. </p> </desc> </func> <func> - <name since="OTP R16B01">block_decrypt(Type, Key, Ivec, CipherText) -> PlainText</name> - <name since="OTP R16B01">block_decrypt(AeadType, Key, Ivec, {AAD, CipherText, CipherTag}) -> PlainText | error</name> - <fsummary>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher</fsummary> - <type> - <v>Type = <seealso marker="#type-block_cipher_with_iv">block_cipher_with_iv()</seealso></v> - <v>AeadType = <seealso marker="#type-aead_cipher">aead_cipher()</seealso></v> - <v>Key = <seealso marker="#type-key">key()</seealso> | <seealso marker="#type-des3_key">des3_key()</seealso></v> - <v>PlainText = iodata()</v> - <v>AAD = IVec = CipherText = CipherTag = binary()</v> - </type> + <name name="crypto_dyn_iv_update" arity="3" since="OTP 22.0"/> + <fsummary>Do an actual crypto operation on a part of the full text and the IV is supplied for each part</fsummary> <desc> - <p>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher. - <c>IVec</c> is an arbitrary initializing vector.</p> - <p>In AEAD (Authenticated Encryption with Associated Data) mode, decrypt - <c>CipherText</c>according to <c>Type</c> block cipher and check the authenticity - the <c>PlainText</c> and <c>AAD</c> (Associated Authenticated Data) using the - <c>CipherTag</c>. May return <c>error</c> if the decryption or validation fail's</p> - <p>May raise exception <c>error:notsup</c> in case the chosen <c>Type</c> - is not supported by the underlying libcrypto implementation.</p> - <p>For keylengths, iv-sizes and blocksizes see the - <seealso marker="crypto:algorithm_details#ciphers">User's Guide</seealso>. + <p>Part of the <seealso marker="crypto:new_api#the-new-api">new API</seealso>. + Do an actual crypto operation on a part of the full text and the IV is supplied for each part. + The <c>State</c> should be created with + <seealso marker="crypto#crypto_dyn_iv_init/3">crypto_dyn_iv_init/3</seealso>. </p> </desc> </func> - <func> + <func> + <name name="crypto_one_time" arity="4" since="OTP 22.0"/> + <fsummary>Do a complete encrypt or decrypt of the full text</fsummary> + <desc> + <p>As <seealso marker="#crypto_one_time/5">crypto_one_time/5</seealso> but for ciphers without IVs.</p> + </desc> + </func> + + <func> + <name name="crypto_one_time" arity="5" since="OTP 22.0"/> + <fsummary>Do a complete encrypt or decrypt of the full text</fsummary> + <desc> + <p>Part of the <seealso marker="crypto:new_api#the-new-api">new API</seealso>. + Do a complete encrypt or decrypt of the full text in the argument <c>Data</c>. + </p> + <p>For encryption, set the <c>EncryptFlag</c> to <c>true</c>. For decryption, set it to <c>false</c>. + </p> + <p>See <seealso marker="crypto:new_api#example-of-crypto_one_time-5">examples in the User's Guide.</seealso> + </p> + </desc> + </func> + + <func> + <name name="crypto_one_time_aead" arity="6" since="OTP 22.0"/> + <name name="crypto_one_time_aead" arity="7" since="OTP 22.0"/> + <fsummary>Do a complete encrypt or decrypt with an AEAD cipher of the full text</fsummary> + <desc> + <p>Part of the <seealso marker="crypto:new_api#the-new-api">new API</seealso>. + Do a complete encrypt or decrypt with an AEAD cipher of the full text. + </p> + <p>For encryption, set the <c>EncryptFlag</c> to <c>true</c> and set the <c>TagOrTagLength</c> + to the wanted size of the tag, that is, the tag length. If the default length is wanted, the + <c>crypto_aead/6</c> form may be used. + </p> + <p>For decryption, set the <c>EncryptFlag</c> to <c>false</c> and put the tag to be checked + in the argument <c>TagOrTagLength</c>. + </p> + <p>See <seealso marker="crypto:new_api#example-of-crypto_one_time_aead-6">examples in the User's Guide.</seealso> + </p> + </desc> + </func> + + <func> + <name name="supports" arity="1" since="OTP 22.0"/> + <fsummary>Provide a list of available crypto algorithms.</fsummary> + <desc> + <p> Can be used to determine which crypto algorithms that are supported + by the underlying libcrypto library</p> + <p>See <seealso marker="#hash_info-1">hash_info/1</seealso> and <seealso marker="#cipher_info-1">cipher_info/1</seealso> + for information about the hash and cipher algorithms. + </p> + </desc> + </func> + + </funcs> + + <section> + <title>API kept from previous versions</title> + </section> + + <funcs> + <func> <name name="bytes_to_integer" arity="1" since="OTP R16B01"/> <fsummary>Convert binary representation, of an integer, to an Erlang integer.</fsummary> <desc> @@ -826,6 +1014,39 @@ </func> <func> + <name name="hash_info" arity="1" since="OTP 22.0"/> + <fsummary>Information about supported hash algorithms.</fsummary> + <desc> + <p>Provides a map with information about block_size, size and possibly other properties of the + hash algorithm in question. + </p> + <p>For a list of supported hash algorithms, see <seealso marker="#supports-0">supports/0</seealso>. + </p> + </desc> + </func> + + <func> + <name name="cipher_info" arity="1" since="OTP 22.0"/> + <fsummary>Information about supported ciphers.</fsummary> + <desc> + <p>Provides a map with information about block_size, key_length, iv_length and possibly other properties of the + cipher algorithm in question. + </p> + <note> + <p>The ciphers <c>aes_cbc</c>, <c>aes_cfb8</c>, <c>aes_cfb128</c>, <c>aes_ctr</c>, + <c>aes_ecb</c>, <c>aes_gcm</c> and <c>aes_ccm</c> + has no keylength in the <c>Type</c> as opposed to for example <c>aes_128_ctr</c>. They adapt to the length of + the key provided in the encrypt and decrypt function. Therefor it is impossible to return a valid keylength + in the map.</p> + <p>Always use a <c>Type</c> with an explicit key length, + </p> + </note> + <p>For a list of supported cipher algorithms, see <seealso marker="#supports-0">supports/0</seealso>. + </p> + </desc> + </func> + + <func> <name name="mod_pow" arity="3" since="OTP R16B01"/> <fsummary>Computes the function: N^P mod M</fsummary> <desc> @@ -978,7 +1199,7 @@ <seealso marker="#rand_seed_s-0">rand_seed_s/0</seealso>. </p> <p> - When using the state object from this function the + When using the state object from this function the <seealso marker="stdlib:rand">rand</seealso> functions using it may raise exception <c>error:low_entropy</c> in case the random generator failed due to lack of secure "randomness". @@ -1004,16 +1225,16 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre> <seealso marker="stdlib:rand#seed_s-1">rand:seed_s/1</seealso>. </p> <p> - When using the state object from this function the + When using the state object from this function the <seealso marker="stdlib:rand">rand</seealso> functions using it may raise exception <c>error:low_entropy</c> in case the random generator failed due to lack of secure "randomness". </p> <note> <p> - The state returned from this function can not be used + The state returned from this function cannot be used to get a reproducable random sequence as from - the other + the other <seealso marker="stdlib:rand">rand</seealso> functions, since reproducability does not match cryptographically safe. @@ -1037,23 +1258,18 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre> <p> Creates state object for <seealso marker="stdlib:rand">random number generation</seealso>, - in order to generate cryptographically strong random numbers. + in order to generate cryptographically strong random numbers, + and saves it in the process dictionary before returning it as well. See also <seealso marker="stdlib:rand#seed-1">rand:seed/1</seealso> and <seealso marker="#rand_seed_alg_s-1">rand_seed_alg_s/1</seealso>. </p> <p> - When using the state object from this function the + When using the state object from this function the <seealso marker="stdlib:rand">rand</seealso> functions using it may raise exception <c>error:low_entropy</c> in case the random generator failed due to lack of secure "randomness". </p> - <p> - The cache size can be changed from its default value using the - <seealso marker="crypto_app"> - crypto app's - </seealso> configuration parameter <c>rand_cache_size</c>. - </p> <p><em>Example</em></p> <pre> _ = crypto:rand_seed_alg(crypto_cache), @@ -1063,6 +1279,34 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre> </func> <func> + <name since="OTP-22.0">rand_seed_alg(Alg, Seed) -> rand:state()</name> + <fsummary>Strong random number generation plugin state</fsummary> + <type> + <v>Alg = crypto_aes</v> + </type> + <desc> + <marker id="rand_seed_alg-2" /> + <p> + Creates a state object for + <seealso marker="stdlib:rand">random number generation</seealso>, + in order to generate cryptographically unpredictable random numbers, + and saves it in the process dictionary before returning it as well. + See also + <seealso marker="#rand_seed_alg_s-2">rand_seed_alg_s/2</seealso>. + </p> + <p><em>Example</em></p> + <pre> +_ = crypto:rand_seed_alg(crypto_aes, "my seed"), +IntegerValue = rand:uniform(42), % [1; 42] +FloatValue = rand:uniform(), % [0.0; 1.0[ +_ = crypto:rand_seed_alg(crypto_aes, "my seed"), +IntegerValue = rand:uniform(42), % Same values +FloatValue = rand:uniform(). % again + </pre> + </desc> + </func> + + <func> <name since="OTP 21.0">rand_seed_alg_s(Alg) -> rand:state()</name> <fsummary>Strong random number generation plugin state</fsummary> <type> @@ -1088,7 +1332,7 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre> of 56 bits that makes calculations fast on 64 bit machines. </p> <p> - When using the state object from this function the + When using the state object from this function the <seealso marker="stdlib:rand">rand</seealso> functions using it may raise exception <c>error:low_entropy</c> in case the random generator failed due to lack of secure "randomness". @@ -1099,11 +1343,17 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre> crypto app's </seealso> configuration parameter <c>rand_cache_size</c>. </p> + <p> + When using the state object from this function the + <seealso marker="stdlib:rand">rand</seealso> functions using it + may throw exception <c>low_entropy</c> in case the random generator + failed due to lack of secure "randomness". + </p> <note> <p> - The state returned from this function can not be used + The state returned from this function cannot be used to get a reproducable random sequence as from - the other + the other <seealso marker="stdlib:rand">rand</seealso> functions, since reproducability does not match cryptographically safe. @@ -1121,63 +1371,67 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre> </func> <func> - <name name="stream_init" arity="2" since="OTP R16B01"/> - <fsummary></fsummary> + <name since="OTP 22.0">rand_seed_alg_s(Alg, Seed) -> rand:state()</name> + <fsummary>Strong random number generation plugin state</fsummary> + <type> + <v>Alg = crypto_aes</v> + </type> <desc> - <p>Initializes the state for use in RC4 stream encryption - <seealso marker="#stream_encrypt-2">stream_encrypt</seealso> and - <seealso marker="#stream_decrypt-2">stream_decrypt</seealso></p> - <p>For keylengths see the - <seealso marker="crypto:algorithm_details#stream-ciphers">User's Guide</seealso>. + <marker id="rand_seed_alg_s-2" /> + <p> + Creates a state object for + <seealso marker="stdlib:rand">random number generation</seealso>, + in order to generate cryptographically unpredictable random numbers. + See also + <seealso marker="#rand_seed_alg-1">rand_seed_alg/1</seealso>. </p> - </desc> - </func> - - <func> - <name name="stream_init" arity="3" since="OTP R16B01"/> - <fsummary></fsummary> - <desc> - <p>Initializes the state for use in streaming AES encryption using Counter mode (CTR). - <c>Key</c> is the AES key and must be either 128, 192, or 256 bits long. <c>IVec</c> is - an arbitrary initializing vector of 128 bits (16 bytes). This state is for use with - <seealso marker="#stream_encrypt-2">stream_encrypt</seealso> and - <seealso marker="#stream_decrypt-2">stream_decrypt</seealso>.</p> - <p>For keylengths and iv-sizes see the - <seealso marker="crypto:algorithm_details#stream-ciphers">User's Guide</seealso>. + <p> + To get a long period the Xoroshiro928 generator from the + <seealso marker="stdlib:rand">rand</seealso> + module is used as a counter (with period 2^928 - 1) + and the generator states are scrambled through AES + to create 58-bit pseudo random values. </p> - </desc> - </func> - - <func> - <name name="stream_encrypt" arity="2" since="OTP R16B01"/> - <fsummary></fsummary> - <desc> - <p>Encrypts <c>PlainText</c> according to the stream cipher <c>Type</c> specified in stream_init/3. - <c>Text</c> can be any number of bytes. The initial <c>State</c> is created using - <seealso marker="#stream_init-2">stream_init</seealso>. - <c>NewState</c> must be passed into the next call to <c>stream_encrypt</c>.</p> - </desc> - </func> - - <func> - <name name="stream_decrypt" arity="2" since="OTP R16B01"/> - <fsummary></fsummary> - <desc> - <p>Decrypts <c>CipherText</c> according to the stream cipher <c>Type</c> specified in stream_init/3. - <c>PlainText</c> can be any number of bytes. The initial <c>State</c> is created using - <seealso marker="#stream_init-2">stream_init</seealso>. - <c>NewState</c> must be passed into the next call to <c>stream_decrypt</c>.</p> - </desc> - </func> - - <func> - <name name="supports" arity="0" since="OTP R16B01"/> - <fsummary>Provide a list of available crypto algorithms.</fsummary> - <desc> - <p> Can be used to determine which crypto algorithms that are supported - by the underlying libcrypto library</p> - <p>Note: the <c>rsa_opts</c> entry is in an experimental state and may change or be removed without notice. - No guarantee for the accuarcy of the rsa option's value list should be assumed. + <p> + The result should be statistically completely unpredictable + random values, since the scrambling is cryptographically strong + and the period is ridiculously long. But the generated numbers + are not to be regarded as cryptographically strong since + there is no re-keying schedule. + </p> + <list type="bulleted"> + <item> + <p> + If you need cryptographically strong random numbers use + <seealso marker="#rand_seed_alg_s-1">rand_seed_alg_s/1</seealso> + with <c>Alg =:= crypto</c> or <c>Alg =:= crypto_cache</c>. + </p> + </item> + <item> + <p> + If you need to be able to repeat the sequence use this function. + </p> + </item> + <item> + <p> + If you do not need the statistical quality of this function, + there are faster algorithms in the + <seealso marker="stdlib:rand">rand</seealso> + module. + </p> + </item> + </list> + <p> + Thanks to the used generator the state object supports the + <seealso marker="stdlib:rand#jump-0"><c>rand:jump/0,1</c></seealso> + function with distance 2^512. + </p> + <p> + Numbers are generated in batches and cached for speed reasons. + The cache size can be changed from its default value using the + <seealso marker="crypto_app"> + crypto app's + </seealso> configuration parameter <c>rand_cache_size</c>. </p> </desc> </func> @@ -1229,6 +1483,12 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre> </desc> </func> + </funcs> + <section> + <title>Engine API</title> + </section> + + <funcs> <!-- Engine functions --> <func> <name name="privkey_to_pubkey" arity="2" since="OTP 20.2"/> @@ -1541,5 +1801,167 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre> </funcs> +<section> + <title>Old API</title> +</section> + + <funcs> + <func> + <name name="block_encrypt" arity="3" since="OTP 18.0"/> + <fsummary>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher</fsummary> + <desc> + <dont><p>Don't use this function for new programs! Use <seealso marker="crypto:new_api">the-new-api</seealso>.</p></dont> + <p>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher.</p> + <p>May raise exception <c>error:notsup</c> in case the chosen <c>Type</c> + is not supported by the underlying libcrypto implementation.</p> + <p>For keylengths and blocksizes see the + <seealso marker="crypto:algorithm_details#ciphers">User's Guide</seealso>. + </p> + </desc> + </func> + + <func> + <name name="block_decrypt" arity="3" since="OTP 18.0"/> + <fsummary>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher</fsummary> + <desc> + <dont><p>Don't use this function for new programs! Use <seealso marker="crypto:new_api">the new api</seealso>.</p></dont> + <p>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher.</p> + <p>May raise exception <c>error:notsup</c> in case the chosen <c>Type</c> + is not supported by the underlying libcrypto implementation.</p> + <p>For keylengths and blocksizes see the + <seealso marker="crypto:algorithm_details#ciphers">User's Guide</seealso>. + </p> + </desc> + </func> + + <func> + <name since="OTP R16B01">block_encrypt(Type, Key, Ivec, PlainText) -> CipherText | Error</name> + <name since="OTP R16B01">block_encrypt(AeadType, Key, Ivec, {AAD, PlainText}) -> {CipherText, CipherTag} | Error</name> + <name since="OTP R16B01">block_encrypt(aes_gcm | aes_ccm, Key, Ivec, {AAD, PlainText, TagLength}) -> {CipherText, CipherTag} | Error </name> + <fsummary>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher</fsummary> + <type> + <v>Type = <seealso marker="#type-block_cipher_with_iv">block_cipher_with_iv()</seealso></v> + <v>AeadType = <seealso marker="#type-aead_cipher">aead_cipher()</seealso></v> + <v>Key = <seealso marker="#type-key">key()</seealso> | <seealso marker="#type-des3_key">des3_key()</seealso></v> + <v>PlainText = iodata()</v> + <v>AAD = IVec = CipherText = CipherTag = binary()</v> + <v>TagLength = 1..16</v> + <v>Error = <seealso marker="#type-run_time_error">run_time_error()</seealso></v> + </type> + <desc> + <dont><p>Don't use this function for new programs! Use <seealso marker="crypto:new_api">the new api</seealso>.</p></dont> + <p>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher. + <c>IVec</c> is an arbitrary initializing vector.</p> + <p>In AEAD (Authenticated Encryption with Associated Data) mode, encrypt + <c>PlainText</c>according to <c>Type</c> block cipher and calculate + <c>CipherTag</c> that also authenticates the <c>AAD</c> (Associated Authenticated Data).</p> + <p>May raise exception <c>error:notsup</c> in case the chosen <c>Type</c> + is not supported by the underlying libcrypto implementation.</p> + <p>For keylengths, iv-sizes and blocksizes see the + <seealso marker="crypto:algorithm_details#ciphers">User's Guide</seealso>. + </p> + </desc> + </func> + + <func> + <name since="OTP R16B01">block_decrypt(Type, Key, Ivec, CipherText) -> PlainText | Error</name> + <name since="OTP R16B01">block_decrypt(AeadType, Key, Ivec, {AAD, CipherText, CipherTag}) -> PlainText | Error</name> + <fsummary>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher</fsummary> + <type> + <v>Type = <seealso marker="#type-block_cipher_with_iv">block_cipher_with_iv()</seealso></v> + <v>AeadType = <seealso marker="#type-aead_cipher">aead_cipher()</seealso></v> + <v>Key = <seealso marker="#type-key">key()</seealso> | <seealso marker="#type-des3_key">des3_key()</seealso></v> + <v>PlainText = iodata()</v> + <v>AAD = IVec = CipherText = CipherTag = binary()</v> + <v>Error = BadTag | <seealso marker="#type-run_time_error">run_time_error()</seealso></v> + <v>BadTag = error</v> + </type> + <desc> + <dont><p>Don't use this function for new programs! Use <seealso marker="crypto:new_api">the new api</seealso>.</p></dont> + <p>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher. + <c>IVec</c> is an arbitrary initializing vector.</p> + <p>In AEAD (Authenticated Encryption with Associated Data) mode, decrypt + <c>CipherText</c>according to <c>Type</c> block cipher and check the authenticity + the <c>PlainText</c> and <c>AAD</c> (Associated Authenticated Data) using the + <c>CipherTag</c>. May return <c>error</c> if the decryption or validation fail's</p> + <p>May raise exception <c>error:notsup</c> in case the chosen <c>Type</c> + is not supported by the underlying libcrypto implementation.</p> + <p>For keylengths, iv-sizes and blocksizes see the + <seealso marker="crypto:algorithm_details#ciphers">User's Guide</seealso>. + </p> + </desc> + </func> + + <func> + <name name="stream_init" arity="2" since="OTP R16B01"/> + <fsummary></fsummary> + <desc> + <dont><p>Don't use this function for new programs! Use <seealso marker="crypto:new_api">the new api</seealso>.</p></dont> + <p>Initializes the state for use in RC4 stream encryption + <seealso marker="#stream_encrypt-2">stream_encrypt</seealso> and + <seealso marker="#stream_decrypt-2">stream_decrypt</seealso></p> + <p>For keylengths see the + <seealso marker="crypto:algorithm_details#stream-ciphers">User's Guide</seealso>. + </p> + </desc> + </func> + + <func> + <name name="stream_init" arity="3" since="OTP R16B01"/> + <fsummary></fsummary> + <desc> + <dont><p>Don't use this function for new programs! Use <seealso marker="crypto:new_api">the new api</seealso>.</p></dont> + <p>Initializes the state for use in streaming AES encryption using Counter mode (CTR). + <c>Key</c> is the AES key and must be either 128, 192, or 256 bits long. <c>IVec</c> is + an arbitrary initializing vector of 128 bits (16 bytes). This state is for use with + <seealso marker="#stream_encrypt-2">stream_encrypt</seealso> and + <seealso marker="#stream_decrypt-2">stream_decrypt</seealso>.</p> + <p>For keylengths and iv-sizes see the + <seealso marker="crypto:algorithm_details#stream-ciphers">User's Guide</seealso>. + </p> + </desc> + </func> + + <func> + <name name="stream_encrypt" arity="2" since="OTP R16B01"/> + <fsummary></fsummary> + <desc> + <dont><p>Don't use this function for new programs! Use <seealso marker="crypto:new_api">the new api</seealso>.</p></dont> + <p>Encrypts <c>PlainText</c> according to the stream cipher <c>Type</c> specified in stream_init/3. + <c>Text</c> can be any number of bytes. The initial <c>State</c> is created using + <seealso marker="#stream_init-2">stream_init</seealso>. + <c>NewState</c> must be passed into the next call to <c>stream_encrypt</c>.</p> + </desc> + </func> + + <func> + <name name="stream_decrypt" arity="2" since="OTP R16B01"/> + <fsummary></fsummary> + <desc> + <dont><p>Don't use this function for new programs! Use <seealso marker="crypto:new_api">the new api</seealso>.</p></dont> + <p>Decrypts <c>CipherText</c> according to the stream cipher <c>Type</c> specified in stream_init/3. + <c>PlainText</c> can be any number of bytes. The initial <c>State</c> is created using + <seealso marker="#stream_init-2">stream_init</seealso>. + <c>NewState</c> must be passed into the next call to <c>stream_decrypt</c>.</p> + </desc> + </func> + + <func> + <name name="supports" arity="0" since="OTP R16B01"/> + <fsummary>Provide a list of available crypto algorithms.</fsummary> + <desc> + <dont><p>Don't use this function for new programs! Use + <seealso marker="crypto#supports-1">supports/1</seealso> in + <seealso marker="crypto:new_api">the new api</seealso>.</p></dont> + <p> Can be used to determine which crypto algorithms that are supported + by the underlying libcrypto library</p> + <p>See <seealso marker="#hash_info-1">hash_info/1</seealso> and <seealso marker="#cipher_info-1">cipher_info/1</seealso> + for information about the hash and cipher algorithms. + </p> + </desc> + </func> + + </funcs> + </erlref> diff --git a/lib/crypto/doc/src/engine_keys.xml b/lib/crypto/doc/src/engine_keys.xml index 5ac690eb90..f78bb81bba 100644 --- a/lib/crypto/doc/src/engine_keys.xml +++ b/lib/crypto/doc/src/engine_keys.xml @@ -40,7 +40,7 @@ </p> <p> An engine could among other tasks provide a storage for - private or public keys. Such a storage could be made safer than the normal file system. Thoose techniques are not + private or public keys. Such a storage could be made safer than the normal file system. Those techniques are not described in this User's Guide. Here we concentrate on how to use private or public keys stored in such an engine. </p> diff --git a/lib/crypto/doc/src/new_api.xml b/lib/crypto/doc/src/new_api.xml new file mode 100644 index 0000000000..bd2334ac9f --- /dev/null +++ b/lib/crypto/doc/src/new_api.xml @@ -0,0 +1,260 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2014</year><year>2019</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>New and Old API</title> + <prepared>Hans Nilsson</prepared> + <docno></docno> + <date>2019-08-22</date> + <rev>A</rev> + <file>new_api.xml</file> + </header> + <p> + This chapter describes the new api to encryption and decryption. + </p> + + <section> + <title>Background</title> + <p>The CRYPTO app has evolved during its lifetime. Since also the OpenSSL cryptolib has changed the + API several times, there are parts of the CRYPTO app that uses a very old one internally and + other parts that uses the latest one. The internal definitions of e.g cipher names was a bit hard + to maintain. + </p> + <p>It turned out that using the old api in the new way (more about that later), and still keep it + backwards compatible, was not possible. Specially as more precision in the error messages was wanted + it could not be combined with the old standard. + </p> + <p>Therefore the old api (see next section) is kept for now but internally implemented with new primitives. + </p> + </section> + + <section> + <title>The old API</title> + <p>The old functions - not recommended for new programs - are:</p> + <list> + <item><seealso marker="crypto#block_encrypt-3">block_encrypt/3</seealso></item> + <item><seealso marker="crypto#block_encrypt-4">block_encrypt/4</seealso></item> + <item><seealso marker="crypto#block_decrypt-3">block_decrypt/3</seealso></item> + <item><seealso marker="crypto#block_decrypt-4">block_decrypt/4</seealso></item> + <item><seealso marker="crypto#stream_init-2">stream_init/2</seealso></item> + <item><seealso marker="crypto#stream_init-2">stream_init/3</seealso></item> + <item><seealso marker="crypto#stream_encrypt-2">stream_encrypt/2</seealso></item> + <item><seealso marker="crypto#stream_decrypt-2">stream_decrypt/2</seealso></item> + <item><seealso marker="crypto#supports-0">supports/0</seealso></item> + </list> + <p>They are not deprecated for now, but may be in a future release. + </p> + </section> + + <section> + <title>The new API</title> + <p>The new functions for encrypting or decrypting one single binary are: + </p> + <list> + <item><seealso marker="crypto#crypto_one_time/4">crypto_one_time/4</seealso></item> + <item><seealso marker="crypto#crypto_one_time/5">crypto_one_time/5</seealso></item> + <item><seealso marker="crypto#crypto_one_time_aead/6">crypto_one_time_aead/6</seealso></item> + <item><seealso marker="crypto#crypto_one_time_aead/7">crypto_one_time_aead/7</seealso></item> + </list> + <p>In those functions the internal crypto state is first created and initialized + with the cipher type, the key and possibly other data. Then the single binary is encrypted + or decrypted, + the crypto state is de-allocated and the result of the crypto operation is returned. + </p> + <p>The <c>crypto_one_time_aead</c> functions are for the ciphers of mode <c>ccm</c> or + <c>gcm</c>, and for the cipher <c>chacha20-poly1305</c>. + </p> + <p>For repeated encryption or decryption of a text divided in parts, where the internal + crypto state is initialized once, and then many binaries are encrypted or decrypted with + the same state, the functions are: + </p> + <list> + <item><seealso marker="crypto#crypto_init/4">crypto_init/4</seealso></item> + <item><seealso marker="crypto#crypto_init/3">crypto_init/3</seealso></item> + <item><seealso marker="crypto#crypto_update/2">crypto_update/2</seealso></item> + </list> + <p>The <c>crypto_init</c> initialies an internal cipher state, and one or more calls of + <c>crypto_update</c> does the acual encryption or decryption. Note that AEAD ciphers + can't be handled this way due to their nature. + </p> + <p>For repeated encryption or decryption of a text divided in parts where the + same cipher and same key is used, but a new initialization vector (nounce) should be applied + for each part, the functions are: + </p> + <list> + <item><seealso marker="crypto#crypto_dyn_iv_init/3">crypto_dyn_iv_init/3</seealso></item> + <item><seealso marker="crypto#crypto_dyn_iv_update/3">crypto_dyn_iv_update/3</seealso></item> + </list> + <p>An example of where those functions are needed, is when handling the TLS protocol.</p> + <p>For information about available algorithms, use: + </p> + <list> + <item><seealso marker="crypto#supports-1">supports/1</seealso></item> + <item><seealso marker="crypto#hash_info-1">hash_info/1</seealso></item> + <item><seealso marker="crypto#cipher_info-1">cipher_info/1</seealso></item> + </list> + + <section> + <title>Examples of crypto_init/4 and crypto_update/2</title> + <p>The functions <seealso marker="crypto#crypto_init/4">crypto_init/4</seealso> + and <seealso marker="crypto#crypto_update/2">crypto_update/2</seealso> are intended + to be used for encrypting or decrypting a sequence of blocks. First one call of + <c>crypto_init/4</c> initialises the crypto context. One or more calls <c>crypto_update/2</c> + does the actual encryption or decryption for each block. + </p> + <p>This example shows first the encryption of two blocks and then decryptions of the cipher + text, but divided into three blocks just to show that it is possible to divide the plain text and + cipher text differently for some ciphers:</p> + <code type="erl"> + 1> crypto:start(). + ok + 2> Key = <<1:128>>. + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1>> + 3> IV = <<0:128>>. + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>> + 4> StateEnc = crypto:crypto_init(aes_128_ctr, Key, IV, true). % encrypt -> true + #Ref<0.3768901617.1128660993.124047> + 5> crypto:crypto_update(StateEnc, <<"First bytes">>). + <<67,44,216,166,25,130,203,5,66,6,162>> + 6> crypto:crypto_update(StateEnc, <<"Second bytes">>). + <<16,79,94,115,234,197,94,253,16,144,151,41>> + 7> + 7> StateDec = crypto:crypto_init(aes_128_ctr, Key, IV, false). % decrypt -> false + #Ref<0.3768901617.1128660994.124255> + 8> crypto:crypto_update(StateDec, <<67,44,216,166,25,130,203>>). + <<"First b">> + 9> crypto:crypto_update(StateDec, <<5,66,6,162,16,79,94,115,234,197, + 94,253,16,144,151>>). + <<"ytesSecond byte">> + 10> crypto:crypto_update(StateDec, <<41>>). + <<"s">> + 11> + </code> + <p>Note that the internal data that the <c>StateEnc</c> and <c>StateDec</c> references are + destructivly updated by the calls to <seealso marker="crypto#crypto_update/2">crypto_update/2</seealso>. + This is to gain time in the calls of the nifs interfacing the cryptolib. In a loop where the + state is saved in the loop's state, it also saves one update of the loop state per crypto operation. + </p> + <p>For example, a simple server receiving text parts to encrypt and send the result back to the + one who sent them (the <c>Requester</c>): + </p> + <code type="erl"> + encode(Crypto, Key, IV) -> + crypto_loop(crypto:crypto_init(Crypto, Key, IV, true)). + + crypto_loop(State) -> + receive + {Text, Requester} -> + Requester ! crypto:crypto_update(State, Text), + loop(State) + end. + </code> + </section> + + <section> + <title>Example of crypto_one_time/5</title> + <p>The same example as in the + <seealso marker="#examples-of-crypto_init-4-and-crypto_update-2">previous section</seealso>, + but now with one call to <seealso marker="crypto#crypto_one_time/5">crypto_one_time/5</seealso>: + </p> + <code> + 1> Key = <<1:128>>. + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1>> + 2> IV = <<0:128>>. + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>> + 3> Txt = [<<"First bytes">>,<<"Second bytes">>]. + [<<"First bytes">>,<<"Second bytes">>] + 4> crypto:crypto_one_time(aes_128_ctr, Key, IV, Txt, true). + <<67,44,216,166,25,130,203,5,66,6,162,16,79,94,115,234, + 197,94,253,16,144,151,41>> + 5> + </code> + <p>The <c>[<<"First bytes">>,<<"Second bytes">>]</c> could of course have been one + single binary: <c><<"First bytesSecond bytes">></c>. + </p> + </section> + + <section> + <title>Example of crypto_one_time_aead/6</title> + <p>The same example as in the + <seealso marker="#example-of-crypto_one_time-5">previous section</seealso>, + but now with one call to <seealso marker="crypto#crypto_one_time_aead/6">crypto_one_time_aead/6</seealso>: + </p> + <code> + 1> Key = <<1:128>>. + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1>> + 2> IV = <<0:128>>. + <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>> + 3> Txt = [<<"First bytes">>,<<"Second bytes">>]. + [<<"First bytes">>,<<"Second bytes">>] + 4> AAD = <<"Some bytes">>. + <<"Some bytes">> + 5> crypto:crypto_one_time_aead(aes_128_gcm, Key, IV, Txt, AAD, true). + {<<240,130,38,96,130,241,189,52,3,190,179,213,132,1,72, + 192,103,176,90,104,15,71,158>>, + <<131,47,45,91,142,85,9,244,21,141,214,71,31,135,2,155>>} + 9> + </code> + <p>The <c>[<<"First bytes">>,<<"Second bytes">>]</c> could of course have been one + single binary: <c><<"First bytesSecond bytes">></c>. + </p> + </section> + + </section> + + <section> + <title>Retired cipher names</title> + <p>This table lists the retired cipher names in the first column and suggests names to replace them with + in the second column. + </p> + <p>The new names follows the OpenSSL libcrypto names. The format is ALGORITM_KEYSIZE_MODE. + </p> + <p>Examples of algorithms are aes, chacha20 and des. The keysize is the number of bits + and examples of the mode are cbc, ctr and gcm. The mode may be followed by a number depending + on the mode. An example is the ccm mode which has a variant called ccm8 where the so called tag + has a length of eight bits. + </p> + <p>The old names had by time lost any common naming which the new names now introduces. The new names include + the key length which improves the error checking in the lower levels of the crypto application. + </p> + + <table> + <row><cell><strong>Instead of:</strong></cell> <cell><strong>Use:</strong> </cell></row> + + <row><cell><c>aes_cbc128</c> </cell> <cell> <c>aes_128_cbc</c> </cell></row> + <row><cell><c>aes_cbc256</c> </cell> <cell> <c>aes_256_cbc</c> </cell></row> + <row><cell><c>aes_cbc</c> </cell> <cell> <c>aes_128_cbc, aes_192_cbc, aes_256_cbc</c></cell></row> + <row><cell><c>aes_ccm</c> </cell> <cell> <c>aes_128_ccm, aes_192_ccm, aes_256_ccm</c></cell></row> + <row><cell><c>aes_cfb128</c> </cell> <cell> <c>aes_128_cfb128, aes_192_cfb128, aes_256_cfb128</c></cell></row> + <row><cell><c>aes_cfb8</c> </cell> <cell> <c>aes_128_cfb8, aes_192_cfb8, aes_256_cfb8</c></cell></row> + <row><cell><c>aes_ctr</c> </cell> <cell> <c>aes_128_ctr, aes_192_ctr, aes_256_ctr</c></cell></row> + <row><cell><c>aes_gcm</c> </cell> <cell> <c>aes_128_gcm, aes_192_gcm, aes_256_gcm</c></cell></row> + <row><cell><c>des3_cbc</c> </cell> <cell> <c>des_ede3_cbc</c></cell></row> + <row><cell><c>des3_cbf</c> </cell> <cell> <c>des_ede3_cfb</c></cell></row> + <row><cell><c>des3_cfb</c> </cell> <cell> <c>des_ede3_cfb</c></cell></row> + <row><cell><c>des_ede3</c> </cell> <cell> <c>des_ede3_cbc</c></cell></row> + <row><cell><c>des_ede3_cbf</c> </cell> <cell> <c>des_ede3_cfb</c></cell></row> + <tcaption></tcaption> + </table> + </section> + +</chapter> diff --git a/lib/crypto/doc/src/usersguide.xml b/lib/crypto/doc/src/usersguide.xml index 2dfc966609..134f900d4c 100644 --- a/lib/crypto/doc/src/usersguide.xml +++ b/lib/crypto/doc/src/usersguide.xml @@ -51,4 +51,5 @@ <xi:include href="engine_load.xml"/> <xi:include href="engine_keys.xml"/> <xi:include href="algorithm_details.xml"/> + <xi:include href="new_api.xml"/> </part> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index bc8b124b10..8ffdde2b90 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -24,6 +24,7 @@ -export([start/0, stop/0, info_lib/0, info_fips/0, supports/0, enable_fips_mode/1, version/0, bytes_to_integer/1]). +-export([cipher_info/1, hash_info/1]). -export([hash/2, hash_init/1, hash_update/2, hash_final/1]). -export([sign/4, sign/5, verify/5, verify/6]). -export([generate_key/2, generate_key/3, compute_key/4]). @@ -31,21 +32,40 @@ -export([cmac/3, cmac/4]). -export([poly1305/2]). -export([exor/2, strong_rand_bytes/1, mod_pow/3]). --export([rand_seed/0, rand_seed_alg/1]). --export([rand_seed_s/0, rand_seed_alg_s/1]). +-export([rand_seed/0, rand_seed_alg/1, rand_seed_alg/2]). +-export([rand_seed_s/0, rand_seed_alg_s/1, rand_seed_alg_s/2]). -export([rand_plugin_next/1]). +-export([rand_plugin_aes_next/1, rand_plugin_aes_jump/1]). -export([rand_plugin_uniform/1]). -export([rand_plugin_uniform/2]). -export([rand_cache_plugin_next/1]). -export([rand_uniform/2]). --export([block_encrypt/3, block_decrypt/3, block_encrypt/4, block_decrypt/4]). -export([next_iv/2, next_iv/3]). --export([stream_init/2, stream_init/3, stream_encrypt/2, stream_decrypt/2]). -export([public_encrypt/4, private_decrypt/4]). -export([private_encrypt/4, public_decrypt/4]). -export([privkey_to_pubkey/2]). -export([ec_curve/1, ec_curves/0]). -export([rand_seed/1]). + +%% Old interface. Now implemented with the New interface +-export([stream_init/2, stream_init/3, + stream_encrypt/2, + stream_decrypt/2, + block_encrypt/3, block_encrypt/4, + block_decrypt/3, block_decrypt/4 + ]). + +%% New interface +-export([crypto_init/4, crypto_init/3, + crypto_update/2, + crypto_one_time/4, crypto_one_time/5, + crypto_one_time_aead/6, crypto_one_time_aead/7, + crypto_dyn_iv_init/3, + crypto_dyn_iv_update/3, + supports/1 + ]). + + %% Engine -export([ engine_get_all_methods/0, @@ -88,11 +108,14 @@ -export_type([ stream_state/0, hmac_state/0, - hash_state/0 + hash_state/0, + crypto_state/0 ]). %% Private. For tests. --export([packed_openssl_version/4, engine_methods_convert_to_bitmask/2, get_test_engine/0]). +-export([packed_openssl_version/4, engine_methods_convert_to_bitmask/2, + get_test_engine/0]). +-export([rand_plugin_aes_jump_2pow20/1]). -deprecated({rand_uniform, 2, next_major_release}). @@ -255,23 +278,153 @@ -type edwards_curve_ed() :: ed25519 | ed448 . -%%% +%%%---------------------------------------------------------------- +%%% New cipher schema +%%% +-type cipher() :: cipher_no_iv() + | cipher_iv() + | cipher_aead() . + +-type cipher_no_iv() :: aes_128_ecb + | aes_192_ecb + | aes_256_ecb + + | blowfish_ecb + | des_ecb + | rc4 . + +-type cipher_iv() :: aes_128_cbc + | aes_192_cbc + | aes_256_cbc + + | aes_128_cfb128 + | aes_192_cfb128 + | aes_256_cfb128 + + | aes_128_cfb8 + | aes_192_cfb8 + | aes_256_cfb8 + + | aes_128_ctr + | aes_192_ctr + | aes_256_ctr + + | aes_ige256 + + | blowfish_cbc + | blowfish_cfb64 + | blowfish_ofb64 + | chacha20 + | des_ede3_cbc + | des_ede3_cfb + + | des_cbc + | des_cfb + | rc2_cbc . + + +-type cipher_aead() :: aes_128_ccm + | aes_192_ccm + | aes_256_ccm + + | aes_128_gcm + | aes_192_gcm + | aes_256_gcm + + | chacha20_poly1305 . + + +%% -type retired_cipher_no_iv_aliases() :: aes_ecb . + +%% -type retired_cipher_iv_aliases() :: aes_cbc +%% | aes_cbc128 % aes_128_cbc +%% | aes_cbc256 % aes_256_cbc +%% | aes_cfb128 +%% | aes_cfb8 +%% | aes_ctr +%% | des3_cbc % des_ede3_cbc +%% | des_ede3 % des_ede3_cbc +%% | des_ede3_cbf % des_ede3_cfb +%% | des3_cbf % des_ede3_cfb +%% | des3_cfb . % des_ede3_cfb + +%% -type retired_cipher_aead_aliases() :: aes_ccm +%% | aes_gcm . + +%%%---------------------------------------------------------------- +%%% Old cipher scheme +%%% +%%% +-type block_cipher_without_iv() :: ecb_cipher() . + -type block_cipher_with_iv() :: cbc_cipher() | cfb_cipher() - | aes_cbc128 - | aes_cbc256 - | aes_ige256 | blowfish_ofb64 - | des3_cbf % cfb misspelled - | des_ede3 - | rc2_cbc . - --type cbc_cipher() :: des_cbc | des3_cbc | aes_cbc | blowfish_cbc . + | aes_ige256 . + +-type stream_cipher() :: ctr_cipher() + | chacha20 + | rc4 . + + +%%%---- +-type cbc_cipher() :: aes_128_cbc + | aes_192_cbc + | aes_256_cbc + | blowfish_cbc + | des_cbc + | des_ede3_cbc + | rc2_cbc + | retired_cbc_cipher_aliases() . + +-type retired_cbc_cipher_aliases() :: aes_cbc % aes_*_cbc + | aes_cbc128 % aes_128_cbc + | aes_cbc256 % aes_256_cbc + | des3_cbc % des_ede3_cbc + | des_ede3 . % des_ede3_cbc + +%%%---- +-type cfb_cipher() :: aes_128_cfb128 + | aes_192_cfb128 + | aes_256_cfb128 + | aes_128_cfb8 + | aes_192_cfb8 + | aes_256_cfb8 + | blowfish_cfb64 + | des_cfb + | des_ede3_cfb + | retired_cfb_cipher_aliases() . + +-type retired_cfb_cipher_aliases() :: aes_cfb8 % aes_*_cfb8 + | aes_cfb128 % aes_*_cfb128 + | des3_cbf % des_ede3_cfb, cfb misspelled + | des3_cfb % des_ede3_cfb + | des_ede3_cbf .% cfb misspelled + + +%%%---- +-type ctr_cipher() :: aes_128_ctr + | aes_192_ctr + | aes_256_ctr + | retired_ctr_cipher_aliases() . + +-type retired_ctr_cipher_aliases() :: aes_ctr . % aes_*_ctr + +%%%---- +-type ecb_cipher() :: aes_128_ecb + | aes_192_ecb + | aes_256_ecb + | blowfish_ecb + | retired_ecb_cipher_aliases() . + +-type retired_ecb_cipher_aliases() :: aes_ecb . + +%%%---- -type aead_cipher() :: aes_gcm | aes_ccm | chacha20_poly1305 . --type cfb_cipher() :: aes_cfb128 | aes_cfb8 | blowfish_cfb64 | des3_cfb | des_cfb . --type block_cipher_without_iv() :: ecb_cipher() . --type ecb_cipher() :: des_ecb | blowfish_ecb | aes_ecb . + +%%%----- end old cipher schema ------------------------------------ +%%%---------------------------------------------------------------- -type key() :: iodata(). -type des3_key() :: [key()]. @@ -284,11 +437,26 @@ -type sha1() :: sha . -type sha2() :: sha224 | sha256 | sha384 | sha512 . -type sha3() :: sha3_224 | sha3_256 | sha3_384 | sha3_512 . +-type blake2() :: blake2b | blake2s . -type compatibility_only_hash() :: md5 | md4 . -type crypto_integer() :: binary() | integer(). +%%% +%% Exceptions +%% error:badarg +%% error:notsup +-type run_time_error() :: no_return(). + +%% Exceptions +%% error:{badarg,Reason::term()} +%% error:{notsup,Reason::term()} +%% error:{error,Reason::term()} +-type descriptive_error() :: no_return() . + + +%%-------------------------------------------------------------------- -compile(no_native). -on_load(on_load/0). -define(CRYPTO_NIF_VSN,302). @@ -326,24 +494,49 @@ stop() -> | {macs, Macs} | {curves, Curves} | {rsa_opts, RSAopts}, - Hashs :: [sha1() | sha2() | sha3() | ripemd160 | compatibility_only_hash()], - Ciphers :: [stream_cipher() - | block_cipher_with_iv() | block_cipher_without_iv() - | aead_cipher() - ], + Hashs :: [sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash()], + Ciphers :: [cipher()], + PKs :: [rsa | dss | ecdsa | dh | ecdh | ec_gf2m], + Macs :: [hmac | cmac | poly1305], + Curves :: [ec_named_curve() | edwards_curve_dh() | edwards_curve_ed()], + RSAopts :: [rsa_sign_verify_opt() | rsa_opt()] . +supports() -> + [{hashs, hash_algorithms()}, + {ciphers, prepend_old_aliases( cipher_algorithms())}, + {public_keys, pubkey_algorithms()}, + {macs, mac_algorithms()}, + {curves, curve_algorithms()}, + {rsa_opts, rsa_opts_algorithms()} + ]. + + +-spec supports(Type) -> Support + when Type :: hashs + | ciphers + | public_keys + | macs + | curves + | rsa_opts, + Support :: Hashs + | Ciphers + | PKs + | Macs + | Curves + | RSAopts, + Hashs :: [sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash()], + Ciphers :: [cipher()], PKs :: [rsa | dss | ecdsa | dh | ecdh | ec_gf2m], Macs :: [hmac | cmac | poly1305], Curves :: [ec_named_curve() | edwards_curve_dh() | edwards_curve_ed()], RSAopts :: [rsa_sign_verify_opt() | rsa_opt()] . -supports()-> - {Hashs, PubKeys, Ciphers, Macs, Curves, RsaOpts} = algorithms(), - [{hashs, Hashs}, - {ciphers, Ciphers}, - {public_keys, PubKeys}, - {macs, Macs}, - {curves, Curves}, - {rsa_opts, RsaOpts} - ]. + +supports(hashs) -> hash_algorithms(); +supports(public_keys) -> pubkey_algorithms(); +supports(ciphers) -> cipher_algorithms(); +supports(macs) -> mac_algorithms(); +supports(curves) -> curve_algorithms(); +supports(rsa_opts) -> rsa_opts_algorithms(). + -spec info_lib() -> [{Name,VerNum,VerStr}] when Name :: binary(), VerNum :: integer(), @@ -364,9 +557,18 @@ enable_fips_mode(_) -> ?nif_stub. %%% %%%================================================================ --define(HASH_HASH_ALGORITHM, sha1() | sha2() | sha3() | ripemd160 | compatibility_only_hash() ). +-type hash_algorithm() :: sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash() . + +-spec hash_info(Type) -> Result | run_time_error() + when Type :: hash_algorithm(), + Result :: #{size := integer(), + block_size := integer(), + type := integer() + } . +hash_info(Type) -> + notsup_to_error(hash_info_nif(Type)). --spec hash(Type, Data) -> Digest when Type :: ?HASH_HASH_ALGORITHM, +-spec hash(Type, Data) -> Digest when Type :: hash_algorithm(), Data :: iodata(), Digest :: binary(). hash(Type, Data) -> @@ -376,7 +578,7 @@ hash(Type, Data) -> -opaque hash_state() :: reference(). --spec hash_init(Type) -> State when Type :: ?HASH_HASH_ALGORITHM, +-spec hash_init(Type) -> State when Type :: hash_algorithm(), State :: hash_state(). hash_init(Type) -> notsup_to_error(hash_init_nif(Type)). @@ -402,12 +604,12 @@ hash_final(Context) -> %%%---- HMAC --define(HMAC_HASH_ALGORITHM, sha1() | sha2() | sha3() | compatibility_only_hash()). +-type hmac_hash_algorithm() :: sha1() | sha2() | sha3() | compatibility_only_hash(). %%%---- hmac/3,4 -spec hmac(Type, Key, Data) -> - Mac when Type :: ?HMAC_HASH_ALGORITHM, + Mac when Type :: hmac_hash_algorithm(), Key :: iodata(), Data :: iodata(), Mac :: binary() . @@ -416,7 +618,7 @@ hmac(Type, Key, Data) -> hmac(Type, Key, Data1, undefined, erlang:byte_size(Data1), max_bytes()). -spec hmac(Type, Key, Data, MacLength) -> - Mac when Type :: ?HMAC_HASH_ALGORITHM, + Mac when Type :: hmac_hash_algorithm(), Key :: iodata(), Data :: iodata(), MacLength :: integer(), @@ -431,7 +633,7 @@ hmac(Type, Key, Data, MacLength) -> -opaque hmac_state() :: binary(). -spec hmac_init(Type, Key) -> - State when Type :: ?HMAC_HASH_ALGORITHM, + State when Type :: hmac_hash_algorithm(), Key :: iodata(), State :: hmac_state() . hmac_init(Type, Key) -> @@ -469,7 +671,7 @@ hmac_final_n(Context, HashLen) -> Data :: iodata(), Mac :: binary(). cmac(Type, Key, Data) -> - notsup_to_error(cmac_nif(Type, Key, Data)). + notsup_to_error(cmac_nif(alias(Type), Key, Data)). -spec cmac(Type, Key, Data, MacLength) -> Mac when Type :: ?CMAC_CIPHER_ALGORITHM, @@ -477,8 +679,9 @@ cmac(Type, Key, Data) -> Data :: iodata(), MacLength :: integer(), Mac :: binary(). + cmac(Type, Key, Data, MacLength) -> - erlang:binary_part(cmac(Type, Key, Data), 0, MacLength). + erlang:binary_part(cmac(alias(Type), Key, Data), 0, MacLength). %%%---- POLY1305 @@ -489,97 +692,209 @@ poly1305(Key, Data) -> %%%================================================================ %%% -%%% Encrypt/decrypt +%%% Encrypt/decrypt, The "Old API" %%% %%%================================================================ -%%%---- Block ciphers +-define(COMPAT(CALL), + try begin CALL end + catch + error:{error, {_File,_Line}, _Reason} -> + error(badarg); + error:{E, {_File,_Line}, _Reason} when E==notsup ; E==badarg -> + error(E) + end). --spec block_encrypt(Type::block_cipher_with_iv(), Key::key()|des3_key(), Ivec::binary(), PlainText::iodata()) -> binary(); +%%%---- Cipher info +%%%---------------------------------------------------------------- +-spec cipher_info(Type) -> Result | run_time_error() + when Type :: cipher(), + Result :: #{key_length := integer(), + iv_length := integer(), + block_size := integer(), + mode := CipherModes, + type := undefined | integer() + }, + CipherModes :: undefined + | cbc_mode + | ccm_mode + | cfb_mode + | ctr_mode + | ecb_mode + | gcm_mode + | ige_mode + | ocb_mode + | ofb_mode + | wrap_mode + | xts_mode + . + +%% %% These ciphers are not available via the EVP interface on older cryptolibs. +cipher_info(aes_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 32,mode => ctr_mode,type => undefined}; +cipher_info(aes_128_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 16,mode => ctr_mode,type => undefined}; +cipher_info(aes_192_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 24,mode => ctr_mode,type => undefined}; +cipher_info(aes_256_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 32,mode => ctr_mode,type => undefined}; +%% %% This cipher is handled specialy. +cipher_info(aes_ige256) -> + #{block_size => 16,iv_length => 32,key_length => 16,mode => ige_mode,type => undefined}; +%% %% These ciphers belong to the "old" interface: +%% cipher_info(aes_cbc) -> +%% #{block_size => 16,iv_length => 16,key_length => 24,mode => cbc_mode,type => 423}; +%% cipher_info(aes_cbc128) -> +%% #{block_size => 16,iv_length => 16,key_length => 16,mode => cbc_mode,type => 419}; +%% cipher_info(aes_cbc256) -> +%% #{block_size => 16,iv_length => 16,key_length => 32,mode => cbc_mode,type => 427}; +%% cipher_info(aes_ccm) -> +%% #{block_size => 1,iv_length => 12,key_length => 24,mode => ccm_mode,type => 899}; +%% cipher_info(aes_cfb128) -> +%% #{block_size => 1,iv_length => 16,key_length => 32,mode => cfb_mode,type => 429}; +%% cipher_info(aes_cfb8) -> +%% #{block_size => 1,iv_length => 16,key_length => 32,mode => cfb_mode,type => 429}; +%% cipher_info(aes_ecb) -> +%% #{block_size => 16,iv_length => 0,key_length => 24,mode => ecb_mode,type => 422}; +%% cipher_info(aes_gcm) -> +%% #{block_size => 1,iv_length => 12,key_length => 24,mode => gcm_mode,type => 898}; +%% cipher_info(des3_cbc) -> +%% #{block_size => 8,iv_length => 8,key_length => 24,mode => cbc_mode,type => 44}; +%% cipher_info(des3_cbf) -> +%% #{block_size => 1,iv_length => 8,key_length => 24,mode => cfb_mode,type => 30}; +%% cipher_info(des3_cfb) -> +%% #{block_size => 1,iv_length => 8,key_length => 24,mode => cfb_mode,type => 30}; +%% cipher_info(des_ede3) -> +%% #{block_size => 8,iv_length => 8,key_length => 24,mode => cbc_mode,type => 44}; +%% cipher_info(des_ede3_cbf) -> +%% #{block_size => 1,iv_length => 8,key_length => 24,mode => cfb_mode,type => 30}; +cipher_info(Type) -> + cipher_info_nif(alias(Type)). + +%%%---- Block ciphers +%%%---------------------------------------------------------------- +-spec block_encrypt(Type::block_cipher_with_iv(), Key::key()|des3_key(), Ivec::binary(), PlainText::iodata()) -> + binary() | run_time_error(); (Type::aead_cipher(), Key::iodata(), Ivec::binary(), {AAD::binary(), PlainText::iodata()}) -> - {binary(), binary()}; + {binary(), binary()} | run_time_error(); (aes_gcm | aes_ccm, Key::iodata(), Ivec::binary(), {AAD::binary(), PlainText::iodata(), TagLength::1..16}) -> - {binary(), binary()}. - -block_encrypt(Type, Key, Ivec, PlainText) when Type =:= des_cbc; - Type =:= des_cfb; - Type =:= blowfish_cbc; - Type =:= blowfish_cfb64; - Type =:= blowfish_ofb64; - Type =:= aes_cbc128; - Type =:= aes_cfb8; - Type =:= aes_cfb128; - Type =:= aes_cbc256; - Type =:= aes_cbc; - Type =:= rc2_cbc -> - notsup_to_error(block_crypt_nif(Type, Key, Ivec, PlainText, true)); -block_encrypt(Type, Key0, Ivec, PlainText) when Type =:= des3_cbc; - Type =:= des_ede3 -> - Key = check_des3_key(Key0), - notsup_to_error(block_crypt_nif(des_ede3_cbc, Key, Ivec, PlainText, true)); -block_encrypt(des3_cbf, Key0, Ivec, PlainText) -> % cfb misspelled - Key = check_des3_key(Key0), - notsup_to_error(block_crypt_nif(des_ede3_cbf, Key, Ivec, PlainText, true)); -block_encrypt(des3_cfb, Key0, Ivec, PlainText) -> - Key = check_des3_key(Key0), - notsup_to_error(block_crypt_nif(des_ede3_cfb, Key, Ivec, PlainText, true)); + {binary(), binary()} | run_time_error(). + + block_encrypt(aes_ige256, Key, Ivec, PlainText) -> notsup_to_error(aes_ige_crypt_nif(Key, Ivec, PlainText, true)); -block_encrypt(Type, Key, Ivec, {AAD, PlainText}) when Type =:= aes_gcm; - Type =:= aes_ccm -> - aead_encrypt(Type, Key, Ivec, AAD, PlainText); -block_encrypt(Type, Key, Ivec, {AAD, PlainText, TagLength}) when Type =:= aes_gcm; - Type =:= aes_ccm -> - aead_encrypt(Type, Key, Ivec, AAD, PlainText, TagLength); -block_encrypt(chacha20_poly1305=Type, Key, Ivec, {AAD, PlainText}) -> - aead_encrypt(Type, Key, Ivec, AAD, PlainText, 16). +block_encrypt(Type, Key0, Ivec, Data) -> + Key = iolist_to_binary(Key0), + ?COMPAT( + case Data of + {AAD, PlainText} -> + crypto_one_time_aead(alias(Type,Key), Key, Ivec, PlainText, AAD, true); + {AAD, PlainText, TagLength} -> + crypto_one_time_aead(alias(Type,Key), Key, Ivec, PlainText, AAD, TagLength, true); + PlainText -> + crypto_one_time(alias(Type,Key), Key, Ivec, PlainText, true) + end). --spec block_decrypt(Type::block_cipher_with_iv(), Key::key()|des3_key(), Ivec::binary(), Data::iodata()) -> binary(); - (Type::aead_cipher(), Key::iodata(), Ivec::binary(), - {AAD::binary(), Data::iodata(), Tag::binary()}) -> binary() | error. -block_decrypt(Type, Key, Ivec, Data) when Type =:= des_cbc; - Type =:= des_cfb; - Type =:= blowfish_cbc; - Type =:= blowfish_cfb64; - Type =:= blowfish_ofb64; - Type =:= aes_cbc; - Type =:= aes_cbc128; - Type =:= aes_cfb8; - Type =:= aes_cfb128; - Type =:= aes_cbc256; - Type =:= rc2_cbc -> - notsup_to_error(block_crypt_nif(Type, Key, Ivec, Data, false)); -block_decrypt(Type, Key0, Ivec, Data) when Type =:= des3_cbc; - Type =:= des_ede3 -> - Key = check_des3_key(Key0), - notsup_to_error(block_crypt_nif(des_ede3_cbc, Key, Ivec, Data, false)); -block_decrypt(des3_cbf, Key0, Ivec, Data) -> % cfb misspelled - Key = check_des3_key(Key0), - notsup_to_error(block_crypt_nif(des_ede3_cbf, Key, Ivec, Data, false)); -block_decrypt(des3_cfb, Key0, Ivec, Data) -> - Key = check_des3_key(Key0), - notsup_to_error(block_crypt_nif(des_ede3_cfb, Key, Ivec, Data, false)); -block_decrypt(aes_ige256, Key, Ivec, Data) -> - notsup_to_error(aes_ige_crypt_nif(Key, Ivec, Data, false)); -block_decrypt(Type, Key, Ivec, {AAD, Data, Tag}) when Type =:= aes_gcm; - Type =:= aes_ccm; - Type =:= chacha20_poly1305 -> - aead_decrypt(Type, Key, Ivec, AAD, Data, Tag). +-spec block_encrypt(Type::block_cipher_without_iv(), Key::key(), PlainText::iodata()) -> + binary() | run_time_error(). +block_encrypt(Type, Key0, PlainText) -> + Key = iolist_to_binary(Key0), + ?COMPAT(crypto_one_time(alias(Type,Key), Key, PlainText, true)). --spec block_encrypt(Type::block_cipher_without_iv(), Key::key(), PlainText::iodata()) -> binary(). -block_encrypt(Type, Key, PlainText) -> - notsup_to_error(block_crypt_nif(Type, Key, PlainText, true)). +%%%---------------------------------------------------------------- +%%%---------------------------------------------------------------- +-spec block_decrypt(Type::block_cipher_with_iv(), Key::key()|des3_key(), Ivec::binary(), Data::iodata()) -> + binary() | run_time_error(); + (Type::aead_cipher(), Key::iodata(), Ivec::binary(), + {AAD::binary(), Data::iodata(), Tag::binary()}) -> + binary() | error | run_time_error() . +block_decrypt(aes_ige256, Key, Ivec, Data) -> + notsup_to_error(aes_ige_crypt_nif(Key, Ivec, Data, false)); + +block_decrypt(Type, Key0, Ivec, Data) -> + Key = iolist_to_binary(Key0), + ?COMPAT( + case Data of + {AAD, CryptoText, Tag} -> + crypto_one_time_aead(alias(Type,Key), Key, Ivec, CryptoText, AAD, Tag, false); + CryptoText -> + crypto_one_time(alias(Type,Key), Key, Ivec, CryptoText, false) + end). + + +-spec block_decrypt(Type::block_cipher_without_iv(), Key::key(), Data::iodata()) -> + binary() | run_time_error(). + +block_decrypt(Type, Key0, CryptoText) -> + Key = iolist_to_binary(Key0), + ?COMPAT(crypto_one_time(alias(Type,Key), Key, CryptoText, false)). + +%%%-------- Stream ciphers API + +-opaque stream_state() :: {stream_cipher(), + crypto_state() | {crypto_state(),flg_undefined} + }. + +%%%---- stream_init +-spec stream_init(Type, Key, IVec) -> State | run_time_error() + when Type :: stream_cipher(), + Key :: iodata(), + IVec ::binary(), + State :: stream_state() . +stream_init(Type, Key0, IVec) when is_binary(IVec) -> + Key = iolist_to_binary(Key0), + Ref = ?COMPAT(ng_crypto_init_nif(alias(Type,Key), + Key, iolist_to_binary(IVec), + undefined) + ), + {Type, {Ref,flg_undefined}}. + + +-spec stream_init(Type, Key) -> State | run_time_error() + when Type :: rc4, + Key :: iodata(), + State :: stream_state() . +stream_init(rc4 = Type, Key0) -> + Key = iolist_to_binary(Key0), + Ref = ?COMPAT(ng_crypto_init_nif(alias(Type,Key), + Key, <<>>, + undefined) + ), + {Type, {Ref,flg_undefined}}. + +%%%---- stream_encrypt +-spec stream_encrypt(State, PlainText) -> {NewState, CipherText} | run_time_error() + when State :: stream_state(), + PlainText :: iodata(), + NewState :: stream_state(), + CipherText :: iodata() . +stream_encrypt(State, Data) -> + crypto_stream_emulate(State, Data, true). --spec block_decrypt(Type::block_cipher_without_iv(), Key::key(), Data::iodata()) -> binary(). +%%%---- stream_decrypt +-spec stream_decrypt(State, CipherText) -> {NewState, PlainText} | run_time_error() + when State :: stream_state(), + CipherText :: iodata(), + NewState :: stream_state(), + PlainText :: iodata() . +stream_decrypt(State, Data) -> + crypto_stream_emulate(State, Data, false). -block_decrypt(Type, Key, Data) -> - notsup_to_error(block_crypt_nif(Type, Key, Data, false)). +%%%-------- helpers +crypto_stream_emulate({Cipher,{Ref0,flg_undefined}}, Data, EncryptFlag) when is_reference(Ref0) -> + ?COMPAT(begin + Ref = ng_crypto_init_nif(Ref0, <<>>, <<>>, EncryptFlag), + {{Cipher,Ref}, crypto_update(Ref, Data)} + end); +crypto_stream_emulate({Cipher,Ref}, Data, _) when is_reference(Ref) -> + ?COMPAT({{Cipher,Ref}, crypto_update(Ref, Data)}). +%%%---------------------------------------------------------------- -spec next_iv(Type:: cbc_cipher(), Data) -> NextIVec when % Type :: cbc_cipher(), %des_cbc | des3_cbc | aes_cbc | aes_ige, Data :: iodata(), NextIVec :: binary(). @@ -606,47 +921,274 @@ next_iv(des_cfb, Data, IVec) -> next_iv(Type, Data, _Ivec) -> next_iv(Type, Data). -%%%---- Stream ciphers +%%%================================================================ +%%% +%%% Encrypt/decrypt, The "New API" +%%% +%%%================================================================ --opaque stream_state() :: {stream_cipher(), reference()}. +-opaque crypto_state() :: reference() . --type stream_cipher() :: rc4 | aes_ctr | chacha20 . --spec stream_init(Type, Key, IVec) -> State when Type :: aes_ctr | chacha20, - Key :: iodata(), - IVec :: binary(), - State :: stream_state() . -stream_init(aes_ctr, Key, Ivec) -> - {aes_ctr, aes_ctr_stream_init(Key, Ivec)}; -stream_init(chacha20, Key, Ivec) -> - {chacha20, chacha20_stream_init(Key,Ivec)}. +%%%---------------------------------------------------------------- +%%% +%%% Create and initialize a new state for encryption or decryption +%%% --spec stream_init(Type, Key) -> State when Type :: rc4, - Key :: iodata(), - State :: stream_state() . -stream_init(rc4, Key) -> - {rc4, notsup_to_error(rc4_set_key(Key))}. +-spec crypto_init(Cipher, Key, EncryptFlag) -> State | descriptive_error() + when Cipher :: cipher_no_iv(), + Key :: iodata(), + EncryptFlag :: boolean(), + State :: crypto_state() . +crypto_init(Cipher, Key, EncryptFlag) -> + %% The IV is supposed to be supplied by calling crypto_update/3 + ng_crypto_init_nif(Cipher, iolist_to_binary(Key), <<>>, EncryptFlag). --spec stream_encrypt(State, PlainText) -> {NewState, CipherText} - when State :: stream_state(), - PlainText :: iodata(), - NewState :: stream_state(), - CipherText :: iodata() . -stream_encrypt(State, Data0) -> - Data = iolist_to_binary(Data0), - MaxByts = max_bytes(), - stream_crypt(fun do_stream_encrypt/2, State, Data, erlang:byte_size(Data), MaxByts, []). --spec stream_decrypt(State, CipherText) -> {NewState, PlainText} - when State :: stream_state(), - CipherText :: iodata(), - NewState :: stream_state(), - PlainText :: iodata() . -stream_decrypt(State, Data0) -> - Data = iolist_to_binary(Data0), - MaxByts = max_bytes(), - stream_crypt(fun do_stream_decrypt/2, State, Data, erlang:byte_size(Data), MaxByts, []). +-spec crypto_init(Cipher, Key, IV, EncryptFlag) -> State | descriptive_error() + when Cipher :: cipher_iv(), + Key :: iodata(), + IV :: iodata(), + EncryptFlag :: boolean(), + State :: crypto_state() . +crypto_init(Cipher, Key, IV, EncryptFlag) -> + ng_crypto_init_nif(Cipher, iolist_to_binary(Key), iolist_to_binary(IV), EncryptFlag). + + + +%%%---------------------------------------------------------------- +-spec crypto_dyn_iv_init(Cipher, Key, EncryptFlag) -> State | descriptive_error() + when Cipher :: cipher_iv(), + Key :: iodata(), + EncryptFlag :: boolean(), + State :: crypto_state() . +crypto_dyn_iv_init(Cipher, Key, EncryptFlag) -> + %% The IV is supposed to be supplied by calling crypto_update/3 + ng_crypto_init_nif(Cipher, iolist_to_binary(Key), undefined, EncryptFlag). + +%%%---------------------------------------------------------------- +%%% +%%% Encrypt/decrypt a sequence of bytes. The sum of the sizes +%%% of all blocks must be an integer multiple of the crypto's +%%% blocksize. +%%% + +-spec crypto_update(State, Data) -> Result | descriptive_error() + when State :: crypto_state(), + Data :: iodata(), + Result :: binary() . +crypto_update(State, Data0) -> + case iolist_to_binary(Data0) of + <<>> -> + <<>>; % Known to fail on OpenSSL 0.9.8h + Data -> + ng_crypto_update_nif(State, Data) + end. + + +%%%---------------------------------------------------------------- +-spec crypto_dyn_iv_update(State, Data, IV) -> Result | descriptive_error() + when State :: crypto_state(), + Data :: iodata(), + IV :: iodata(), + Result :: binary() . +crypto_dyn_iv_update(State, Data0, IV) -> + %% When State is from State = crypto_init(Cipher, Key, undefined, EncryptFlag) + case iolist_to_binary(Data0) of + <<>> -> + <<>>; % Known to fail on OpenSSL 0.9.8h + Data -> + ng_crypto_update_nif(State, Data, iolist_to_binary(IV)) + end. + +%%%---------------------------------------------------------------- +%%% +%%% Encrypt/decrypt one set bytes. +%%% The size must be an integer multiple of the crypto's blocksize. +%%% + +-spec crypto_one_time(Cipher, Key, Data, EncryptFlag) -> + Result | descriptive_error() + when Cipher :: cipher_no_iv(), + Key :: iodata(), + Data :: iodata(), + EncryptFlag :: boolean(), + Result :: binary() . +crypto_one_time(Cipher, Key, Data, EncryptFlag) -> + crypto_one_time(Cipher, Key, <<>>, Data, EncryptFlag). + +-spec crypto_one_time(Cipher, Key, IV, Data, EncryptFlag) -> + Result | descriptive_error() + when Cipher :: cipher_iv(), + Key :: iodata(), + IV :: iodata(), + Data :: iodata(), + EncryptFlag :: boolean(), + Result :: binary() . + +crypto_one_time(Cipher, Key, IV, Data0, EncryptFlag) -> + case iolist_to_binary(Data0) of + <<>> -> + <<>>; % Known to fail on OpenSSL 0.9.8h + Data -> + ng_crypto_one_time_nif(Cipher, + iolist_to_binary(Key), iolist_to_binary(IV), Data, + EncryptFlag) + end. + + +-spec crypto_one_time_aead(Cipher, Key, IV, InText, AAD, EncFlag::true) -> + Result | descriptive_error() + when Cipher :: cipher_aead(), + Key :: iodata(), + IV :: iodata(), + InText :: iodata(), + AAD :: iodata(), + Result :: EncryptResult, + EncryptResult :: {OutCryptoText, OutTag}, + OutCryptoText :: binary(), + OutTag :: binary(). + +crypto_one_time_aead(Cipher, Key, IV, PlainText, AAD, true) -> + crypto_one_time_aead(Cipher, Key, IV, PlainText, AAD, aead_tag_len(Cipher), true). + + +-spec crypto_one_time_aead(Cipher, Key, IV, InText, AAD, TagOrTagLength, EncFlag) -> + Result | descriptive_error() + when Cipher :: cipher_aead(), + Key :: iodata(), + IV :: iodata(), + InText :: iodata(), + AAD :: iodata(), + TagOrTagLength :: EncryptTagLength | DecryptTag, + EncryptTagLength :: non_neg_integer(), % or pos_integer() 1.. + DecryptTag :: iodata(), + EncFlag :: boolean(), + Result :: EncryptResult | DecryptResult, + EncryptResult :: {OutCryptoText, OutTag}, + DecryptResult :: OutPlainText | error, + OutCryptoText :: binary(), + OutTag :: binary(), + OutPlainText :: binary(). + +crypto_one_time_aead(Cipher, Key, IV, TextIn, AAD, TagOrTagLength, EncFlg) -> + aead_cipher(Cipher, Key, IV, TextIn, AAD, TagOrTagLength, EncFlg). + + +aead_tag_len(chacha20_poly1305) -> 16; +aead_tag_len(aes_ccm ) -> 12; +aead_tag_len(aes_128_ccm) -> 12; +aead_tag_len(aes_192_ccm) -> 12; +aead_tag_len(aes_256_ccm) -> 12; +aead_tag_len(aes_gcm ) -> 16; +aead_tag_len(aes_128_gcm) -> 16; +aead_tag_len(aes_192_gcm) -> 16; +aead_tag_len(aes_256_gcm) -> 16; +aead_tag_len(_) -> error({badarg, "Not an AEAD cipher"}). + +%%%---------------------------------------------------------------- +%%% NIFs + +-spec ng_crypto_init_nif(atom(), binary(), binary()|undefined, boolean()|undefined ) -> + crypto_state() | descriptive_error() + ; (crypto_state(), <<>>, <<>>, boolean()) + -> crypto_state() | descriptive_error(). + +ng_crypto_init_nif(_Cipher, _Key, _IVec, _EncryptFlg) -> ?nif_stub. + + +-spec ng_crypto_update_nif(crypto_state(), binary()) -> + binary() | descriptive_error() . +ng_crypto_update_nif(_State, _Data) -> ?nif_stub. + +-spec ng_crypto_update_nif(crypto_state(), binary(), binary()) -> + binary() | descriptive_error() . +ng_crypto_update_nif(_State, _Data, _IV) -> ?nif_stub. + + +-spec ng_crypto_one_time_nif(atom(), binary(), binary(), binary(), boolean() ) -> + binary() | descriptive_error(). +ng_crypto_one_time_nif(_Cipher, _Key, _IVec, _Data, _EncryptFlg) -> ?nif_stub. + +%%%---------------------------------------------------------------- +%%% Cipher aliases +%%% +-define(if_also(Cipher, Ciphers, AliasCiphers), + case lists:member(Cipher, Ciphers) of + true -> + AliasCiphers; + false -> + Ciphers + end). + + +prepend_old_aliases(L0) -> + L1 = ?if_also(des_ede3_cbc, L0, + [des3_cbc, des_ede3, des_ede3_cbf, des3_cbf, des3_cfb | L0]), + L2 = ?if_also(aes_128_cbc, L1, + [aes_cbc, aes_cbc128, aes_cbc256 | L1]), + L3 = ?if_also(aes_128_ctr, L2, + [aes_ctr | L2]), + L4 = ?if_also(aes_128_ccm, L3, + [aes_ccm | L3]), + L5 = ?if_also(aes_128_gcm, L4, + [aes_gcm | L4]), + L6 = ?if_also(aes_128_cfb8, L5, + [aes_cfb8 | L5]), + L7 = ?if_also(aes_128_cfb128, L6, + [aes_cfb128 | L6]), + L8 = ?if_also(aes_128_ecb, L7, + [aes_ecb | L7]), + L8. + + + +%%%---- des_ede3_cbc +alias(des3_cbc) -> des_ede3_cbc; +alias(des_ede3) -> des_ede3_cbc; +%%%---- des_ede3_cfb +alias(des_ede3_cbf) -> des_ede3_cfb; +alias(des3_cbf) -> des_ede3_cfb; +alias(des3_cfb) -> des_ede3_cfb; +%%%---- aes_*_cbc +alias(aes_cbc128) -> aes_128_cbc; +alias(aes_cbc256) -> aes_256_cbc; + +alias(Alg) -> Alg. + + +alias(Ciph, Key) -> alias2(alias(Ciph), Key). + +alias2(aes_cbc, Key) when size(Key)==16 -> aes_128_cbc; +alias2(aes_cbc, Key) when size(Key)==24 -> aes_192_cbc; +alias2(aes_cbc, Key) when size(Key)==32 -> aes_256_cbc; + +alias2(aes_cfb8, Key) when size(Key)==16 -> aes_128_cfb8; +alias2(aes_cfb8, Key) when size(Key)==24 -> aes_192_cfb8; +alias2(aes_cfb8, Key) when size(Key)==32 -> aes_256_cfb8; + +alias2(aes_cfb128, Key) when size(Key)==16 -> aes_128_cfb128; +alias2(aes_cfb128, Key) when size(Key)==24 -> aes_192_cfb128; +alias2(aes_cfb128, Key) when size(Key)==32 -> aes_256_cfb128; + +alias2(aes_ctr, Key) when size(Key)==16 -> aes_128_ctr; +alias2(aes_ctr, Key) when size(Key)==24 -> aes_192_ctr; +alias2(aes_ctr, Key) when size(Key)==32 -> aes_256_ctr; + +alias2(aes_ecb, Key) when size(Key)==16 -> aes_128_ecb; +alias2(aes_ecb, Key) when size(Key)==24 -> aes_192_ecb; +alias2(aes_ecb, Key) when size(Key)==32 -> aes_256_ecb; + +alias2(aes_gcm, Key) when size(Key)==16 -> aes_128_gcm; +alias2(aes_gcm, Key) when size(Key)==24 -> aes_192_gcm; +alias2(aes_gcm, Key) when size(Key)==32 -> aes_256_gcm; + +alias2(aes_ccm, Key) when size(Key)==16 -> aes_128_ccm; +alias2(aes_ccm, Key) when size(Key)==24 -> aes_192_ccm; +alias2(aes_ccm, Key) when size(Key)==32 -> aes_256_ccm; + +alias2(Alg, _) -> Alg. %%%================================================================ %%% @@ -679,34 +1221,73 @@ rand_seed_s() -> rand_seed_alg(Alg) -> rand:seed(rand_seed_alg_s(Alg)). +-spec rand_seed_alg(Alg :: atom(), Seed :: term()) -> + {rand:alg_handler(), + atom() | rand_cache_seed()}. +rand_seed_alg(Alg, Seed) -> + rand:seed(rand_seed_alg_s(Alg, Seed)). + -define(CRYPTO_CACHE_BITS, 56). +-define(CRYPTO_AES_BITS, 58). + -spec rand_seed_alg_s(Alg :: atom()) -> {rand:alg_handler(), atom() | rand_cache_seed()}. -rand_seed_alg_s(?MODULE) -> - {#{ type => ?MODULE, - bits => 64, - next => fun ?MODULE:rand_plugin_next/1, - uniform => fun ?MODULE:rand_plugin_uniform/1, - uniform_n => fun ?MODULE:rand_plugin_uniform/2}, - no_seed}; -rand_seed_alg_s(crypto_cache) -> +rand_seed_alg_s({AlgHandler, _AlgState} = State) when is_map(AlgHandler) -> + State; +rand_seed_alg_s({Alg, AlgState}) when is_atom(Alg) -> + {mk_alg_handler(Alg),AlgState}; + rand_seed_alg_s(Alg) when is_atom(Alg) -> + {mk_alg_handler(Alg),mk_alg_state(Alg)}. +%% +-spec rand_seed_alg_s(Alg :: atom(), Seed :: term()) -> + {rand:alg_handler(), + atom() | rand_cache_seed()}. +rand_seed_alg_s(Alg, Seed) when is_atom(Alg) -> + {mk_alg_handler(Alg),mk_alg_state({Alg,Seed})}. + +mk_alg_handler(?MODULE = Alg) -> + #{ type => Alg, + bits => 64, + next => fun ?MODULE:rand_plugin_next/1, + uniform => fun ?MODULE:rand_plugin_uniform/1, + uniform_n => fun ?MODULE:rand_plugin_uniform/2}; +mk_alg_handler(crypto_cache = Alg) -> + #{ type => Alg, + bits => ?CRYPTO_CACHE_BITS, + next => fun ?MODULE:rand_cache_plugin_next/1}; +mk_alg_handler(crypto_aes = Alg) -> + #{ type => Alg, + bits => ?CRYPTO_AES_BITS, + next => fun ?MODULE:rand_plugin_aes_next/1, + jump => fun ?MODULE:rand_plugin_aes_jump/1}. + +mk_alg_state(?MODULE) -> + no_seed; +mk_alg_state(crypto_cache) -> CacheBits = ?CRYPTO_CACHE_BITS, - EnvCacheSize = - application:get_env( - crypto, rand_cache_size, CacheBits * 16), % Cache 16 * 8 words - Bytes = (CacheBits + 7) div 8, + BytesPerWord = (CacheBits + 7) div 8, + GenBytes = + ((rand_cache_size() + (2*BytesPerWord - 1)) div BytesPerWord) + * BytesPerWord, + {CacheBits, GenBytes, <<>>}; +mk_alg_state({crypto_aes,Seed}) -> + %% 16 byte words (128 bit crypto blocks) + GenWords = (rand_cache_size() + 31) div 16, + Key = crypto:hash(sha256, Seed), + {F,Count} = longcount_seed(Seed), + {Key,GenWords,F,Count}. + +rand_cache_size() -> + DefaultCacheSize = 1024, CacheSize = - case ((EnvCacheSize + (Bytes - 1)) div Bytes) * Bytes of - Sz when is_integer(Sz), Bytes =< Sz -> - Sz; - _ -> - Bytes - end, - {#{ type => crypto_cache, - bits => CacheBits, - next => fun ?MODULE:rand_cache_plugin_next/1}, - {CacheBits, CacheSize, <<>>}}. + application:get_env(crypto, rand_cache_size, DefaultCacheSize), + if + is_integer(CacheSize), 0 =< CacheSize -> + CacheSize; + true -> + DefaultCacheSize + end. rand_plugin_next(Seed) -> {bytes_to_integer(strong_rand_range(1 bsl 64)), Seed}. @@ -717,12 +1298,97 @@ rand_plugin_uniform(State) -> rand_plugin_uniform(Max, State) -> {bytes_to_integer(strong_rand_range(Max)) + 1, State}. -rand_cache_plugin_next({CacheBits, CacheSize, <<>>}) -> + +rand_cache_plugin_next({CacheBits, GenBytes, <<>>}) -> rand_cache_plugin_next( - {CacheBits, CacheSize, strong_rand_bytes(CacheSize)}); -rand_cache_plugin_next({CacheBits, CacheSize, Cache}) -> + {CacheBits, GenBytes, strong_rand_bytes(GenBytes)}); +rand_cache_plugin_next({CacheBits, GenBytes, Cache}) -> <<I:CacheBits, NewCache/binary>> = Cache, - {I, {CacheBits, CacheSize, NewCache}}. + {I, {CacheBits, GenBytes, NewCache}}. + + +%% Encrypt 128 bit counter values and use the 58 lowest +%% encrypted bits as random numbers. +%% +%% The 128 bit counter is handled as 4 32 bit words +%% to avoid bignums. Generate a bunch of numbers +%% at the time and cache them. +%% +-dialyzer({no_improper_lists, rand_plugin_aes_next/1}). +rand_plugin_aes_next([V|Cache]) -> + {V,Cache}; +rand_plugin_aes_next({Key,GenWords,F,Count}) -> + rand_plugin_aes_next(Key, GenWords, F, Count); +rand_plugin_aes_next({Key,GenWords,F,_JumpBase,Count}) -> + rand_plugin_aes_next(Key, GenWords, F, Count). +%% +rand_plugin_aes_next(Key, GenWords, F, Count) -> + {Cleartext,NewCount} = aes_cleartext(<<>>, F, Count, GenWords), + Encrypted = crypto:block_encrypt(aes_ecb, Key, Cleartext), + [V|Cache] = aes_cache(Encrypted, {Key,GenWords,F,Count,NewCount}), + {V,Cache}. + +%% A jump advances the counter 2^512 steps; the jump function +%% is applied to the jump base and then the number of used +%% numbers from the cache has to be wasted for the jump to be correct +%% +rand_plugin_aes_jump({#{type := crypto_aes} = Alg, Cache}) -> + {Alg,rand_plugin_aes_jump(fun longcount_jump/1, 0, Cache)}. +%% Count cached words and subtract their number from jump +-dialyzer({no_improper_lists, rand_plugin_aes_jump/3}). +rand_plugin_aes_jump(Jump, J, [_|Cache]) -> + rand_plugin_aes_jump(Jump, J + 1, Cache); +rand_plugin_aes_jump(Jump, J, {Key,GenWords,F,JumpBase, _Count}) -> + rand_plugin_aes_jump(Jump, GenWords - J, Key, GenWords, F, JumpBase); +rand_plugin_aes_jump(Jump, 0, {Key,GenWords,F,JumpBase}) -> + rand_plugin_aes_jump(Jump, 0, Key, GenWords, F, JumpBase). +%% +rand_plugin_aes_jump(Jump, Skip, Key, GenWords, F, JumpBase) -> + Count = longcount_next_count(Skip, Jump(JumpBase)), + {Key,GenWords,F,Count}. + +rand_plugin_aes_jump_2pow20(Cache) -> + rand_plugin_aes_jump(fun longcount_jump_2pow20/1, 0, Cache). + + +longcount_seed(Seed) -> + <<X:64, _:6, F:12, S2:58, S1:58, S0:58>> = + crypto:hash(sha256, [Seed,<<"Xoroshiro928">>]), + {F,rand:exro928_seed([S0,S1,S2|rand:seed58(13, X)])}. + +longcount_next_count(0, Count) -> + Count; +longcount_next_count(N, Count) -> + longcount_next_count(N - 1, rand:exro928_next_state(Count)). + +longcount_next(Count) -> + rand:exro928_next(Count). + +longcount_jump(Count) -> + rand:exro928_jump_2pow512(Count). + +longcount_jump_2pow20(Count) -> + rand:exro928_jump_2pow20(Count). + + +%% Build binary with counter values to cache +aes_cleartext(Cleartext, _F, Count, 0) -> + {Cleartext,Count}; +aes_cleartext(Cleartext, F, Count, GenWords) -> + {{S0,S1}, NewCount} = longcount_next(Count), + aes_cleartext( + <<Cleartext/binary, F:12, S1:58, S0:58>>, + F, NewCount, GenWords - 1). + +%% Parse and cache encrypted counter values aka random numbers +-dialyzer({no_improper_lists, aes_cache/2}). +aes_cache(<<>>, Cache) -> + Cache; +aes_cache( + <<_:(128 - ?CRYPTO_AES_BITS), V:?CRYPTO_AES_BITS, Encrypted/binary>>, + Cache) -> + [V|aes_cache(Encrypted, Cache)]. + strong_rand_range(Range) when is_integer(Range), Range > 0 -> BinRange = int_to_bin(Range), @@ -787,7 +1453,8 @@ rand_seed_nif(_Seed) -> ?nif_stub. -type pk_sign_verify_opts() :: [ rsa_sign_verify_opt() ] . -type rsa_sign_verify_opt() :: {rsa_padding, rsa_sign_verify_padding()} - | {rsa_pss_saltlen, integer()} . + | {rsa_pss_saltlen, integer()} + | {rsa_mgf1_md, sha2()}. -type rsa_sign_verify_padding() :: rsa_pkcs1_padding | rsa_pkcs1_pss_padding | rsa_x931_padding | rsa_no_padding @@ -803,7 +1470,7 @@ rand_seed_nif(_Seed) -> ?nif_stub. DigestType :: rsa_digest_type() | dss_digest_type() | ecdsa_digest_type(), - Msg :: binary() | {digest,binary()}, + Msg :: iodata() | {digest,iodata()}, Key :: rsa_private() | dss_private() | [ecdsa_private() | ecdsa_params()] @@ -822,7 +1489,7 @@ sign(Algorithm, Type, Data, Key) -> | dss_digest_type() | ecdsa_digest_type() | none, - Msg :: binary() | {digest,binary()}, + Msg :: iodata() | {digest,iodata()}, Key :: rsa_private() | dss_private() | [ecdsa_private() | ecdsa_params()] @@ -851,7 +1518,7 @@ pkey_sign_nif(_Algorithm, _Type, _Digest, _Key, _Options) -> ?nif_stub. | dss_digest_type() | ecdsa_digest_type() | none, - Msg :: binary() | {digest,binary()}, + Msg :: iodata() | {digest,iodata()}, Signature :: binary(), Key :: rsa_public() | dss_public() @@ -869,7 +1536,7 @@ verify(Algorithm, Type, Data, Signature, Key) -> DigestType :: rsa_digest_type() | dss_digest_type() | ecdsa_digest_type(), - Msg :: binary() | {digest,binary()}, + Msg :: iodata() | {digest,iodata()}, Signature :: binary(), Key :: rsa_public() | dss_public() @@ -1562,6 +2229,7 @@ hash_update(State0, Data, _, MaxBytes) -> State = notsup_to_error(hash_update_nif(State0, Increment)), hash_update(State, Rest, erlang:byte_size(Rest), MaxBytes). +hash_info_nif(_Hash) -> ?nif_stub. hash_nif(_Hash, _Data) -> ?nif_stub. hash_init_nif(_Hash) -> ?nif_stub. hash_update_nif(_State, _Data) -> ?nif_stub. @@ -1606,28 +2274,13 @@ poly1305_nif(_Key, _Data) -> ?nif_stub. %% CIPHERS -------------------------------------------------------------------- -block_crypt_nif(_Type, _Key, _Ivec, _Text, _IsEncrypt) -> ?nif_stub. -block_crypt_nif(_Type, _Key, _Text, _IsEncrypt) -> ?nif_stub. - -check_des3_key(Key) -> - case lists:map(fun erlang:iolist_to_binary/1, Key) of - ValidKey = [B1, B2, B3] when byte_size(B1) =:= 8, - byte_size(B2) =:= 8, - byte_size(B3) =:= 8 -> - ValidKey; - _ -> - error(badarg) - end. +cipher_info_nif(_Type) -> ?nif_stub. %% %% AES - in Galois/Counter Mode (GCM) %% %% The default tag length is EVP_GCM_TLS_TAG_LEN(16), -aead_encrypt(Type=aes_ccm, Key, Ivec, AAD, In) -> aead_encrypt(Type, Key, Ivec, AAD, In, 12); -aead_encrypt(Type=aes_gcm, Key, Ivec, AAD, In) -> aead_encrypt(Type, Key, Ivec, AAD, In, 16). - -aead_encrypt(_Type, _Key, _Ivec, _AAD, _In, _TagLength) -> ?nif_stub. -aead_decrypt(_Type, _Key, _Ivec, _AAD, _In, _Tag) -> ?nif_stub. +aead_cipher(_Type, _Key, _Ivec, _AAD, _In, _TagOrTagLength, _EncFlg) -> ?nif_stub. %% %% AES - with 256 bit key in infinite garble extension mode (IGE) @@ -1635,59 +2288,7 @@ aead_decrypt(_Type, _Key, _Ivec, _AAD, _In, _Tag) -> ?nif_stub. aes_ige_crypt_nif(_Key, _IVec, _Data, _IsEncrypt) -> ?nif_stub. - -%% Stream ciphers -------------------------------------------------------------------- - -stream_crypt(Fun, State, Data, Size, MaxByts, []) when Size =< MaxByts -> - Fun(State, Data); -stream_crypt(Fun, State0, Data, Size, MaxByts, Acc) when Size =< MaxByts -> - {State, Cipher} = Fun(State0, Data), - {State, list_to_binary(lists:reverse([Cipher | Acc]))}; -stream_crypt(Fun, State0, Data, _, MaxByts, Acc) -> - <<Increment:MaxByts/binary, Rest/binary>> = Data, - {State, CipherText} = Fun(State0, Increment), - stream_crypt(Fun, State, Rest, erlang:byte_size(Rest), MaxByts, [CipherText | Acc]). - -do_stream_encrypt({aes_ctr, State0}, Data) -> - {State, Cipher} = aes_ctr_stream_encrypt(State0, Data), - {{aes_ctr, State}, Cipher}; -do_stream_encrypt({rc4, State0}, Data) -> - {State, Cipher} = rc4_encrypt_with_state(State0, Data), - {{rc4, State}, Cipher}; -do_stream_encrypt({chacha20, State0}, Data) -> - {State, Cipher} = chacha20_stream_encrypt(State0, Data), - {{chacha20, State}, Cipher}. - -do_stream_decrypt({aes_ctr, State0}, Data) -> - {State, Text} = aes_ctr_stream_decrypt(State0, Data), - {{aes_ctr, State}, Text}; -do_stream_decrypt({rc4, State0}, Data) -> - {State, Text} = rc4_encrypt_with_state(State0, Data), - {{rc4, State}, Text}; -do_stream_decrypt({chacha20, State0}, Data) -> - {State, Cipher} = chacha20_stream_decrypt(State0, Data), - {{chacha20, State}, Cipher}. - - -%% -%% AES - in counter mode (CTR) with state maintained for multi-call streaming -%% -aes_ctr_stream_init(_Key, _IVec) -> ?nif_stub. -aes_ctr_stream_encrypt(_State, _Data) -> ?nif_stub. -aes_ctr_stream_decrypt(_State, _Cipher) -> ?nif_stub. - -%% -%% RC4 - symmetric stream cipher -%% -rc4_set_key(_Key) -> ?nif_stub. -rc4_encrypt_with_state(_State, _Data) -> ?nif_stub. - -%% -%% CHACHA20 - stream cipher -%% -chacha20_stream_init(_Key, _IVec) -> ?nif_stub. -chacha20_stream_encrypt(_State, _Data) -> ?nif_stub. -chacha20_stream_decrypt(_State, _Data) -> ?nif_stub. +%%%================================================================ %% Secure remote password ------------------------------------------------------------------- @@ -1851,7 +2452,13 @@ exor(Data1, Data2, _Size, MaxByts, Acc) -> do_exor(_A, _B) -> ?nif_stub. -algorithms() -> ?nif_stub. +hash_algorithms() -> ?nif_stub. +pubkey_algorithms() -> ?nif_stub. +cipher_algorithms() -> ?nif_stub. +mac_algorithms() -> ?nif_stub. +curve_algorithms() -> ?nif_stub. +rsa_opts_algorithms() -> ?nif_stub. + int_to_bin(X) when X < 0 -> int_to_bin_neg(X, []); int_to_bin(X) -> int_to_bin_pos(X, []). @@ -2051,3 +2658,5 @@ check_otp_test_engine(LibDir) -> {error, notexist} end end. + + diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index cbfa96cc16..56691223c4 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -9,7 +9,7 @@ %% %% http://www.apache.org/licenses/LICENSE-2.0 %% -%% Unless required by applicable law or agreed to in writing, software + %% distributed under the License is distributed on an "AS IS" BASIS, %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and @@ -40,55 +40,109 @@ all() -> rand_uniform, rand_threads, rand_plugin, - rand_plugin_s + rand_plugin_s, + cipher_info, + hash_info ]. +-define(NEW_CIPHER_TYPE_SCHEMA, + {group, des_ede3_cbc}, + {group, des_ede3_cfb}, + {group, aes_128_cbc}, + {group, aes_192_cbc}, + {group, aes_256_cbc}, + {group, aes_128_ctr}, + {group, aes_192_ctr}, + {group, aes_256_ctr}, + {group, aes_128_ccm}, + {group, aes_192_ccm}, + {group, aes_256_ccm}, + {group, aes_128_ecb}, + {group, aes_192_ecb}, + {group, aes_256_ecb}, + {group, aes_128_gcm}, + {group, aes_192_gcm}, + {group, aes_256_gcm}, + {group, des_ede3_cbc}, + {group, des_ede3_cfb} + ). + +-define(RETIRED_TYPE_ALIASES, + {group, aes_cbc}, + {group, aes_cbc128}, + {group, aes_cbc256}, + {group, aes_ccm}, + {group, aes_ctr}, + {group, aes_gcm}, + {group, aes_ecb}, + {group, des3_cfb}, + {group, des3_cbc}, + {group, des3_cbf}, + {group, des_ede3} + ). + groups() -> - [{non_fips, [], [{group, md4}, + [{non_fips, [], [ + {group, blake2b}, + {group, blake2s}, + {group, dss}, + {group, ecdsa}, + {group, ed25519}, + {group, ed448}, + {group, rsa}, + + {group, md4}, {group, md5}, {group, ripemd160}, - {group, sha}, {group, sha224}, {group, sha256}, {group, sha384}, - {group, sha512}, {group, sha3_224}, {group, sha3_256}, {group, sha3_384}, {group, sha3_512}, - {group, rsa}, - {group, dss}, - {group, ecdsa}, - {group, ed25519}, - {group, ed448}, + {group, sha512}, + {group, sha}, + {group, poly1305}, + {group, dh}, {group, ecdh}, {group, srp}, - {group, des_cbc}, - {group, des_cfb}, - {group, des3_cbc}, - {group, des3_cbf}, - {group, des3_cfb}, - {group, des_ede3}, + + {group, chacha20_poly1305}, + {group, chacha20}, {group, blowfish_cbc}, - {group, blowfish_ecb}, {group, blowfish_cfb64}, + {group, blowfish_ecb}, {group, blowfish_ofb64}, - {group, aes_cbc128}, - {group, aes_cfb8}, - {group, aes_cfb128}, - {group, aes_cbc256}, + {group, aes_ige256}, + {group, des_cbc}, + {group, des_cfb}, {group, rc2_cbc}, {group, rc4}, - {group, aes_ctr}, - {group, aes_ccm}, - {group, aes_gcm}, - {group, chacha20_poly1305}, - {group, chacha20}, - {group, poly1305}, - {group, aes_cbc}]}, - {fips, [], [{group, no_md4}, + + ?NEW_CIPHER_TYPE_SCHEMA, + {group, aes_128_cfb128}, + {group, aes_192_cfb128}, + {group, aes_256_cfb128}, + {group, aes_128_cfb8}, + {group, aes_192_cfb8}, + {group, aes_256_cfb8}, + ?RETIRED_TYPE_ALIASES, + {group, aes_cfb128}, + {group, aes_cfb8} + ]}, + {fips, [], [ + {group, no_blake2b}, + {group, no_blake2s}, + {group, dss}, + {group, ecdsa}, + {group, no_ed25519}, + {group, no_ed448}, + {group, rsa}, + + {group, no_md4}, {group, no_md5}, {group, no_ripemd160}, {group, sha}, @@ -96,119 +150,148 @@ groups() -> {group, sha256}, {group, sha384}, {group, sha512}, - {group, rsa}, - {group, dss}, - {group, ecdsa}, - {group, no_ed25519}, - {group, no_ed448}, + {group, no_poly1305}, + {group, dh}, {group, ecdh}, {group, no_srp}, - {group, no_des_cbc}, - {group, no_des_cfb}, - {group, des3_cbc}, - {group, des3_cbf}, - {group, des3_cfb}, - {group, des_ede3}, + + {group, no_chacha20_poly1305}, + {group, no_chacha20}, {group, no_blowfish_cbc}, - {group, no_blowfish_ecb}, {group, no_blowfish_cfb64}, + {group, no_blowfish_ecb}, {group, no_blowfish_ofb64}, - {group, aes_cbc128}, - {group, no_aes_cfb8}, + {group, no_aes_cfb128}, - {group, aes_cbc256}, + {group, no_aes_cfb8}, {group, no_aes_ige256}, + {group, no_des_cbc}, + {group, no_des_cfb}, {group, no_rc2_cbc}, {group, no_rc4}, - {group, aes_ctr}, - {group, aes_ccm}, - {group, aes_gcm}, - {group, no_chacha20_poly1305}, - {group, no_chacha20}, - {group, aes_cbc}]}, - {md4, [], [hash]}, - {md5, [], [hash, hmac]}, - {ripemd160, [], [hash]}, - {sha, [], [hash, hmac]}, - {sha224, [], [hash, hmac]}, - {sha256, [], [hash, hmac]}, - {sha384, [], [hash, hmac]}, - {sha512, [], [hash, hmac]}, - {sha3_224, [], [hash, hmac]}, - {sha3_256, [], [hash, hmac]}, - {sha3_384, [], [hash, hmac]}, - {sha3_512, [], [hash, hmac]}, - {rsa, [], [sign_verify, - public_encrypt, - private_encrypt, - generate - ]}, - {dss, [], [sign_verify - %% Does not work yet: ,public_encrypt, private_encrypt - ]}, - {ecdsa, [], [sign_verify - %% Does not work yet: ,public_encrypt, private_encrypt - ]}, - {ed25519, [], [sign_verify - %% Does not work yet: ,public_encrypt, private_encrypt - ]}, - {ed448, [], [sign_verify - %% Does not work yet: ,public_encrypt, private_encrypt - ]}, - {dh, [], [generate_compute, - compute_bug]}, - {ecdh, [], [use_all_elliptic_curves, compute, generate]}, - {srp, [], [generate_compute]}, - {des_cbc, [], [block]}, - {des_cfb, [], [block]}, - {des3_cbc,[], [block]}, - {des_ede3,[], [block]}, - {des3_cbf,[], [block]}, - {des3_cfb,[], [block]}, - {rc2_cbc,[], [block]}, - {aes_cbc128,[], [block, cmac]}, - {aes_cfb8,[], [block]}, - {aes_cfb128,[], [block]}, - {aes_cbc256,[], [block, cmac]}, - {aes_ecb,[], [block]}, - {aes_ige256,[], [block]}, - {blowfish_cbc, [], [block]}, - {blowfish_ecb, [], [block]}, - {blowfish_cfb64, [], [block]}, - {blowfish_ofb64,[], [block]}, - {rc4, [], [stream]}, - {aes_ctr, [], [stream]}, - {aes_ccm, [], [aead]}, - {aes_gcm, [], [aead]}, - {chacha20_poly1305, [], [aead]}, - {chacha20, [], [stream]}, - {poly1305, [], [poly1305]}, - {aes_cbc, [], [block]}, - {no_aes_cfb8,[], [no_support, no_block]}, - {no_aes_cfb128,[], [no_support, no_block]}, - {no_md4, [], [no_support, no_hash]}, - {no_md5, [], [no_support, no_hash, no_hmac]}, - {no_ed25519, [], [no_support, no_sign_verify - %% Does not work yet: ,public_encrypt, private_encrypt - ]}, - {no_ed448, [], [no_support, no_sign_verify - %% Does not work yet: ,public_encrypt, private_encrypt - ]}, - {no_ripemd160, [], [no_support, no_hash]}, - {no_srp, [], [no_support, no_generate_compute]}, - {no_des_cbc, [], [no_support, no_block]}, - {no_des_cfb, [], [no_support, no_block]}, - {no_blowfish_cbc, [], [no_support, no_block]}, - {no_blowfish_ecb, [], [no_support, no_block]}, - {no_blowfish_cfb64, [], [no_support, no_block]}, - {no_blowfish_ofb64, [], [no_support, no_block]}, - {no_aes_ige256, [], [no_support, no_block]}, + + ?NEW_CIPHER_TYPE_SCHEMA, + ?RETIRED_TYPE_ALIASES + ]}, + + {md4, [], [hash]}, + {md5, [], [hash, hmac]}, + {ripemd160, [], [hash]}, + {sha, [], [hash, hmac]}, + {sha224, [], [hash, hmac]}, + {sha256, [], [hash, hmac]}, + {sha384, [], [hash, hmac]}, + {sha512, [], [hash, hmac]}, + {sha3_224, [], [hash, hmac]}, + {sha3_256, [], [hash, hmac]}, + {sha3_384, [], [hash, hmac]}, + {sha3_512, [], [hash, hmac]}, + {blake2b, [], [hash, hmac]}, + {blake2s, [], [hash, hmac]}, + {no_blake2b, [], [no_hash, no_hmac]}, + {no_blake2s, [], [no_hash, no_hmac]}, + {rsa, [], [sign_verify, + public_encrypt, + private_encrypt, + generate + ]}, + {dss, [], [sign_verify + %% Does not work yet: ,public_encrypt, private_encrypt + ]}, + {ecdsa, [], [sign_verify + %% Does not work yet: ,public_encrypt, private_encrypt + ]}, + {ed25519, [], [sign_verify + %% Does not work yet: ,public_encrypt, private_encrypt + ]}, + {ed448, [], [sign_verify + %% Does not work yet: ,public_encrypt, private_encrypt + ]}, + {dh, [], [generate_compute, compute_bug]}, + {ecdh, [], [use_all_elliptic_curves, compute, generate]}, + {srp, [], [generate_compute]}, + {des_cbc, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des_cfb, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des_ede3_cbc, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {des_ede3_cfb, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {rc2_cbc, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_cfb8, [], [block]}, + {aes_128_cfb8, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_192_cfb8, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_256_cfb8, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {no_aes_cfb8, [], [no_support, no_block]}, + {aes_cfb128, [], [block]}, + {aes_128_cfb128, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_192_cfb128, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_256_cfb128, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {no_aes_cfb128, [], [no_support, no_block]}, + {aes_ige256, [], [block]}, + {no_aes_ige256, [], [no_support, no_block]}, + {blowfish_cbc, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {blowfish_ecb, [], [block, api_ng, api_ng_one_shot]}, + {blowfish_cfb64, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {blowfish_ofb64, [], [block, api_ng, api_ng_one_shot, api_ng_tls]}, + {rc4, [], [stream, api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_ctr, [], [stream]}, + {chacha20_poly1305, [], [aead]}, + {chacha20, [], [stream, api_ng, api_ng_one_shot, api_ng_tls]}, + {poly1305, [], [poly1305]}, + {no_poly1305, [], [no_poly1305]}, + {no_aes_cfb128, [], [no_support, no_block]}, + {no_md4, [], [no_support, no_hash]}, + {no_md5, [], [no_support, no_hash, no_hmac]}, + {no_ed25519, [], [no_support, no_sign_verify + %% Does not work yet: ,public_encrypt, private_encrypt + ]}, + {no_ed448, [], [no_support, no_sign_verify + %% Does not work yet: ,public_encrypt, private_encrypt + ]}, + {no_ripemd160, [], [no_support, no_hash]}, + {no_srp, [], [no_support, no_generate_compute]}, + {no_des_cbc, [], [no_support, no_block]}, + {no_des_cfb, [], [no_support, no_block]}, + {no_blowfish_cbc, [], [no_support, no_block]}, + {no_blowfish_ecb, [], [no_support, no_block]}, + {no_blowfish_cfb64, [], [no_support, no_block]}, + {no_blowfish_ofb64, [], [no_support, no_block]}, + {no_aes_ige256, [], [no_support, no_block]}, {no_chacha20_poly1305, [], [no_support, no_aead]}, - {no_chacha20, [], [no_support, no_stream_ivec]}, - {no_rc2_cbc, [], [no_support, no_block]}, - {no_rc4, [], [no_support, no_stream]}, - {api_errors, [], [api_errors_ecdh]} + {no_chacha20, [], [no_support, no_stream_ivec]}, + {no_rc2_cbc, [], [no_support, no_block]}, + {no_rc4, [], [no_support, no_stream]}, + {api_errors, [], [api_errors_ecdh]}, + + %% New cipher nameing schema + {des_ede3_cbc, [], [api_ng, api_ng_one_shot, api_ng_tls]}, + {des_ede3_cfb, [], [api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_128_cbc, [], [api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_192_cbc, [], [api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_256_cbc, [], [api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_128_ctr, [], [api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_192_ctr, [], [api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_256_ctr, [], [api_ng, api_ng_one_shot, api_ng_tls]}, + {aes_128_ccm, [], [aead]}, + {aes_192_ccm, [], [aead]}, + {aes_256_ccm, [], [aead]}, + {aes_128_ecb, [], [api_ng, api_ng_one_shot]}, + {aes_192_ecb, [], [api_ng, api_ng_one_shot]}, + {aes_256_ecb, [], [api_ng, api_ng_one_shot]}, + {aes_128_gcm, [], [aead]}, + {aes_192_gcm, [], [aead]}, + {aes_256_gcm, [], [aead]}, + + %% Retired aliases + {aes_cbc, [], [block]}, + {aes_cbc128, [], [block]}, + {aes_cbc256, [], [block]}, + {aes_ccm, [], [aead]}, + {aes_ecb, [], [block]}, + {aes_gcm, [], [aead]}, + {des3_cbc, [], [block]}, + {des_ede3, [], [block]}, + {des3_cbf, [], [block]}, + {des3_cfb, [], [block]} ]. %%------------------------------------------------------------------- @@ -301,12 +384,11 @@ end_per_group(_GroupName, Config) -> init_per_testcase(info, Config) -> Config; init_per_testcase(cmac, Config) -> - case crypto:info_lib() of - [{<<"OpenSSL">>,LibVer,_}] when is_integer(LibVer), LibVer > 16#10001000 -> + case is_supported(cmac) of + true -> Config; - _Else -> - % The CMAC functionality was introduced in OpenSSL 1.0.1 - {skip, "OpenSSL is too old"} + false -> + {skip, "CMAC is not supported"} end; init_per_testcase(generate, Config) -> case proplists:get_value(type, Config) of @@ -406,12 +488,19 @@ poly1305(Config) -> end, proplists:get_value(poly1305, Config)). %%-------------------------------------------------------------------- +no_poly1305() -> + [{doc, "Test disabled poly1305 function"}]. +no_poly1305(_Config) -> + Key = <<133,214,190,120,87,85,109,51,127,68,82,254,66,213,6,168,1, + 3,128,138,251,13,178,253,74,191,246,175,65,73,245,27>>, + Txt = <<"Cryptographic Forum Research Group">>, + notsup(fun crypto:poly1305/2, [Key,Txt]). + +%%-------------------------------------------------------------------- block() -> [{doc, "Test block ciphers"}]. block(Config) when is_list(Config) -> - Fips = proplists:get_bool(fips, Config), - Type = ?config(type, Config), - Blocks = lazy_eval(proplists:get_value(block, Config)), + [_|_] = Blocks = lazy_eval(proplists:get_value(cipher, Config)), lists:foreach(fun block_cipher/1, Blocks), lists:foreach(fun block_cipher/1, block_iolistify(Blocks)), lists:foreach(fun block_cipher_increment/1, block_iolistify(Blocks)). @@ -420,7 +509,7 @@ block(Config) when is_list(Config) -> no_block() -> [{doc, "Test disabled block ciphers"}]. no_block(Config) when is_list(Config) -> - Blocks = lazy_eval(proplists:get_value(block, Config)), + [_|_] = Blocks = lazy_eval(proplists:get_value(cipher, Config)), Args = case Blocks of [{_Type, _Key, _PlainText} = A | _] -> tuple_to_list(A); @@ -433,11 +522,157 @@ no_block(Config) when is_list(Config) -> notsup(fun crypto:block_encrypt/N, Args), notsup(fun crypto:block_decrypt/N, Args). %%-------------------------------------------------------------------- +api_ng() -> + [{doc, "Test new api"}]. + +api_ng(Config) when is_list(Config) -> + [_|_] = Ciphers = lazy_eval(proplists:get_value(cipher, Config, [])), + lists:foreach(fun api_ng_cipher_increment/1, Ciphers). + +api_ng_cipher_increment({Type, Key, PlainTexts}=_X) -> + ct:log("~p",[_X]), + api_ng_cipher_increment({Type, Key, <<>>, PlainTexts}); + +api_ng_cipher_increment({Type, Key, IV, PlainTexts}=_X) -> + ct:log("~p",[_X]), + api_ng_cipher_increment({Type, Key, IV, PlainTexts, undefined}); + +api_ng_cipher_increment({Type, Key, IV, PlainText0, ExpectedEncText}=_X) -> + ct:log("~p",[_X]), + PlainTexts = iolistify(PlainText0), + RefEnc = crypto:crypto_init(Type, Key, IV, true), + RefDec = crypto:crypto_init(Type, Key, IV, false), + EncTexts = api_ng_cipher_increment_loop(RefEnc, PlainTexts), + Enc = iolist_to_binary(EncTexts), + case ExpectedEncText of + undefined -> + ok; + Enc -> + ok; + _ -> + ct:log("encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainTexts}, ExpectedEncText, Enc]), + ct:fail("api_ng_cipher_increment (encode)",[]) + end, + Plain = iolist_to_binary(PlainTexts), + case iolist_to_binary(api_ng_cipher_increment_loop(RefDec, EncTexts)) of + Plain -> + ok; + OtherPT -> + ct:log("decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTexts}, Plain, OtherPT]), + ct:fail("api_ng_cipher_increment (encode)",[]) + end. + + +api_ng_cipher_increment_loop(Ref, InTexts) -> + lists:map(fun(Txt) -> + try crypto:crypto_update(Ref, Txt) + of + Bin when is_binary(Bin) -> + Bin + catch + error:Error -> + ct:pal("Txt = ~p",[Txt]), + ct:fail("~p",[Error]) + end + end, InTexts). + +%%-------------------------------------------------------------------- +api_ng_one_shot() -> + [{doc, "Test new api"}]. + +api_ng_one_shot(Config) when is_list(Config) -> + [_|_] = Ciphers = lazy_eval(proplists:get_value(cipher, Config, [])), + lists:foreach(fun do_api_ng_one_shot/1, Ciphers). + +do_api_ng_one_shot({Type, Key, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_one_shot({Type, Key, <<>>, PlainTexts}); + +do_api_ng_one_shot({Type, Key, IV, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_one_shot({Type, Key, IV, PlainTexts, undefined}); + +do_api_ng_one_shot({Type, Key, IV, PlainText0, ExpectedEncText}=_X) -> + ct:log("~p",[_X]), + PlainText = iolist_to_binary(lazy_eval(PlainText0)), + EncTxt = crypto:crypto_one_time(Type, Key, IV, PlainText, true), + case ExpectedEncText of + undefined -> + ok; + EncTxt -> + ok; + _ -> + ct:log("encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainText}, ExpectedEncText, EncTxt]), + ct:fail("api_ng_one_time (encode)",[]) + end, + case crypto:crypto_one_time(Type, Key, IV, EncTxt, false) of + PlainText -> + ok; + OtherPT -> + ct:log("decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTxt}, PlainText, OtherPT]), + ct:fail("api_ng_one_time (decode)",[]) + end. + +%%-------------------------------------------------------------------- +api_ng_tls() -> + [{doc, "Test special tls api"}]. + +api_ng_tls(Config) when is_list(Config) -> + [_|_] = Ciphers = lazy_eval(proplists:get_value(cipher, Config, [])), + lists:foreach(fun do_api_ng_tls/1, Ciphers). + + +do_api_ng_tls({Type, Key, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_tls({Type, Key, <<>>, PlainTexts}); + +do_api_ng_tls({Type, Key, IV, PlainTexts}=_X) -> + ct:log("~p",[_X]), + do_api_ng_tls({Type, Key, IV, PlainTexts, undefined}); + +do_api_ng_tls({Type, Key, IV, PlainText0, ExpectedEncText}=_X) -> + ct:log("~p",[_X]), + PlainText = iolist_to_binary(lazy_eval(PlainText0)), + Renc = crypto:crypto_dyn_iv_init(Type, Key, true), + Rdec = crypto:crypto_dyn_iv_init(Type, Key, false), + EncTxt = crypto:crypto_dyn_iv_update(Renc, PlainText, IV), + case ExpectedEncText of + undefined -> + ok; + EncTxt -> + %% Now check that the state is NOT updated: + case crypto:crypto_dyn_iv_update(Renc, PlainText, IV) of + EncTxt -> + ok; + EncTxt2 -> + ct:log("2nd encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainText}, EncTxt, EncTxt2]), + ct:fail("api_ng_tls (second encode)",[]) + end; + OtherEnc -> + ct:log("1st encode~nIn: ~p~nExpected: ~p~nEnc: ~p~n", [{Type,Key,IV,PlainText}, ExpectedEncText, OtherEnc]), + ct:fail("api_ng_tls (encode)",[]) + end, + case crypto:crypto_dyn_iv_update(Rdec, EncTxt, IV) of + PlainText -> + %% Now check that the state is NOT updated: + case crypto:crypto_dyn_iv_update(Rdec, EncTxt, IV) of + PlainText -> + ok; + PlainText2 -> + ct:log("2nd decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTxt}, PlainText, PlainText2]), + ct:fail("api_ng_tls (second decode)",[]) + end; + OtherPT -> + ct:log("1st decode~nIn: ~p~nExpected: ~p~nDec: ~p~n", [{Type,Key,IV,EncTxt}, PlainText, OtherPT]), + ct:fail("api_ng_tlst (decode)",[]) + end. + +%%-------------------------------------------------------------------- no_aead() -> [{doc, "Test disabled aead ciphers"}]. no_aead(Config) when is_list(Config) -> EncArg4 = - case lazy_eval(proplists:get_value(aead, Config)) of + case lazy_eval(proplists:get_value(cipher, Config)) of [{Type, Key, PlainText, Nonce, AAD, CipherText, CipherTag, TagLen, _Info} | _] -> {AAD, PlainText, TagLen}; [{Type, Key, PlainText, Nonce, AAD, CipherText, CipherTag, _Info} | _] -> @@ -452,7 +687,7 @@ no_aead(Config) when is_list(Config) -> stream() -> [{doc, "Test stream ciphers"}]. stream(Config) when is_list(Config) -> - Streams = lazy_eval(proplists:get_value(stream, Config)), + [_|_] = Streams = lazy_eval(proplists:get_value(cipher, Config)), lists:foreach(fun stream_cipher/1, Streams), lists:foreach(fun stream_cipher/1, stream_iolistify(Streams)), @@ -475,8 +710,7 @@ no_stream_ivec(Config) when is_list(Config) -> aead() -> [{doc, "Test AEAD ciphers"}]. aead(Config) when is_list(Config) -> - AEADs = lazy_eval(proplists:get_value(aead, Config)), - + [_|_] = AEADs = lazy_eval(proplists:get_value(cipher, Config)), FilteredAEADs = case proplists:get_bool(fips, Config) of false -> @@ -489,7 +723,6 @@ aead(Config) when is_list(Config) -> IVLen >= 12 end, AEADs) end, - lists:foreach(fun aead_cipher/1, FilteredAEADs). %%-------------------------------------------------------------------- @@ -593,7 +826,7 @@ use_all_elliptic_curves(_Config) -> {C,E} end} || Curve <- Curves -- [ed25519, ed448, x25519, x448, ipsec3, ipsec4], - Hash <- Hashs -- [md4, md5, ripemd160, sha3_224, sha3_256, sha3_384, sha3_512] + Hash <- Hashs -- [md4, md5, ripemd160, sha3_224, sha3_256, sha3_384, sha3_512, blake2b, blake2s] ], Fails = lists:filter(fun({_,true}) -> false; @@ -662,6 +895,41 @@ rand_plugin_s(Config) when is_list(Config) -> rand_plugin_aux(explicit_state). %%-------------------------------------------------------------------- +cipher_info() -> + [{doc, "crypto cipher_info testing"}]. +cipher_info(Config) when is_list(Config) -> + #{type := _,key_length := _,iv_length := _, + block_size := _,mode := _} = crypto:cipher_info(aes_128_cbc), + {'EXIT',_} = (catch crypto:cipher_info(not_a_cipher)), + case lists:foldl(fun(C,Ok) -> + try crypto:cipher_info(C) + of + _ -> Ok + catch Cls:Exc -> + ct:pal("~p:~p ~p",[Cls,Exc,C]), + false + end + end, + true, +crypto:supports(ciphers)) of +%% proplists:get_value(ciphers, crypto:supports())) of + true -> + ok; + false -> + ct:fail('Cipher unsupported',[]) + end. + + +%%-------------------------------------------------------------------- +hash_info() -> + [{doc, "crypto hash_info testing"}]. +hash_info(Config) when is_list(Config) -> + #{type := _,size := _,block_size := _} = crypto:hash_info(sha256), + {'EXIT',_} = (catch crypto:hash_info(not_a_hash)), + lists:foreach(fun(H) -> crypto:hash_info(H) end, + proplists:get_value(hashs, crypto:supports())). + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- hash(_, [], []) -> @@ -751,6 +1019,7 @@ cmac_check({Type, Key, Text, Size, CMac}) -> ct:fail({{crypto, cmac, [Type, Key, Text, Size]}, {expected, ExpCMac}, {got, Other}}) end. + block_cipher({Type, Key, PlainText}) -> Plain = iolist_to_binary(PlainText), CipherText = crypto:block_encrypt(Type, Key, PlainText), @@ -786,13 +1055,27 @@ block_cipher({Type, Key, IV, PlainText, CipherText}) -> ct:fail({{crypto, block_decrypt, [Type, Key, IV, CipherText]}, {expected, Plain}, {got, Other1}}) end. -block_cipher_increment({Type, Key, IV, PlainTexts}) - when Type == des_cbc; Type == aes_cbc; Type == des3_cbc -> +block_cipher_increment({Type, Key, IV, PlainTexts}) when Type == des_cbc ; + Type == des3_cbc ; + Type == aes_128_cbc ; + Type == aes_192_cbc ; + Type == aes_256_cbc + -> block_cipher_increment(Type, Key, IV, IV, PlainTexts, iolist_to_binary(PlainTexts), []); -block_cipher_increment({Type, Key, IV, PlainTexts, CipherText}) - when Type == des_cbc; Type == des3_cbc -> +block_cipher_increment({Type, Key, IV, PlainTexts, CipherText}) when Type == des_cbc; + Type == des_ede3_cbc ; + Type == des3_cbc ; + Type == des_ede3 ; + Type == des_ede3_cfb ; + Type == des_ede3_cbf ; + Type == des3_cbf ; + Type == des3_cfb + -> block_cipher_increment(Type, Key, IV, IV, PlainTexts, iolist_to_binary(PlainTexts), CipherText, []); -block_cipher_increment({Type, Key, IV, PlainTexts, _CipherText}) when Type == aes_cbc -> +block_cipher_increment({Type, Key, IV, PlainTexts, _CipherText}) when Type == aes_128_cbc ; + Type == aes_192_cbc ; + Type == aes_256_cbc + -> Plain = iolist_to_binary(PlainTexts), Blocks = [iolistify(Block) || << Block:128/bitstring >> <= Plain], block_cipher_increment(Type, Key, IV, IV, Blocks, Plain, []); @@ -826,48 +1109,56 @@ block_cipher_increment(Type, Key, IV0, IV, [PlainText | PlainTexts], Plain, Ciph NextIV = crypto:next_iv(Type, CT), block_cipher_increment(Type, Key, IV0, NextIV, PlainTexts, Plain, CipherText, [CT | Acc]). -stream_cipher({Type, Key, PlainText}) -> - Plain = iolist_to_binary(PlainText), - State = crypto:stream_init(Type, Key), - {_, CipherText} = crypto:stream_encrypt(State, PlainText), - case crypto:stream_decrypt(State, CipherText) of +stream_cipher({Type, Key, PlainText0}) -> + PlainText = lazy_eval(PlainText0), + Plain = iolist_to_binary(lazy_eval(PlainText)), + StateE = crypto:stream_init(Type, Key), + StateD = crypto:stream_init(Type, Key), + {_, CipherText} = crypto:stream_encrypt(StateE, PlainText), + case crypto:stream_decrypt(StateD, CipherText) of {_, Plain} -> ok; Other -> - ct:fail({{crypto, stream_decrypt, [State, CipherText]}, {expected, PlainText}, {got, Other}}) + ct:fail({{crypto, stream_decrypt, [StateD, CipherText]}, {expected, PlainText}, {got, Other}}) end; -stream_cipher({Type, Key, IV, PlainText}) -> +stream_cipher({Type, Key, IV, PlainText0}) -> + PlainText = lazy_eval(PlainText0), Plain = iolist_to_binary(PlainText), - State = crypto:stream_init(Type, Key, IV), - {_, CipherText} = crypto:stream_encrypt(State, PlainText), - case crypto:stream_decrypt(State, CipherText) of + StateE = crypto:stream_init(Type, Key, IV), + StateD = crypto:stream_init(Type, Key, IV), + {_, CipherText} = crypto:stream_encrypt(StateE, PlainText), + case crypto:stream_decrypt(StateD, CipherText) of {_, Plain} -> ok; Other -> - ct:fail({{crypto, stream_decrypt, [State, CipherText]}, {expected, PlainText}, {got, Other}}) + ct:fail({{crypto, stream_decrypt, [StateD, CipherText]}, {expected, PlainText}, {got, Other}}) end; -stream_cipher({Type, Key, IV, PlainText, CipherText}) -> +stream_cipher({Type, Key, IV, PlainText0, CipherText}) -> + PlainText = lazy_eval(PlainText0), Plain = iolist_to_binary(PlainText), - State = crypto:stream_init(Type, Key, IV), - case crypto:stream_encrypt(State, PlainText) of + StateE = crypto:stream_init(Type, Key, IV), + StateD = crypto:stream_init(Type, Key, IV), + case crypto:stream_encrypt(StateE, PlainText) of {_, CipherText} -> ok; {_, Other0} -> - ct:fail({{crypto, stream_encrypt, [State, Type, Key, IV, Plain]}, {expected, CipherText}, {got, Other0}}) + ct:fail({{crypto, stream_encrypt, [StateE, Type, Key, IV, Plain]}, {expected, CipherText}, {got, Other0}}) end, - case crypto:stream_decrypt(State, CipherText) of + case crypto:stream_decrypt(StateD, CipherText) of {_, Plain} -> ok; Other1 -> - ct:fail({{crypto, stream_decrypt, [State, CipherText]}, {expected, PlainText}, {got, Other1}}) + ct:fail({{crypto, stream_decrypt, [StateD, CipherText]}, {expected, PlainText}, {got, Other1}}) end. stream_cipher_incment({Type, Key, PlainTexts}) -> - State = crypto:stream_init(Type, Key), - stream_cipher_incment_loop(State, State, PlainTexts, [], iolist_to_binary(PlainTexts)); + StateE = crypto:stream_init(Type, Key), + StateD = crypto:stream_init(Type, Key), + stream_cipher_incment_loop(StateE, StateD, PlainTexts, [], iolist_to_binary(PlainTexts)); stream_cipher_incment({Type, Key, IV, PlainTexts}) -> - State = crypto:stream_init(Type, Key, IV), - stream_cipher_incment_loop(State, State, PlainTexts, [], iolist_to_binary(PlainTexts)); + StateE = crypto:stream_init(Type, Key, IV), + StateD = crypto:stream_init(Type, Key, IV), + stream_cipher_incment_loop(StateE, StateD, PlainTexts, [], iolist_to_binary(PlainTexts)); stream_cipher_incment({Type, Key, IV, PlainTexts, _CipherText}) -> stream_cipher_incment({Type, Key, IV, PlainTexts}). @@ -908,7 +1199,7 @@ aead_cipher({Type, Key, PlainText, IV, AAD, CipherText, CipherTag, Info}) -> aead_cipher({Type, Key, PlainText, IV, AAD, CipherText, CipherTag, TagLen, Info}) -> <<TruncatedCipherTag:TagLen/binary, _/binary>> = CipherTag, Plain = iolist_to_binary(PlainText), - case crypto:block_encrypt(Type, Key, IV, {AAD, Plain, TagLen}) of + try crypto:block_encrypt(Type, Key, IV, {AAD, Plain, TagLen}) of {CipherText, TruncatedCipherTag} -> ok; Other0 -> @@ -917,6 +1208,18 @@ aead_cipher({Type, Key, PlainText, IV, AAD, CipherText, CipherTag, TagLen, Info} [{info,Info}, {key,Key}, {pt,PlainText}, {iv,IV}, {aad,AAD}, {ct,CipherText}, {tag,CipherTag}, {taglen,TagLen}]}, {expected, {CipherText, TruncatedCipherTag}}, {got, Other0}}) + catch + error:E -> + ct:log("~p",[{Type, Key, PlainText, IV, AAD, CipherText, CipherTag, TagLen, Info}]), + try crypto:crypto_one_time_aead(Type, Key, IV, PlainText, AAD, TagLen, true) + of + RR -> + ct:log("Works: ~p",[RR]) + catch + CC:EE -> + ct:log("~p:~p", [CC,EE]) + end, + ct:fail("~p",[E]) end, case crypto:block_decrypt(Type, Key, IV, {AAD, CipherText, TruncatedCipherTag}) of Plain -> @@ -987,7 +1290,7 @@ do_sign_verify({Type, Hash, Public, Private, Msg, Options}) -> error:notsup when NotSupLow == true, is_integer(LibVer), LibVer < 16#10001000 -> - %% Thoose opts where introduced in 1.0.1 + %% Those opts where introduced in 1.0.1 ct:log("notsup but OK in old cryptolib crypto:sign(~p, ~p, ..., ..., ..., ~p)", [Type,Hash,Options]), true; @@ -1165,16 +1468,15 @@ do_stream_iolistify({Type, Key, IV, PlainText}) -> {Type, iolistify(Key), IV, iolistify(PlainText)}; do_stream_iolistify({Type, Key, IV, PlainText, CipherText}) -> {Type, iolistify(Key), IV, iolistify(PlainText), CipherText}. - -do_block_iolistify({des_cbc = Type, Key, IV, PlainText}) -> - {Type, Key, IV, des_iolistify(PlainText)}; -do_block_iolistify({des3_cbc = Type, Key, IV, PlainText}) -> - {Type, Key, IV, des_iolistify(PlainText)}; -do_block_iolistify({des3_cbf = Type, Key, IV, PlainText}) -> - {Type, Key, IV, des_iolistify(PlainText)}; -do_block_iolistify({des3_cfb = Type, Key, IV, PlainText}) -> - {Type, Key, IV, des_iolistify(PlainText)}; -do_block_iolistify({des_ede3 = Type, Key, IV, PlainText}) -> +do_block_iolistify({Type, Key, IV, PlainText}) when Type == des_cbc ; + Type == des_ede3_cbc ; + Type == des3_cbc ; + Type == des_ede3 ; + Type == des_ede3_cfb ; + Type == des_ede3_cbf ; + Type == des3_cbf ; + Type == des3_cfb + -> {Type, Key, IV, des_iolistify(PlainText)}; do_block_iolistify({Type, Key, PlainText}) -> {Type, iolistify(Key), iolistify(PlainText)}; @@ -1183,10 +1485,13 @@ do_block_iolistify({Type, Key, IV, PlainText}) -> do_block_iolistify({Type, Key, IV, PlainText, CipherText}) -> {Type, iolistify(Key), IV, iolistify(PlainText), CipherText}. -iolistify(<<"Test With Truncation">>)-> +iolistify(X) -> + iolistify1(lazy_eval(X)). + +iolistify1(<<"Test With Truncation">>)-> %% Do not iolistify as it spoils this special case <<"Test With Truncation">>; -iolistify(Msg) when is_binary(Msg) -> +iolistify1(Msg) when is_binary(Msg) -> Length = erlang:byte_size(Msg), Split = Length div 2, List0 = binary_to_list(Msg), @@ -1196,8 +1501,8 @@ iolistify(Msg) when is_binary(Msg) -> {List1, List2}-> [List1, List2] end; -iolistify(Msg) -> - iolistify(list_to_binary(Msg)). +iolistify1(Msg) when is_list(Msg) -> + iolistify1(list_to_binary(Msg)). des_iolistify(Msg) -> des_iolist(erlang:byte_size(Msg) div 8, Msg, []). @@ -1276,7 +1581,7 @@ rand_uniform_aux_test(N) -> rand_uniform_aux_test(N-1). crypto_rand_uniform(L,H) -> - R1 = crypto:rand_uniform(L, H), + R1 = (L-1) + rand:uniform(H-L), case (R1 >= L) and (R1 < H) of true -> ok; @@ -1444,6 +1749,12 @@ group_config(sha3_384 = Type, Config) -> group_config(sha3_512 = Type, Config) -> {Msgs,Digests} = sha3_test_vectors(Type), [{hash, {Type, Msgs, Digests}}, {hmac, hmac_sha3(Type)} | Config]; +group_config(blake2b = Type, Config) -> + {Msgs, Digests} = blake2_test_vectors(Type), + [{hash, {Type, Msgs, Digests}}, {hmac, blake2_hmac(Type)} | Config]; +group_config(blake2s = Type, Config) -> + {Msgs, Digests} = blake2_test_vectors(Type), + [{hash, {Type, Msgs, Digests}}, {hmac, blake2_hmac(Type)} | Config]; group_config(rsa, Config) -> Msg = rsa_plain(), Public = rsa_public(), @@ -1500,7 +1811,6 @@ group_config(dss = Type, Config) -> MsgPubEnc = <<"7896345786348 Asldi">>, PubPrivEnc = [{dss, Public, Private, MsgPubEnc, []}], [{sign_verify, SignVerify}, {pub_priv_encrypt, PubPrivEnc} | Config]; - group_config(ecdsa = Type, Config) -> {Private, Public} = ec_key_named(), Msg = ec_msg(), @@ -1512,15 +1822,13 @@ group_config(ecdsa = Type, Config) -> MsgPubEnc = <<"7896345786348 Asldi">>, PubPrivEnc = [{ecdsa, Public, Private, MsgPubEnc, []}], [{sign_verify, SignVerify}, {pub_priv_encrypt, PubPrivEnc} | Config]; - group_config(Type, Config) when Type == ed25519 ; Type == ed448 -> TestVectors = eddsa(Type), [{sign_verify,TestVectors} | Config]; - - group_config(srp, Config) -> GenerateCompute = [srp3(), srp6(), srp6a(), srp6a_smaller_prime()], [{generate_compute, GenerateCompute} | Config]; + group_config(ecdh, Config) -> Compute = ecdh(), Generate = ecc(), @@ -1528,77 +1836,19 @@ group_config(ecdh, Config) -> group_config(dh, Config) -> GenerateCompute = [dh()], [{generate_compute, GenerateCompute} | Config]; -group_config(des_cbc, Config) -> - Block = des_cbc(), - [{block, Block} | Config]; -group_config(des_cfb, Config) -> - Block = des_cfb(), - [{block, Block} | Config]; -group_config(des3_cbc, Config) -> - Block = des3_cbc(), - [{block, Block} | Config]; -group_config(des3_cbf, Config) -> - Block = des3_cbf(), - [{block, Block} | Config]; -group_config(des3_cfb, Config) -> - Block = des3_cfb(), - [{block, Block} | Config]; -group_config(des_ede3, Config) -> - Block = des_ede3(), - [{block, Block} | Config]; -group_config(rc2_cbc, Config) -> - Block = rc2_cbc(), - [{block, Block} | Config]; + group_config(aes_cbc128 = Type, Config) -> Block = fun() -> aes_cbc128(Config) end, Pairs = fun() -> cmac_nist(Config, Type) end, - [{block, Block}, {cmac, Pairs} | Config]; + [{cipher, Block}, {cmac, Pairs} | Config]; group_config(aes_cbc256 = Type, Config) -> Block = fun() -> aes_cbc256(Config) end, Pairs = fun() -> cmac_nist(Config, Type) end, - [{block, Block}, {cmac, Pairs} | Config]; -group_config(aes_ecb, Config) -> - Block = fun() -> aes_ecb(Config) end, - [{block, Block} | Config]; -group_config(aes_ige256, Config) -> - Block = aes_ige256(), - [{block, Block} | Config]; -group_config(aes_cfb8, Config) -> - Block = fun() -> aes_cfb8(Config) end, - [{block, Block} | Config]; -group_config(aes_cfb128, Config) -> - Block = fun() -> aes_cfb128(Config) end, - [{block, Block} | Config]; -group_config(blowfish_cbc, Config) -> - Block = blowfish_cbc(), - [{block, Block} | Config]; -group_config(blowfish_ecb, Config) -> - Block = blowfish_ecb(), - [{block, Block} | Config]; -group_config(blowfish_cfb64, Config) -> - Block = blowfish_cfb64(), - [{block, Block} | Config]; -group_config(blowfish_ofb64, Config) -> - Block = blowfish_ofb64(), - [{block, Block} | Config]; -group_config(rc4, Config) -> - Stream = rc4(), - [{stream, Stream} | Config]; -group_config(aes_ctr, Config) -> - Stream = aes_ctr(), - [{stream, Stream} | Config]; -group_config(aes_ccm, Config) -> - AEAD = fun() -> aes_ccm(Config) end, - [{aead, AEAD} | Config]; -group_config(aes_gcm, Config) -> - AEAD = fun() -> aes_gcm(Config) end, - [{aead, AEAD} | Config]; + [{cipher, Block}, {cmac, Pairs} | Config]; group_config(chacha20_poly1305, Config) -> - AEAD = chacha20_poly1305(), - [{aead, AEAD} | Config]; -group_config(chacha20, Config) -> - Stream = chacha20(), - [{stream, Stream} | Config]; + AEAD = chacha20_poly1305(Config), + [{cipher, AEAD} | Config]; + group_config(poly1305, Config) -> V = [%% {Key, Txt, Expect} {%% RFC7539 2.5.2 @@ -1608,11 +1858,12 @@ group_config(poly1305, Config) -> } ], [{poly1305,V} | Config]; -group_config(aes_cbc, Config) -> - Block = aes_cbc(Config), - [{block, Block} | Config]; -group_config(_, Config) -> - Config. + +group_config(F, Config) -> + TestVectors = fun() -> ?MODULE:F(Config) end, + [{cipher, TestVectors} | Config]. + + rsa_sign_verify_tests(Config, Msg, Public, Private, PublicS, PrivateS, OptsToTry) -> case ?config(fips, Config) of @@ -1710,6 +1961,71 @@ rfc_1321_md5_digests() -> hexstr2bin("d174ab98d277d9f5a5611c2c9f419d9f"), hexstr2bin("57edf4a22be3c955ac49da2e2107b67a")]. + +%% BLAKE2 re-use SHA3 test vectors. +blake2_test_vectors(blake2b) -> + {sha3_msgs(), + [ <<186,128,165,63,152,28,77,13,106,39,151,182,159,18,246,233,76,33,47,20,104,90,196,183,75,18,187,111,219,255,162,209,125,135,197,57,42,171,121,45,194,82,213,222,69,51,204,149,24,211,138,168,219,241,146,90,185,35,134,237,212,0,153,35>> + , <<120,106,2,247,66,1,89,3,198,198,253,133,37,82,210,114,145,47,71,64,225,88,71,97,138,134,226,23,247,31,84,25,210,94,16,49,175,238,88,83,19,137,100,68,147,78,176,75,144,58,104,91,20,72,183,85,213,111,112,26,254,155,226,206>> + , <<114,133,255,62,139,215,104,214,155,230,43,59,241,135,101,163,37,145,127,169,116,74,194,245,130,162,8,80,188,43,17,65,237,27,62,69,40,89,90,204,144,119,43,223,45,55,220,138,71,19,11,68,243,58,2,232,115,14,90,216,225,102,232,136>> + , <<206,116,26,197,147,15,227,70,129,17,117,197,34,123,183,191,205,71,244,38,18,250,228,108,8,9,81,79,158,14,58,17,238,23,115,40,113,71,205,234,238,223,245,7,9,170,113,99,65,254,101,36,15,74,214,119,125,107,250,249,114,110,94,82>> + , <<152,251,62,251,114,6,253,25,235,246,155,111,49,44,247,182,78,59,148,219,225,161,113,7,145,57,117,167,147,241,119,225,208,119,96,157,127,186,54,60,187,160,13,5,247,170,78,79,168,113,93,100,40,16,76,10,117,100,59,15,243,253,62,175>> + ]}; +blake2_test_vectors(blake2s) -> + {sha3_msgs(), + [ <<80,140,94,140,50,124,20,226,225,167,43,163,78,235,69,47,55,69,139,32,158,214,58,41,77,153,155,76,134,103,89,130>> + , <<105,33,122,48,121,144,128,148,225,17,33,208,66,53,74,124,31,85,182,72,44,161,165,30,27,37,13,253,30,208,238,249>> + , <<111,77,245,17,106,111,51,46,218,177,217,225,14,232,125,246,85,123,234,182,37,157,118,99,243,188,213,114,44,19,241,137>> + , <<53,141,210,237,7,128,212,5,78,118,203,111,58,91,206,40,65,232,226,245,71,67,29,77,9,219,33,182,109,148,31,199>> + , <<190,192,192,230,205,229,182,122,203,115,184,31,121,166,122,64,121,174,28,96,218,201,210,102,26,241,142,159,139,80,223,165>> + ]}. + +blake2_hmac(Type) -> + {Ks, Ds, Hs} = lists:unzip3( + [ {hexstr2bin(K), hexstr2bin(D), H} + || {{K, D}, H} <- lists:zip(blake2_hmac_key_data(), blake2_hmac_hmac(Type)) ]), + {Type, Ks, Ds, Hs}. + +blake2_hmac_key_data() -> + [ {"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b 0b0b0b0b", + "4869205468657265"} + , {"4a656665", + "7768617420646f2079612077616e7420 666f72206e6f7468696e673f"} + , {"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaa", + "dddddddddddddddddddddddddddddddd dddddddddddddddddddddddddddddddd dddddddddddddddddddddddddddddddd dddd"} + , {"0102030405060708090a0b0c0d0e0f10 111213141516171819", + "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd cdcd"} + , {"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaa", + "54657374205573696e67204c61726765 72205468616e20426c6f636b2d53697a 65204b6579202d2048617368204b6579 204669727374"} + , {"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaa", + "54657374205573696e67204c61726765 72205468616e20426c6f636b2d53697a 65204b6579202d2048617368204b6579 204669727374"} + , {"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaa", + "54686973206973206120746573742075 73696e672061206c6172676572207468 616e20626c6f636b2d73697a65206b65 7920616e642061206c61726765722074 68616e20626c6f636b2d73697a652064 6174612e20546865206b6579206e6565 647320746f2062652068617368656420 6265666f7265206265696e6720757365 642062792074686520484d414320616c 676f726974686d2e"} + , {"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaa", + "54686973206973206120746573742075 73696e672061206c6172676572207468 616e20626c6f636b2d73697a65206b65 7920616e642061206c61726765722074 68616e20626c6f636b2d73697a652064 6174612e20546865206b6579206e6565 647320746f2062652068617368656420 6265666f7265206265696e6720757365 642062792074686520484d414320616c 676f726974686d2e"} + ]. + +blake2_hmac_hmac(blake2b) -> + [ <<53,138,106,24,73,36,137,79,195,75,238,86,128,238,223,87,216,74,55,187,56,131,47,40,142,59,39,220,99,169,140,200,201,30,118,218,71,107,80,139,198,178,212,8,162,72,133,116,82,144,110,74,32,180,140,107,75,85,210,223,15,225,221,36>> + , <<111,248,132,248,221,194,166,88,107,60,152,164,205,110,189,241,78,193,2,4,182,113,0,115,235,88,101,173,227,122,38,67,184,128,124,19,53,209,7,236,219,159,254,174,182,130,140,70,37,186,23,44,102,55,158,252,210,34,194,222,17,114,122,180>> + , <<244,59,198,44,122,153,53,60,59,44,96,232,239,36,251,189,66,233,84,120,102,220,156,91,228,237,198,244,167,212,188,10,198,32,194,198,0,52,208,64,240,219,175,134,249,233,205,120,145,160,149,89,94,237,85,226,169,150,33,95,12,21,192,24>> + , <<229,219,182,222,47,238,66,161,202,160,110,78,123,132,206,64,143,250,92,74,157,226,99,46,202,118,156,222,136,117,1,76,114,208,114,15,234,245,63,118,230,161,128,53,127,82,141,123,244,132,250,58,20,232,204,31,15,59,173,167,23,180,52,145>> + , <<165,75,41,67,178,162,2,39,212,28,164,108,9,69,175,9,188,31,174,251,47,73,137,76,35,174,188,85,127,183,156,72,137,220,167,68,8,220,134,80,134,102,122,237,238,74,49,133,197,58,73,200,11,129,76,76,88,19,234,12,139,56,168,248>> + , <<180,214,140,139,182,82,151,170,52,132,168,110,29,51,183,138,70,159,33,234,170,158,212,218,159,236,145,218,71,23,34,61,44,15,163,134,170,47,209,241,255,207,89,23,178,103,84,96,53,237,48,238,164,178,19,162,133,148,211,211,169,179,140,170>> + , <<171,52,121,128,166,75,94,130,93,209,14,125,50,253,67,160,26,142,109,234,38,122,185,173,125,145,53,36,82,102,24,146,83,17,175,188,176,196,149,25,203,235,221,112,149,64,168,215,37,251,145,26,194,174,233,178,163,170,67,215,150,18,51,147>> + , <<97,220,242,140,166,12,169,92,130,89,147,39,171,215,169,161,152,111,242,219,211,199,73,69,198,227,35,186,203,76,159,26,94,103,82,93,20,186,141,98,36,177,98,229,102,23,21,37,83,3,69,169,178,86,8,178,125,251,163,180,146,115,213,6>> + ]; +blake2_hmac_hmac(blake2s) -> + [ <<101,168,183,197,204,145,54,212,36,232,44,55,226,112,126,116,233,19,192,101,91,153,199,95,64,237,243,135,69,58,50,96>> + , <<144,182,40,30,47,48,56,201,5,106,240,180,167,231,99,202,230,254,93,158,180,56,106,14,201,82,55,137,12,16,79,240>> + , <<252,196,245,149,41,80,46,52,195,216,218,63,253,171,130,150,106,44,182,55,255,94,155,215,1,19,92,46,148,105,231,144>> + , <<70,68,52,220,190,206,9,93,69,106,29,98,214,236,86,248,152,230,37,163,158,92,82,189,249,77,175,17,27,173,131,170>> + , <<210,61,121,57,79,83,213,54,160,150,230,81,68,71,238,170,187,5,222,208,27,227,44,25,55,218,106,143,113,3,188,78>> + , <<92,76,83,46,110,69,89,83,133,78,21,16,149,38,110,224,127,213,88,129,190,223,139,57,8,217,95,13,190,54,159,234>> + , <<203,96,246,167,145,241,64,191,138,162,229,31,243,88,205,178,204,92,3,51,4,91,127,183,122,186,122,179,176,207,178,55>> + , <<190,53,233,217,99,171,215,108,1,184,171,181,22,36,240,209,16,96,16,92,213,22,16,58,114,241,117,214,211,189,30,202>> + ]. + %%% https://www.di-mgt.com.au/sha_testvectors.html sha3_msgs() -> ["abc", @@ -2138,19 +2454,19 @@ rfc4231_hmac_sha512() -> "debd71f8867289865df5a32d20cdc944" "b6022cac3c4982b10d5eeb55c3e4de15" "134676fb6de0446065c97440fa8c6a58")]. -des_cbc() -> +des_cbc(_) -> [{des_cbc, hexstr2bin("0123456789abcdef"), hexstr2bin("1234567890abcdef"), <<"Now is the time for all ">> }]. -des_cfb() -> +des_cfb(_) -> [{des_cfb, hexstr2bin("0123456789abcdef"), hexstr2bin("1234567890abcdef"), <<"Now is the">>}]. -des3_cbc() -> +des3_cbc(_) -> [{des3_cbc, [hexstr2bin("0123456789abcdef"), hexstr2bin("fedcba9876543210"), @@ -2159,7 +2475,7 @@ des3_cbc() -> <<"Now is the time for all ">> }]. -des_ede3() -> +des_ede3(_) -> [{des_ede3, [hexstr2bin("8000000000000000"), hexstr2bin("4000000000000000"), @@ -2168,7 +2484,23 @@ des_ede3() -> hexstr2bin("0000000000000000") }]. -des3_cbf() -> +des_ede3_cbc(_) -> + [{des_ede3_cbc, + [hexstr2bin("0123456789abcdef"), + hexstr2bin("fedcba9876543210"), + hexstr2bin("0f2d4b6987a5c3e1")], + hexstr2bin("1234567890abcdef"), + <<"Now is the time for all ">> + }, + {des_ede3_cbc, + [hexstr2bin("8000000000000000"), + hexstr2bin("4000000000000000"), + hexstr2bin("2000000000000000")], + hexstr2bin("7AD16FFB79C45926"), + hexstr2bin("0000000000000000") + }]. + +des3_cbf(_) -> [{des3_cbf, [hexstr2bin("0123456789abcdef"), hexstr2bin("fedcba9876543210"), @@ -2177,7 +2509,7 @@ des3_cbf() -> <<"Now is the time for all ">> }]. -des3_cfb() -> +des3_cfb(_) -> [{des3_cfb, [hexstr2bin("0123456789abcdef"), hexstr2bin("fedcba9876543210"), @@ -2186,7 +2518,16 @@ des3_cfb() -> <<"Now is the time for all ">> }]. -rc2_cbc() -> +des_ede3_cfb(_) -> + [{des_ede3_cfb, + [hexstr2bin("0123456789abcdef"), + hexstr2bin("fedcba9876543210"), + hexstr2bin("0f2d4b6987a5c3e1")], + hexstr2bin("1234567890abcdef"), + <<"Now is the time for all ">> + }]. + +rc2_cbc(_) -> [{rc2_cbc, <<146,210,160,124,215,227,153,239,227,17,222,140,3,93,27,191>>, <<72,91,135,182,25,42,35,210>>, @@ -2195,7 +2536,8 @@ rc2_cbc() -> %% AES CBC test vectors from http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf aes_cbc(Config) -> - read_rsp(Config, aes_cbc, + %% RETIRED aes_*_cbc + read_rsp(Config, aes_cbc, ["CBCVarTxt128.rsp", "CBCVarKey128.rsp", "CBCGFSbox128.rsp", "CBCKeySbox128.rsp", "CBCVarTxt192.rsp", "CBCVarKey192.rsp", "CBCGFSbox192.rsp", "CBCKeySbox192.rsp", "CBCVarTxt256.rsp", "CBCVarKey256.rsp", "CBCGFSbox256.rsp", "CBCKeySbox256.rsp", @@ -2203,15 +2545,32 @@ aes_cbc(Config) -> ]). aes_cbc128(Config) -> + %% RETIRED aes_128_cbc read_rsp(Config, aes_cbc128, ["CBCVarTxt128.rsp", "CBCVarKey128.rsp", "CBCGFSbox128.rsp", "CBCKeySbox128.rsp", "CBCMMT128.rsp"]). aes_cbc256(Config) -> + %% RETIRED aes_256_cbc read_rsp(Config, aes_cbc256, ["CBCVarTxt256.rsp", "CBCVarKey256.rsp", "CBCGFSbox256.rsp", "CBCKeySbox256.rsp", "CBCMMT256.rsp"]). +aes_128_cbc(Config) -> + read_rsp(Config, aes_128_cbc, + ["CBCVarTxt128.rsp", "CBCVarKey128.rsp", "CBCGFSbox128.rsp", "CBCKeySbox128.rsp", + "CBCMMT128.rsp"]). + +aes_192_cbc(Config) -> + read_rsp(Config, aes_192_cbc, + ["CBCVarTxt192.rsp", "CBCVarKey192.rsp", "CBCGFSbox192.rsp", "CBCKeySbox192.rsp", + "CBCMMT192.rsp"]). + +aes_256_cbc(Config) -> + read_rsp(Config, aes_256_cbc, + ["CBCVarTxt256.rsp", "CBCVarKey256.rsp", "CBCGFSbox256.rsp", "CBCKeySbox256.rsp", + "CBCMMT256.rsp"]). + aes_ecb(Config) -> read_rsp(Config, aes_ecb, ["ECBVarTxt128.rsp", "ECBVarKey128.rsp", "ECBGFSbox128.rsp", "ECBKeySbox128.rsp", @@ -2219,7 +2578,22 @@ aes_ecb(Config) -> "ECBVarTxt256.rsp", "ECBVarKey256.rsp", "ECBGFSbox256.rsp", "ECBKeySbox256.rsp", "ECBMMT128.rsp", "ECBMMT192.rsp", "ECBMMT256.rsp"]). -aes_ige256() -> +aes_128_ecb(Config) -> + read_rsp(Config, aes_128_ecb, + ["ECBVarTxt128.rsp", "ECBVarKey128.rsp", "ECBGFSbox128.rsp", "ECBKeySbox128.rsp", + "ECBMMT128.rsp"]). + +aes_192_ecb(Config) -> + read_rsp(Config, aes_192_ecb, + ["ECBVarTxt192.rsp", "ECBVarKey192.rsp", "ECBGFSbox192.rsp", "ECBKeySbox192.rsp", + "ECBMMT192.rsp"]). + +aes_256_ecb(Config) -> + read_rsp(Config, aes_256_ecb, + ["ECBVarTxt256.rsp", "ECBVarKey256.rsp", "ECBGFSbox256.rsp", "ECBKeySbox256.rsp", + "ECBMMT256.rsp"]). + +aes_ige256(_) -> [{aes_ige256, hexstr2bin("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4"), hexstr2bin("000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F"), @@ -2245,6 +2619,22 @@ aes_cfb8(Config) -> "CFB8VarTxt256.rsp", "CFB8VarKey256.rsp", "CFB8GFSbox256.rsp", "CFB8KeySbox256.rsp", "CFB8MMT128.rsp", "CFB8MMT192.rsp", "CFB8MMT256.rsp"]). +aes_128_cfb8(Config) -> + read_rsp(Config, aes_128_cfb8, + ["CFB8VarTxt128.rsp", "CFB8VarKey128.rsp", "CFB8GFSbox128.rsp", "CFB8KeySbox128.rsp", + "CFB8MMT128.rsp"]). + +aes_192_cfb8(Config) -> + read_rsp(Config, aes_192_cfb8, + ["CFB8VarTxt192.rsp", "CFB8VarKey192.rsp", "CFB8GFSbox192.rsp", "CFB8KeySbox192.rsp", + "CFB8MMT192.rsp"]). + +aes_256_cfb8(Config) -> + read_rsp(Config, aes_256_cfb8, + ["CFB8VarTxt256.rsp", "CFB8VarKey256.rsp", "CFB8GFSbox256.rsp", "CFB8KeySbox256.rsp", + "CFB8MMT256.rsp"]). + + aes_cfb128(Config) -> read_rsp(Config, aes_cfb128, ["CFB128VarTxt128.rsp", "CFB128VarKey128.rsp", "CFB128GFSbox128.rsp", "CFB128KeySbox128.rsp", @@ -2252,14 +2642,30 @@ aes_cfb128(Config) -> "CFB128VarTxt256.rsp", "CFB128VarKey256.rsp", "CFB128GFSbox256.rsp", "CFB128KeySbox256.rsp", "CFB128MMT128.rsp", "CFB128MMT192.rsp", "CFB128MMT256.rsp"]). -blowfish_cbc() -> +aes_128_cfb128(Config) -> + read_rsp(Config, aes_128_cfb128, + ["CFB128VarTxt128.rsp", "CFB128VarKey128.rsp", "CFB128GFSbox128.rsp", "CFB128KeySbox128.rsp", + "CFB128MMT128.rsp"]). + +aes_192_cfb128(Config) -> + read_rsp(Config, aes_192_cfb128, + ["CFB128VarTxt192.rsp", "CFB128VarKey192.rsp", "CFB128GFSbox192.rsp", "CFB128KeySbox192.rsp", + "CFB128MMT192.rsp"]). + +aes_256_cfb128(Config) -> + read_rsp(Config, aes_256_cfb128, + ["CFB128VarTxt256.rsp", "CFB128VarKey256.rsp", "CFB128GFSbox256.rsp", "CFB128KeySbox256.rsp", + "CFB128MMT256.rsp"]). + + +blowfish_cbc(_) -> [{blowfish_cbc, hexstr2bin("0123456789ABCDEFF0E1D2C3B4A59687"), hexstr2bin("FEDCBA9876543210"), hexstr2bin("37363534333231204E6F77206973207468652074696D6520666F722000000000") }]. -blowfish_ecb() -> +blowfish_ecb(_) -> [ {blowfish_ecb, hexstr2bin("0000000000000000"), @@ -2356,26 +2762,26 @@ blowfish_ecb() -> hexstr2bin("FFFFFFFFFFFFFFFF")} ]. -blowfish_cfb64() -> +blowfish_cfb64(_) -> [{blowfish_cfb64, hexstr2bin("0123456789ABCDEFF0E1D2C3B4A59687"), hexstr2bin("FEDCBA9876543210"), hexstr2bin("37363534333231204E6F77206973207468652074696D6520666F722000") }]. -blowfish_ofb64() -> +blowfish_ofb64(_) -> [{blowfish_ofb64, hexstr2bin("0123456789ABCDEFF0E1D2C3B4A59687"), hexstr2bin("FEDCBA9876543210"), hexstr2bin("37363534333231204E6F77206973207468652074696D6520666F722000") }]. -rc4() -> +rc4(_) -> [{rc4, <<"apaapa">>, <<"Yo baby yo">>}, {rc4, <<"apaapa">>, list_to_binary(lists:seq(0, 255))}, {rc4, <<"apaapa">>, long_msg()} ]. -aes_ctr() -> +aes_ctr(_) -> [ %% F.5.3 CTR-AES192.Encrypt {aes_ctr, hexstr2bin("2b7e151628aed2a6abf7158809cf4f3c"), hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"), @@ -2424,24 +2830,109 @@ aes_ctr() -> ]. +aes_128_ctr(_) -> + [ %% F.5.3 CTR-AES192.Encrypt + {aes_128_ctr, hexstr2bin("2b7e151628aed2a6abf7158809cf4f3c"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"), + hexstr2bin("6bc1bee22e409f96e93d7e117393172a")}, + {aes_128_ctr, hexstr2bin("2b7e151628aed2a6abf7158809cf4f3c"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdff00"), + hexstr2bin("ae2d8a571e03ac9c9eb76fac45af8e51")}, + {aes_128_ctr, hexstr2bin("2b7e151628aed2a6abf7158809cf4f3c"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdff01"), + hexstr2bin("30c81c46a35ce411e5fbc1191a0a52ef") }, + {aes_128_ctr, hexstr2bin("2b7e151628aed2a6abf7158809cf4f3c"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdff02"), + hexstr2bin("f69f2445df4f9b17ad2b417be66c3710")} + ]. + +aes_192_ctr(_) -> + [ %% F.5.3 CTR-AES192.Encrypt + {aes_192_ctr, hexstr2bin("8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"), + hexstr2bin("6bc1bee22e409f96e93d7e117393172a")}, + {aes_192_ctr, hexstr2bin("8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdff00"), + hexstr2bin("ae2d8a571e03ac9c9eb76fac45af8e51")}, + {aes_192_ctr, hexstr2bin("8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdff01"), + hexstr2bin("30c81c46a35ce411e5fbc1191a0a52ef")}, + {aes_192_ctr, hexstr2bin("8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdff02"), + hexstr2bin("f69f2445df4f9b17ad2b417be66c3710")} + ]. + +aes_256_ctr(_) -> + [ %% F.5.5 CTR-AES256.Encrypt + {aes_256_ctr, hexstr2bin("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"), + hexstr2bin("6bc1bee22e409f96e93d7e117393172a")}, + {aes_256_ctr, hexstr2bin("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdff00"), + hexstr2bin("ae2d8a571e03ac9c9eb76fac45af8e51")}, + {aes_256_ctr, hexstr2bin("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdff01"), + hexstr2bin("30c81c46a35ce411e5fbc1191a0a52ef")}, + {aes_256_ctr, hexstr2bin("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdff02"), + hexstr2bin("f69f2445df4f9b17ad2b417be66c3710")}, + + {aes_256_ctr, hexstr2bin("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4"), + hexstr2bin("f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"), + long_msg()} + ]. + + aes_gcm(Config) -> - read_rsp(Config, aes_gcm, + %% RETIRED aes_*_gcm + read_rsp(Config, aes_gcm, + ["gcmDecrypt128.rsp", + "gcmDecrypt192.rsp", + "gcmDecrypt256.rsp", + "gcmEncryptExtIV128.rsp", + "gcmEncryptExtIV192.rsp", + "gcmEncryptExtIV256.rsp"]). + +aes_128_gcm(Config) -> + read_rsp(Config, aes_128_gcm, ["gcmDecrypt128.rsp", - "gcmDecrypt192.rsp", - "gcmDecrypt256.rsp", - "gcmEncryptExtIV128.rsp", - "gcmEncryptExtIV192.rsp", + "gcmEncryptExtIV128.rsp"]). + +aes_192_gcm(Config) -> + read_rsp(Config, aes_192_gcm, + ["gcmDecrypt192.rsp", + "gcmEncryptExtIV192.rsp"]). + +aes_256_gcm(Config) -> + read_rsp(Config, aes_256_gcm, + ["gcmDecrypt256.rsp", "gcmEncryptExtIV256.rsp"]). + aes_ccm(Config) -> - read_rsp(Config, aes_ccm, - ["VADT128.rsp", "VADT192.rsp", "VADT256.rsp", - "VNT128.rsp", "VNT192.rsp", "VNT256.rsp", - "VPT128.rsp", "VPT192.rsp", "VPT256.rsp" - ]). + %% RETIRED aes_*_ccm + read_rsp(Config, aes_ccm, + ["VADT128.rsp", "VADT192.rsp", "VADT256.rsp", + "VNT128.rsp", "VNT192.rsp", "VNT256.rsp", + "VPT128.rsp", "VPT192.rsp", "VPT256.rsp" + ]). + +aes_128_ccm(Config) -> + read_rsp(Config, aes_128_ccm, + ["VADT128.rsp", "VNT128.rsp", "VPT128.rsp"]). + +aes_192_ccm(Config) -> + read_rsp(Config, aes_192_ccm, + ["VADT192.rsp", "VNT192.rsp", "VPT192.rsp"]). + +aes_256_ccm(Config) -> + read_rsp(Config, aes_256_ccm, + ["VADT256.rsp", "VNT256.rsp", "VPT256.rsp"]). + + %% https://tools.ietf.org/html/rfc7539#appendix-A.5 -chacha20_poly1305() -> +chacha20_poly1305(_) -> [ {chacha20_poly1305, hexstr2bin("1c9240a5eb55d38af333888604f6b5f0" %% Key @@ -2488,7 +2979,7 @@ chacha20_poly1305() -> ]. -chacha20() -> +chacha20(_) -> %%% chacha20 (no mode) test vectors from RFC 7539 A.2 [ %% Test Vector #1: @@ -3422,9 +3913,18 @@ parse_rsp(_Type, [], _State, Acc) -> Acc; parse_rsp(_Type, [<<"DECRYPT">>|_], _State, Acc) -> Acc; +parse_rsp(_Type, [<<"ENCRYPT">>|_], _State, Acc) -> + Acc; %% AES format parse_rsp(Type, [<<"COUNT = ", _/binary>>, <<"KEY = ", Key/binary>>, + <<"PLAINTEXT = ", PlainText/binary>>, + <<"CIPHERTEXT = ", CipherText/binary>>|Next], State, Acc) -> + parse_rsp(Type, Next, State, + [{Type, hexstr2bin(Key), + hexstr2bin(PlainText), hexstr2bin(CipherText)}|Acc]); +parse_rsp(Type, [<<"COUNT = ", _/binary>>, + <<"KEY = ", Key/binary>>, <<"IV = ", IV/binary>>, <<"PLAINTEXT = ", PlainText/binary>>, <<"CIPHERTEXT = ", CipherText/binary>>|Next], State, Acc) -> diff --git a/lib/crypto/test/engine_SUITE.erl b/lib/crypto/test/engine_SUITE.erl index 869db516b4..41cd132734 100644 --- a/lib/crypto/test/engine_SUITE.erl +++ b/lib/crypto/test/engine_SUITE.erl @@ -148,8 +148,21 @@ end_per_group(_, Config) -> end. %%-------------------------------------------------------------------- -init_per_testcase(_Case, Config) -> - Config. +init_per_testcase(Case, Config) -> + case string:tokens(atom_to_list(Case),"_") of + ["sign","verify",Type|_] -> + skip_if_unsup(list_to_atom(Type), Config); + + ["priv","encrypt","pub","decrypt",Type|_] -> + skip_if_unsup(list_to_atom(Type), Config); + + ["get","pub","from","priv","key",Type|_] -> + skip_if_unsup(list_to_atom(Type), Config); + + _ -> + Config + end. + end_per_testcase(_Case, _Config) -> ok. @@ -674,7 +687,7 @@ ensure_load(Config) when is_list(Config) -> end. %%%---------------------------------------------------------------- -%%% Pub/priv key storage tests. Thoose are for testing the crypto.erl +%%% Pub/priv key storage tests. Those are for testing the crypto.erl %%% support for using priv/pub keys stored in an engine. sign_verify_rsa(Config) -> @@ -851,6 +864,19 @@ get_pub_from_priv_key_ecdsa(Config) -> %%%================================================================ %%% Help for engine_stored_pub_priv_keys* test cases %%% +skip_if_unsup(Type, Config) -> + case pkey_supported(Type) of + false -> + {skip, "Unsupported in this cryptolib"}; + true -> + Config + end. + + +pkey_supported(Type) -> + lists:member(Type, proplists:get_value(public_keys, crypto:supports(), [])). + + load_storage_engine(Config) -> load_storage_engine(Config, []). diff --git a/lib/debugger/src/dbg_icmd.erl b/lib/debugger/src/dbg_icmd.erl index ac901c5469..0eb258567f 100644 --- a/lib/debugger/src/dbg_icmd.erl +++ b/lib/debugger/src/dbg_icmd.erl @@ -248,7 +248,7 @@ handle_int_msg({attached, AttPid}, Status, _Bs, tell_attached({attached, M, Line, get(trace)}), %% Give info about status and call level as well - %% In this case, Status can not be exit_at + %% In this case, Status cannot be exit_at Msg = case Status of idle -> {func_at,M,Line,Le}; break -> {break_at,M,Line,Le}; diff --git a/lib/debugger/test/int_eval_SUITE.erl b/lib/debugger/test/int_eval_SUITE.erl index 0542e45142..324a44bad8 100644 --- a/lib/debugger/test/int_eval_SUITE.erl +++ b/lib/debugger/test/int_eval_SUITE.erl @@ -285,7 +285,10 @@ do_eval(Config, Mod) -> DataDir = proplists:get_value(data_dir, Config), ok = file:set_cwd(DataDir), - {ok,Mod} = compile:file(Mod, [report,debug_info]), + %% Turn off type-based optimizations across function calls, as it + %% would turn many body-recursive calls into tail-recursive calls, + %% which would change the stacktrace. + {ok,Mod} = compile:file(Mod, [no_module_opt,report,debug_info]), {module,Mod} = code:load_file(Mod), CompiledRes = Mod:Mod(), ok = io:format("Compiled:\n~p", [CompiledRes]), diff --git a/lib/dialyzer/src/Makefile b/lib/dialyzer/src/Makefile index fc08e7ca2f..bddd761705 100644 --- a/lib/dialyzer/src/Makefile +++ b/lib/dialyzer/src/Makefile @@ -90,8 +90,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif -ERL_COMPILE_FLAGS += +warn_export_vars +warn_unused_import +warn_untyped_record +warn_missing_spec +warnings_as_errors +ERL_COMPILE_FLAGS += +warn_export_vars +warn_unused_import +warn_untyped_record +warn_missing_spec # ---------------------------------------------------- # Targets diff --git a/lib/dialyzer/src/dialyzer.erl b/lib/dialyzer/src/dialyzer.erl index 185c8c9ae6..a168b3c8c5 100644 --- a/lib/dialyzer/src/dialyzer.erl +++ b/lib/dialyzer/src/dialyzer.erl @@ -320,9 +320,12 @@ message_to_string({call_to_missing, [M, F, A]}) -> message_to_string({exact_eq, [Type1, Op, Type2]}) -> io_lib:format("The test ~ts ~s ~ts can never evaluate to 'true'\n", [Type1, Op, Type2]); -message_to_string({fun_app_args, [Args, Type]}) -> +message_to_string({fun_app_args, [ArgNs, Args, Type]}) -> + PositionString = form_position_string(ArgNs), io_lib:format("Fun application with arguments ~ts will fail" - " since the function has type ~ts\n", [Args, Type]); + " since the function has type ~ts," + " which differs in the ~s argument\n", + [Args, Type, PositionString]); message_to_string({fun_app_no_fun, [Op, Type, Arity]}) -> io_lib:format("Fun application will fail since ~ts :: ~ts" " is not a function of arity ~w\n", [Op, Type, Arity]); diff --git a/lib/dialyzer/src/dialyzer_dataflow.erl b/lib/dialyzer/src/dialyzer_dataflow.erl index 45b4abb253..f7aa167f5c 100644 --- a/lib/dialyzer/src/dialyzer_dataflow.erl +++ b/lib/dialyzer/src/dialyzer_dataflow.erl @@ -405,8 +405,13 @@ handle_apply(Tree, Map, State) -> t_fun_args(OpType1, 'universe')), case any_none(NewArgs) of true -> + EnumNewArgs = lists:zip(lists:seq(1, length(NewArgs)), + NewArgs), + ArgNs = [Arg || + {Arg, Type} <- EnumNewArgs, t_is_none(Type)], Msg = {fun_app_args, - [format_args(Args, ArgTypes, State), + [ArgNs, + format_args(Args, ArgTypes, State), format_type(OpType, State)]}, State3 = state__add_warning(State2, ?WARN_FAILING_CALL, Tree, Msg), diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index 310301ee0b..245c099fef 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -46,6 +46,7 @@ ]). -include("dialyzer.hrl"). +-include("../../compiler/src/core_parse.hrl"). %%-define(DEBUG, true). @@ -652,7 +653,7 @@ sets_filter([Mod|Mods], ExpTypes) -> src_compiler_opts() -> [no_copt, to_core, binary, return_errors, no_inline, strict_record_tests, strict_record_updates, - dialyzer]. + dialyzer, no_spawn_compiler_process]. -spec format_errors([{module(), string()}]) -> [string()]. @@ -751,9 +752,13 @@ pp_hook(Node, Ctxt, Cont) -> map -> pp_map(Node, Ctxt, Cont); literal -> - case is_map(cerl:concrete(Node)) of - true -> pp_map(Node, Ctxt, Cont); - false -> Cont(Node, Ctxt) + case cerl:concrete(Node) of + Map when is_map(Map) -> + pp_map(Node, Ctxt, Cont); + Bitstr when is_bitstring(Bitstr) -> + pp_binary(Node, Ctxt, Cont); + _ -> + Cont(Node, Ctxt) end; _ -> Cont(Node, Ctxt) @@ -761,7 +766,7 @@ pp_hook(Node, Ctxt, Cont) -> pp_binary(Node, Ctxt, Cont) -> prettypr:beside(prettypr:text("<<"), - prettypr:beside(pp_segments(cerl:binary_segments(Node), + prettypr:beside(pp_segments(cerl_binary_segments(Node), Ctxt, Cont), prettypr:text(">>"))). @@ -780,10 +785,29 @@ pp_segment(Node, Ctxt, Cont) -> Unit = cerl:bitstr_unit(Node), Type = cerl:bitstr_type(Node), Flags = cerl:bitstr_flags(Node), - prettypr:beside(Cont(Val, Ctxt), - prettypr:beside(pp_size(Size, Ctxt, Cont), - prettypr:beside(pp_opts(Type, Flags), - pp_unit(Unit, Ctxt, Cont)))). + RestPP = + case {concrete(Unit), concrete(Type), concrete(Flags)} of + {1, integer, [unsigned, big]} -> % Simplify common cases. + case concrete(Size) of + 8 -> prettypr:text(""); + _ -> pp_size(Size, Ctxt, Cont) + end; + {8, binary, [unsigned, big]} -> + SizePP = pp_size(Size, Ctxt, Cont), + prettypr:beside(SizePP, + prettypr:beside(prettypr:text("/"), pp_atom(Type))); + _What -> + SizePP = pp_size(Size, Ctxt, Cont), + UnitPP = pp_unit(Unit, Ctxt, Cont), + OptsPP = pp_opts(Type, Flags), + prettypr:beside(SizePP, prettypr:beside(OptsPP, UnitPP)) + end, + prettypr:beside(Cont(Val, Ctxt), RestPP). + +concrete(Cerl) -> + try cerl:concrete(Cerl) + catch _:_ -> anything_unexpected + end. pp_size(Size, Ctxt, Cont) -> case cerl:is_c_atom(Size) of @@ -859,6 +883,31 @@ seq([H | T], Separator, Ctxt, Fun) -> seq([], _, _, _) -> [prettypr:empty()]. +cerl_binary_segments(#c_literal{val = B}) when is_bitstring(B) -> + segs_from_bitstring(B); +cerl_binary_segments(CBinary) -> + cerl:binary_segments(CBinary). + +%% Copied from core_pp. The function cerl:binary_segments/2 should/could +%% be extended to handle literals, but then the cerl module cannot be +%% HiPE-compiled as of Erlang/OTP 22.0 (due to <<I:N>>). +segs_from_bitstring(<<H,T/bitstring>>) -> + [#c_bitstr{val=#c_literal{val=H}, + size=#c_literal{val=8}, + unit=#c_literal{val=1}, + type=#c_literal{val=integer}, + flags=#c_literal{val=[unsigned,big]}}|segs_from_bitstring(T)]; +segs_from_bitstring(<<>>) -> + []; +segs_from_bitstring(Bitstring) -> + N = bit_size(Bitstring), + <<I:N>> = Bitstring, + [#c_bitstr{val=#c_literal{val=I}, + size=#c_literal{val=N}, + unit=#c_literal{val=1}, + type=#c_literal{val=integer}, + flags=#c_literal{val=[unsigned,big]}}]. + %%------------------------------------------------------------------------------ -spec refold_pattern(cerl:cerl()) -> cerl:cerl(). diff --git a/lib/dialyzer/src/typer.erl b/lib/dialyzer/src/typer.erl index 4b99f5f72e..5a1b8619c9 100644 --- a/lib/dialyzer/src/typer.erl +++ b/lib/dialyzer/src/typer.erl @@ -980,7 +980,7 @@ fatal_error(Slogan) -> mode_error(OldMode, NewMode) -> Msg = io_lib:format("Mode was previously set to '~s'; " - "can not set it to '~s' now", + "cannot set it to '~s' now", [OldMode, NewMode]), fatal_error(Msg). diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/simple b/lib/dialyzer/test/opaque_SUITE_data/results/simple index 5cd8916aee..0e1bb934e9 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/simple +++ b/lib/dialyzer/test/opaque_SUITE_data/results/simple @@ -63,9 +63,9 @@ simple1_api.erl:381: Invalid type specification for function simple1_api:bool_ad simple1_api.erl:407: The size simple1_adt:i1() breaks the opacity of A simple1_api.erl:418: The attempt to match a term of type non_neg_integer() against the variable A breaks the opacity of simple1_adt:i1() simple1_api.erl:425: The attempt to match a term of type non_neg_integer() against the variable B breaks the opacity of simple1_adt:i1() -simple1_api.erl:432: The pattern <<_:B/integer-unit:1>> can never match the type any() +simple1_api.erl:432: The pattern <<_:B>> can never match the type any() simple1_api.erl:448: The attempt to match a term of type non_neg_integer() against the variable Sz breaks the opacity of simple1_adt:i1() -simple1_api.erl:460: The attempt to match a term of type simple1_adt:bit1() against the pattern <<_/binary-unit:8>> breaks the opacity of the term +simple1_api.erl:460: The attempt to match a term of type simple1_adt:bit1() against the pattern <<_/binary>> breaks the opacity of the term simple1_api.erl:478: The call 'foo':A(A::simple1_adt:a()) breaks the opacity of the term A :: simple1_adt:a() simple1_api.erl:486: The call A:'foo'(A::simple1_adt:a()) breaks the opacity of the term A :: simple1_adt:a() simple1_api.erl:499: The call 'foo':A(A::simple1_api:i()) requires that A is of type atom() not simple1_api:i() diff --git a/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl b/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl index ea92613781..3606b21932 100644 --- a/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl +++ b/lib/dialyzer/test/options1_SUITE_data/src/compiler/beam_validator.erl @@ -272,7 +272,7 @@ valfun_1(if_end, Vst) -> valfun_1({try_case_end,Src}, Vst) -> assert_term(Src, Vst), kill_state(Vst); -%% Instructions that can not cause exceptions +%% Instructions that cannot cause exceptions valfun_1({move,Src,Dst}, Vst) -> Type = get_term_type(Src, Vst), set_type_reg(Type, Dst, Vst); diff --git a/lib/dialyzer/test/r9c_SUITE_data/results/asn1 b/lib/dialyzer/test/r9c_SUITE_data/results/asn1 index 1cf03346ee..6e51b972af 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/results/asn1 +++ b/lib/dialyzer/test/r9c_SUITE_data/results/asn1 @@ -87,7 +87,7 @@ asn1rt_per_bin.erl:2127: Cons will produce an improper list since its 2nd argume asn1rt_per_bin.erl:2129: Cons will produce an improper list since its 2nd argument is integer() asn1rt_per_bin.erl:446: The variable _ can never match since previous clauses completely covered the type integer() asn1rt_per_bin.erl:467: The variable _ can never match since previous clauses completely covered the type integer() -asn1rt_per_bin.erl:474: The pattern <{_N, <<_:8/integer-unit:1,Bs/binary-unit:8>>}, C> can never match since previous clauses completely covered the type <{0,_},integer()> +asn1rt_per_bin.erl:474: The pattern <{_N, <<_,Bs/binary>>}, C> can never match since previous clauses completely covered the type <{0,_},integer()> asn1rt_per_bin.erl:487: The variable _ can never match since previous clauses completely covered the type integer() asn1rt_per_bin.erl:498: The variable _ can never match since previous clauses completely covered the type integer() asn1rt_per_bin_rt2ct.erl:152: The call asn1rt_per_bin_rt2ct:getbit({0,maybe_improper_list()}) will never return since it differs in the 1st argument from the success typing arguments: (<<_:8,_:_*8>> | {non_neg_integer(),<<_:1,_:_*1>>}) @@ -95,7 +95,7 @@ asn1rt_per_bin_rt2ct.erl:1533: The pattern {'BMPString', {'octets', Ol}} can nev asn1rt_per_bin_rt2ct.erl:1875: The pattern {Name, Val} can never match since previous clauses completely covered the type any() asn1rt_per_bin_rt2ct.erl:443: The variable _ can never match since previous clauses completely covered the type integer() asn1rt_per_bin_rt2ct.erl:464: The variable _ can never match since previous clauses completely covered the type integer() -asn1rt_per_bin_rt2ct.erl:471: The pattern <{_N, <<_B:8/integer-unit:1,Bs/binary-unit:8>>}, C> can never match since previous clauses completely covered the type <{0,_},integer()> +asn1rt_per_bin_rt2ct.erl:471: The pattern <{_N, <<_B,Bs/binary>>}, C> can never match since previous clauses completely covered the type <{0,_},integer()> asn1rt_per_bin_rt2ct.erl:484: The variable _ can never match since previous clauses completely covered the type integer() asn1rt_per_bin_rt2ct.erl:495: The variable _ can never match since previous clauses completely covered the type integer() asn1rt_per_v1.erl:1209: The pattern <_, 'true', _> can never match the type <_,'false',_> diff --git a/lib/dialyzer/test/r9c_SUITE_data/src/asn1/asn1ct_gen_ber.erl b/lib/dialyzer/test/r9c_SUITE_data/src/asn1/asn1ct_gen_ber.erl index 1d065b3723..88b464a721 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/src/asn1/asn1ct_gen_ber.erl +++ b/lib/dialyzer/test/r9c_SUITE_data/src/asn1/asn1ct_gen_ber.erl @@ -300,7 +300,7 @@ gen_encode_prim(_Erules,D,DoTag,Value) when record(D,type) -> 'ASN1_OPEN_TYPE' -> emit_encode_func('open_type', Value,DoTag); XX -> - exit({'can not encode' ,XX}) + exit({'cannot encode' ,XX}) end. @@ -562,7 +562,7 @@ gen_dec_prim(Erules,Att,BytesVar,DoTag,TagIn,Length,_Form,OptOrMand) -> BytesVar,","]), false; Other -> - exit({'can not decode' ,Other}) + exit({'cannot decode' ,Other}) end, NewLength = case DoLength of diff --git a/lib/dialyzer/test/r9c_SUITE_data/src/asn1/asn1ct_gen_ber_bin_v2.erl b/lib/dialyzer/test/r9c_SUITE_data/src/asn1/asn1ct_gen_ber_bin_v2.erl index 9164ec6551..573bb15c92 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/src/asn1/asn1ct_gen_ber_bin_v2.erl +++ b/lib/dialyzer/test/r9c_SUITE_data/src/asn1/asn1ct_gen_ber_bin_v2.erl @@ -290,7 +290,7 @@ gen_encode_prim(_Erules,D,DoTag,Value) when record(D,type) -> 'ASN1_OPEN_TYPE' -> emit_encode_func('open_type', Value,DoTag); XX -> - exit({'can not encode' ,XX}) + exit({'cannot encode' ,XX}) end. @@ -602,7 +602,7 @@ gen_dec_prim(_Erules,Att,BytesVar,DoTag,_TagIn,_Form,_OptOrMand) -> BytesVar,","]), add_func({decode_open_type_as_binary,2}); Other -> - exit({'can not decode' ,Other}) + exit({'cannot decode' ,Other}) end, case {DoTag,NewTypeName} of diff --git a/lib/dialyzer/test/r9c_SUITE_data/src/inets/mod_responsecontrol.erl b/lib/dialyzer/test/r9c_SUITE_data/src/inets/mod_responsecontrol.erl index a997db6880..53eeedc29f 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/src/inets/mod_responsecontrol.erl +++ b/lib/dialyzer/test/r9c_SUITE_data/src/inets/mod_responsecontrol.erl @@ -71,7 +71,7 @@ do_responsecontrol(Info) -> %% If a client sends more then one of the if-XXXX fields in a request -%% The standard says it does not specify the behaviuor so I specified it :-) +%% The standard says it does not specify the behaviour so I specified it :-) %% The priority between the fields is %% 1.If-modified %% 2.If-Unmodified diff --git a/lib/dialyzer/test/r9c_SUITE_data/src/mnesia/mnesia_lib.erl b/lib/dialyzer/test/r9c_SUITE_data/src/mnesia/mnesia_lib.erl index 4008f8d789..9e0cdac2dc 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/src/mnesia/mnesia_lib.erl +++ b/lib/dialyzer/test/r9c_SUITE_data/src/mnesia/mnesia_lib.erl @@ -779,7 +779,7 @@ error_desc(no_transaction) -> "Operation not allowed outside transactions"; error_desc(combine_error) -> "Table options were ilegally combined"; error_desc(bad_index) -> "Index already exists or was out of bounds"; error_desc(already_exists) -> "Some schema option we try to set is already on"; -error_desc(index_exists)-> "Some ops can not be performed on tabs with index"; +error_desc(index_exists)-> "Some ops cannot be performed on tabs with index"; error_desc(no_exists)-> "Tried to perform op on non-existing (non alive) item"; error_desc(system_limit) -> "Some system_limit was exhausted"; error_desc(mnesia_down) -> "A transaction involving objects at some remote " diff --git a/lib/dialyzer/test/r9c_SUITE_data/src/mnesia/mnesia_tm.erl b/lib/dialyzer/test/r9c_SUITE_data/src/mnesia/mnesia_tm.erl index af49ceff72..b4e9edb7ce 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/src/mnesia/mnesia_tm.erl +++ b/lib/dialyzer/test/r9c_SUITE_data/src/mnesia/mnesia_tm.erl @@ -1529,7 +1529,7 @@ commit_participant(Coord, Tid, Bin, C0, DiscNs, _RamNs) -> ?eval_debug_fun({?MODULE, commit_participant, pre}, [{tid, Tid}]), case catch mnesia_schema:prepare_commit(Tid, C0, {part, Coord}) of {Modified, C, DumperMode} when record(C, commit) -> - %% If we can not find any local unclear decision + %% If we cannot find any local unclear decision %% we should presume abort at startup recovery case lists:member(node(), DiscNs) of false -> diff --git a/lib/dialyzer/test/small_SUITE_data/results/bs_fail_constr b/lib/dialyzer/test/small_SUITE_data/results/bs_fail_constr index dbc8241971..797f83956d 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/bs_fail_constr +++ b/lib/dialyzer/test/small_SUITE_data/results/bs_fail_constr @@ -1,9 +1,9 @@ bs_fail_constr.erl:11: Function w3/1 has no local return -bs_fail_constr.erl:12: Binary construction will fail since the size field S in segment 42:S/integer-unit:1 has type neg_integer() +bs_fail_constr.erl:12: Binary construction will fail since the size field S in segment 42:S has type neg_integer() bs_fail_constr.erl:14: Function w4/1 has no local return bs_fail_constr.erl:15: Binary construction will fail since the value field V in segment V/utf32 has type float() bs_fail_constr.erl:5: Function w1/1 has no local return -bs_fail_constr.erl:6: Binary construction will fail since the value field V in segment V:8/integer-unit:1 has type float() +bs_fail_constr.erl:6: Binary construction will fail since the value field V in segment V has type float() bs_fail_constr.erl:8: Function w2/1 has no local return -bs_fail_constr.erl:9: Binary construction will fail since the value field V in segment V/binary-unit:8 has type atom() +bs_fail_constr.erl:9: Binary construction will fail since the value field V in segment V/binary has type atom() diff --git a/lib/dialyzer/test/small_SUITE_data/results/fun_app_args b/lib/dialyzer/test/small_SUITE_data/results/fun_app_args new file mode 100644 index 0000000000..ac153a6fb2 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/fun_app_args @@ -0,0 +1,3 @@ + +fun_app_args.erl:12: Fun application with arguments ('b',[]) will fail since the function has type 'c' | fun(('a',[]) -> any()), which differs in the 1st argument +fun_app_args.erl:12: The created fun has no local return diff --git a/lib/dialyzer/test/small_SUITE_data/results/maps_sum b/lib/dialyzer/test/small_SUITE_data/results/maps_sum index b29ac77d88..daa099e490 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/maps_sum +++ b/lib/dialyzer/test/small_SUITE_data/results/maps_sum @@ -1,4 +1,4 @@ -maps_sum.erl:15: Invalid type specification for function maps_sum:wrong1/1. The success typing is (maps:iterator() | map()) -> any() +maps_sum.erl:15: Invalid type specification for function maps_sum:wrong1/1. The success typing is (maps:iterator(_,_) | map()) -> any() maps_sum.erl:26: Function wrong2/1 has no local return maps_sum.erl:27: The call lists:foldl(fun((_,_,_) -> any()),0,Data::any()) will never return since it differs in the 1st argument from the success typing arguments: (fun((_,_) -> any()),any(),[any()]) diff --git a/lib/dialyzer/test/small_SUITE_data/results/pretty_bitstring b/lib/dialyzer/test/small_SUITE_data/results/pretty_bitstring index e148e5cf22..dc3620fcf0 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/pretty_bitstring +++ b/lib/dialyzer/test/small_SUITE_data/results/pretty_bitstring @@ -1,3 +1,3 @@ pretty_bitstring.erl:7: Function t/0 has no local return -pretty_bitstring.erl:8: The call binary:copy(#{#<1>(8, 1, 'integer', ['unsigned', 'big']), #<2>(8, 1, 'integer', ['unsigned', 'big']), #<3>(3, 1, 'integer', ['unsigned', 'big'])}#,2) breaks the contract (Subject,N) -> binary() when Subject :: binary(), N :: non_neg_integer() +pretty_bitstring.erl:8: The call binary:copy(<<1,2,3:3>>,2) breaks the contract (Subject,N) -> binary() when Subject :: binary(), N :: non_neg_integer() diff --git a/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash b/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash index 8c9df56a4b..7fd1f304cb 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash +++ b/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash @@ -3,12 +3,12 @@ tuple_set_crash.erl:103: Invalid type specification for function tuple_set_crash tuple_set_crash.erl:123: Invalid type specification for function tuple_set_crash:parse_video_target_info/1. The success typing is (<<_:48>>) -> [{'status',byte()} | {'target_id',non_neg_integer()},...] tuple_set_crash.erl:127: Invalid type specification for function tuple_set_crash:parse_audio_target_info/1. The success typing is (<<_:48>>) -> [{'master_volume',char()} | {'status',byte()} | {'target_id',non_neg_integer()},...] tuple_set_crash.erl:138: Invalid type specification for function tuple_set_crash:parse_av_device_info/1. The success typing is (<<_:48>>) -> [{'address',byte()} | {'device_id',non_neg_integer()} | {'model',binary()} | {'status',byte()},...] -tuple_set_crash.erl:143: The pattern <<TargetId:32/integer-little-unit:1,Rest1/binary-unit:8>> can never match the type <<_:8>> +tuple_set_crash.erl:143: The pattern <<TargetId:32/integer-little-unit:1,Rest1/binary>> can never match the type <<_:8>> tuple_set_crash.erl:155: Invalid type specification for function tuple_set_crash:parse_video_output_info/1. The success typing is (<<_:48>>) -> [{'audio_volume',char()} | {'display_type',binary()} | {'output_id',non_neg_integer()},...] -tuple_set_crash.erl:160: The pattern <<DeviceId:32/integer-little-unit:1,Rest1/binary-unit:8>> can never match the type <<_:8>> +tuple_set_crash.erl:160: The pattern <<DeviceId:32/integer-little-unit:1,Rest1/binary>> can never match the type <<_:8>> tuple_set_crash.erl:171: Invalid type specification for function tuple_set_crash:parse_audio_output_info/1. The success typing is (<<_:48>>) -> [{'output_id',non_neg_integer()},...] -tuple_set_crash.erl:176: The pattern <<DeviceId:32/integer-little-unit:1,Rest1/binary-unit:8>> can never match the type <<_:8>> -tuple_set_crash.erl:179: The pattern <<AudioVolume:16/integer-little-unit:1,Rest2/binary-unit:8>> can never match the type <<_:8>> -tuple_set_crash.erl:182: The pattern <<Delay:16/integer-little-unit:1,_Padding/binary-unit:8>> can never match the type <<_:8>> +tuple_set_crash.erl:176: The pattern <<DeviceId:32/integer-little-unit:1,Rest1/binary>> can never match the type <<_:8>> +tuple_set_crash.erl:179: The pattern <<AudioVolume:16/integer-little-unit:1,Rest2/binary>> can never match the type <<_:8>> +tuple_set_crash.erl:182: The pattern <<Delay:16/integer-little-unit:1,_Padding/binary>> can never match the type <<_:8>> tuple_set_crash.erl:62: The pattern {'play_list', _Playlist} can never match the type 'ok' | {'device_properties',[{atom(),_}]} | {'error',[{atom(),_}]} tuple_set_crash.erl:64: The pattern {'error', 17} can never match the type 'ok' | {'device_properties',[{atom(),_}]} | {'error',[{atom(),_}]} diff --git a/lib/dialyzer/test/small_SUITE_data/src/fun_app_args.erl b/lib/dialyzer/test/small_SUITE_data/src/fun_app_args.erl new file mode 100644 index 0000000000..b4409bc550 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/fun_app_args.erl @@ -0,0 +1,12 @@ +-module(fun_app_args). + +-export([t/1]). + +-type ft() :: fun((a, []) -> any()). + +-record(r, { + h = c :: c | ft() +}). + +t(#r{h = H}) -> + fun(_) -> (H)(b, []) end. diff --git a/lib/edoc/doc/edoc.dtd b/lib/edoc/doc/edoc.dtd new file mode 120000 index 0000000000..43f4b27db6 --- /dev/null +++ b/lib/edoc/doc/edoc.dtd @@ -0,0 +1 @@ +../priv/edoc.dtd
\ No newline at end of file diff --git a/lib/edoc/doc/edoc_doclet.hrl b/lib/edoc/doc/edoc_doclet.hrl new file mode 120000 index 0000000000..4623b18bb4 --- /dev/null +++ b/lib/edoc/doc/edoc_doclet.hrl @@ -0,0 +1 @@ +../include/edoc_doclet.hrl
\ No newline at end of file diff --git a/lib/edoc/doc/src/Makefile b/lib/edoc/doc/src/Makefile index aba94a6802..3e53e75c75 100644 --- a/lib/edoc/doc/src/Makefile +++ b/lib/edoc/doc/src/Makefile @@ -79,6 +79,11 @@ HTML_REF_MAN_FILE = $(HTMLDIR)/index.html TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf +INCLUDES_DIR = ../../include +INCLUDES = $(INCLUDES_DIR)/edoc_doclet.hrl + +DTDS_DIR = ../../priv +DTDS = $(DTDS_DIR)/edoc.dtd # ---------------------------------------------------- # FLAGS @@ -135,5 +140,6 @@ release_docs_spec: docs $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" $(INSTALL_DIR) "$(RELEASE_PATH)/man/man3" $(INSTALL_DATA) $(MAN3DIR)/* "$(RELEASE_PATH)/man/man3" + $(INSTALL_DATA) $(INCLUDES) $(DTDS) "$(RELSYSDIR)/doc/html" release_spec: diff --git a/lib/edoc/include/edoc_doclet.hrl b/lib/edoc/include/edoc_doclet.hrl index 1429ee5971..a05a9cb2bc 100644 --- a/lib/edoc/include/edoc_doclet.hrl +++ b/lib/edoc/include/edoc_doclet.hrl @@ -43,7 +43,7 @@ %% @type no_app(). %% A value used to mark absence of an Erlang application %% context. Use the macro `NO_APP' defined in -%% <a href="../include/edoc_doclet.hrl">`edoc_doclet.hrl'</a> +%% <a href="edoc_doclet.hrl">`edoc_doclet.hrl'</a> %% to produce this value. %% @type doclet_gen() = #doclet_gen{sources = [string()], diff --git a/lib/edoc/src/edoc.erl b/lib/edoc/src/edoc.erl index e9d62d3283..62483602aa 100644 --- a/lib/edoc/src/edoc.erl +++ b/lib/edoc/src/edoc.erl @@ -734,7 +734,7 @@ get_doc(File) -> %% %% @type edoc_module(). The EDoc documentation data for a module, %% expressed as an XML document in {@link //xmerl. XMerL} format. See -%% the file <a href="../priv/edoc.dtd">`edoc.dtd'</a> for details. +%% the file <a href="edoc.dtd">`edoc.dtd'</a> for details. %% %% @doc Reads a source code file and extracts EDoc documentation data. %% Note that without an environment parameter (see {@link get_doc/3}), diff --git a/lib/edoc/src/edoc_data.erl b/lib/edoc/src/edoc_data.erl index 7c077d3acd..a8373d6536 100644 --- a/lib/edoc/src/edoc_data.erl +++ b/lib/edoc/src/edoc_data.erl @@ -345,6 +345,8 @@ deprecated(Repl, Env) -> deprecated(Desc) -> [{deprecated, description(Desc)}]. +-dialyzer({no_match, replacement_function/2}). + replacement_function(M0, {M,F,A}) when is_list(A) -> %% refer to the largest listed arity - the most general version replacement_function(M0, {M,F,lists:last(lists:sort(A))}); diff --git a/lib/edoc/src/edoc_doclet.erl b/lib/edoc/src/edoc_doclet.erl index 6cb3095507..604291374a 100644 --- a/lib/edoc/src/edoc_doclet.erl +++ b/lib/edoc/src/edoc_doclet.erl @@ -62,7 +62,7 @@ %% @spec (Command::doclet_gen() | doclet_toc(), edoc_context()) -> ok %% @doc Main doclet entry point. See the file <a -%% href="../include/edoc_doclet.hrl">`edoc_doclet.hrl'</a> for the data +%% href="edoc_doclet.hrl">`edoc_doclet.hrl'</a> for the data %% structures used for passing parameters. %% %% Also see {@link edoc:layout/2} for layout-related options, and diff --git a/lib/eldap/README b/lib/eldap/README index e1bde9d658..238f140e93 100644 --- a/lib/eldap/README +++ b/lib/eldap/README @@ -23,7 +23,7 @@ system has been configured with SSL. In the test directory there are some hints and examples on how to test the code and how to setup and populate an OpenLDAP server. The 'eldap' code has been tested -agains OpenLDAP, IPlanet and ActiveDirectory servers. +against OpenLDAP, IPlanet and ActiveDirectory servers. If you plan to incorporate this code into your system I suggest that you build a server/supervisor harnesk diff --git a/lib/eldap/test/make_certs.erl b/lib/eldap/test/make_certs.erl index cfa43289e1..e8a13ae113 100644 --- a/lib/eldap/test/make_certs.erl +++ b/lib/eldap/test/make_certs.erl @@ -348,7 +348,7 @@ req_cnf(C) -> "default_bits = ", integer_to_list(C#config.default_bits), "\n" "RANDFILE = $ROOTDIR/RAND\n" "encrypt_key = no\n" - "default_md = md5\n" + "default_md = sha1\n" "#string_mask = pkix\n" "x509_extensions = ca_ext\n" "prompt = no\n" @@ -394,7 +394,7 @@ ca_cnf(C) -> ["crl_extensions = crl_ext\n" || C#config.v2_crls], "unique_subject = no\n" "default_days = 3600\n" - "default_md = md5\n" + "default_md = sha1\n" "preserve = no\n" "policy = policy_match\n" "\n" diff --git a/lib/erl_docgen/doc/src/doc-build.xml b/lib/erl_docgen/doc/src/doc-build.xml index 3ea8798639..17e13bff81 100644 --- a/lib/erl_docgen/doc/src/doc-build.xml +++ b/lib/erl_docgen/doc/src/doc-build.xml @@ -178,7 +178,7 @@ </section> <section> - <title>Upcomming changes</title> + <title>Upcoming changes</title> <p> The output from the <c>erl_docgen</c> documentation build process is now just the OTP style. But in a near future we will for example add the possibility to change logo, color in the PDF and diff --git a/lib/erl_docgen/priv/dtd/book.dtd b/lib/erl_docgen/priv/dtd/book.dtd index aa07d38658..326bf3369a 100644 --- a/lib/erl_docgen/priv/dtd/book.dtd +++ b/lib/erl_docgen/priv/dtd/book.dtd @@ -30,7 +30,7 @@ insidecover?, pagetext, preamble, - (applications|parts|headline|pagetext)+, + (applications|parts|internals|headline|pagetext)+, (listoffigures?, listoftables?, listofterms?, @@ -56,6 +56,7 @@ <!ELEMENT applications (include)* > <!ELEMENT parts (title?,description?,(include|onepart)*) > <!ATTLIST parts lift (yes|no) "no" > +<!ELEMENT internals (include)* > <!ELEMENT headline (#PCDATA) > <!ELEMENT index EMPTY > <!ELEMENT listoffigures EMPTY > diff --git a/lib/erl_docgen/priv/dtd/common.dtd b/lib/erl_docgen/priv/dtd/common.dtd index b1578ad9d4..0ccd52068b 100644 --- a/lib/erl_docgen/priv/dtd/common.dtd +++ b/lib/erl_docgen/priv/dtd/common.dtd @@ -25,7 +25,7 @@ <!ENTITY % block "p|pre|code|list|taglist|codeinclude| erleval" > <!ENTITY % inline "#PCDATA|c|i|em|strong|term|cite|br|path|seealso| - url|marker|anno" > + url|marker|anno|image" > <!-- XXX --> <!ELEMENT p (%inline;)* > <!ELEMENT pre (#PCDATA|seealso|url|input|anno)* > diff --git a/lib/erl_docgen/priv/xsl/db_html.xsl b/lib/erl_docgen/priv/xsl/db_html.xsl index c9be926e1e..18bc8cd1cf 100644 --- a/lib/erl_docgen/priv/xsl/db_html.xsl +++ b/lib/erl_docgen/priv/xsl/db_html.xsl @@ -836,6 +836,10 @@ <!-- .../part --> <xsl:call-template name="part.content" /> </xsl:if> + <xsl:if test="$lname = 'internal'"> + <!-- .../internals --> + <xsl:call-template name="internal.content" /> + </xsl:if> <xsl:if test="$lname = 'chapter'"> <!-- .../part/chapter --> <xsl:call-template name="chapter.content"> @@ -859,12 +863,24 @@ <xsl:param name="chapnum"/> <xsl:param name="curModule"/> <xsl:if test="(local-name() = 'part') or ((local-name() = 'chapter') and ancestor::part)"> - <!-- .../part or.../part/chapter --> + <!-- .../part or .../part/chapter --> <xsl:call-template name="menu.ug"> <xsl:with-param name="chapnum" select="$chapnum"/> </xsl:call-template> </xsl:if> - <xsl:if test="(local-name() = 'application') or (local-name() = 'erlref')or (local-name() = 'comref')or (local-name() = 'cref')or (local-name() = 'fileref')or (local-name() = 'appref')"> + <xsl:if test="(local-name() = 'internal' and descendant::chapter) or ((local-name() = 'chapter') and ancestor::internal)"> + <!-- .../internal or .../internal/chapter --> + <xsl:call-template name="menu.internal.ug"> + <xsl:with-param name="chapnum" select="$chapnum"/> + </xsl:call-template> + </xsl:if> + <xsl:if test="(local-name() = 'internal' and descendant::erlref) or (((local-name() = 'erlref') or (local-name() = 'comref') or (local-name() = 'cref') or (local-name() = 'fileref') or (local-name() = 'appref')) and ancestor::internal)"> + <!-- .../internal,.../internal/erlref, .../internal/comref or .../internal/cref or .../internal/fileref or .../internal/appref --> + <xsl:call-template name="menu.internal.ref"> + <xsl:with-param name="curModule" select="$curModule"/> + </xsl:call-template> + </xsl:if> + <xsl:if test="(local-name() = 'application') or (((local-name() = 'erlref') or (local-name() = 'comref') or (local-name() = 'cref') or (local-name() = 'fileref') or (local-name() = 'appref')) and ancestor::application)"> <!-- .../application,.../application/erlref, .../application/comref or .../application/cref or .../application/fileref or .../application/appref --> <xsl:call-template name="menu.ref"> <xsl:with-param name="curModule" select="$curModule"/> @@ -902,6 +918,9 @@ <xsl:if test="boolean(/book/applications)"> <li><a href="index.html">Reference Manual</a></li> </xsl:if> + <xsl:if test="boolean(/book/internals)"> + <li><a href="internal_docs.html">Internal Documentation</a></li> + </xsl:if> <xsl:if test="boolean(/book/releasenotes)"> <li><a href="release_notes.html">Release Notes</a></li> </xsl:if> @@ -942,6 +961,7 @@ <xsl:template match="/book"> <xsl:apply-templates select="parts"/> <xsl:apply-templates select="applications"/> + <xsl:apply-templates select="internals"/> <xsl:apply-templates select="releasenotes"/> </xsl:template> @@ -955,6 +975,11 @@ <xsl:apply-templates select="application"/> </xsl:template> + <!-- Internals --> + <xsl:template match="internals"> + <xsl:apply-templates select="internal"/> + </xsl:template> + <!-- Header --> <xsl:template match="header"/> @@ -1311,6 +1336,90 @@ </xsl:template> + <!-- Internal Docs --> + + <!-- Part --> + <xsl:template match="internal"> + + <xsl:document href="{$outdir}/internal_docs.html" method="html" encoding="UTF-8" indent="yes" doctype-public="-//W3C//DTD HTML 4.01 Transitional//EN"> + <xsl:call-template name="pagelayout"/> + </xsl:document> + </xsl:template> + + + <!-- Part content--> + <xsl:template name="internal.content"> + <div class="frontpage"/> + + <center><h1><xsl:value-of select="/book/header/title"/> Internal Docs</h1></center> + + <center><h4>Version <xsl:value-of select="$appver"/></h4></center> + <center><h4><xsl:value-of select="$gendate"/></h4></center> + <div class="extrafrontpageinfo"> + <center><xsl:value-of select="$extra_front_page_info"/></center> + </div> + + <xsl:apply-templates select="chapter|erlref"/> + + </xsl:template> + + <!-- Menu.internal.chapter --> + <xsl:template name="menu.internal.ug"> + <xsl:param name="chapnum"/> + + <div id="leftnav"> + <div class="innertube"> + + <xsl:call-template name="erlang_logo"/> + + <p class="section-title"><xsl:value-of select="/book/header/title"/></p> + <p class="section-subtitle">Internal Documentation</p> + <p class="section-version">Version <xsl:value-of select="$appver"/></p> + + <xsl:call-template name="menu_top"/> + + <xsl:call-template name="menu_middle"/> + + <h3>Chapters</h3> + + <ul class="flipMenu" imagepath="{$topdocdir}/js/flipmenu"> + <xsl:call-template name="menu.chapter"> + <xsl:with-param name="entries" select="/book/internals/internal/chapter[header/title]"/> + <xsl:with-param name="chapnum" select="$chapnum"/> + </xsl:call-template> + </ul> + </div> + </div> + </xsl:template> + + <!-- Menu.internal.ref --> + <xsl:template name="menu.internal.ref"> + <xsl:param name="curModule"/> + <div id="leftnav"> + <div class="innertube"> + + <xsl:call-template name="erlang_logo"/> + + <p class="section-title"><xsl:value-of select="/book/header/title"/></p> + <p class="section-subtitle">Reference Manual</p> + <p class="section-version">Version <xsl:value-of select="$appver"/></p> + + <xsl:call-template name="menu_top"/> + + <xsl:call-template name="menu_middle"/> + + <h3>Table of Contents</h3> + + <ul class="flipMenu"> + <xsl:call-template name="menu.ref2"> + <xsl:with-param name="entries" select="/book/internals/internal/erlref[module]|/book/internals/internal/cref[lib]|/book/internals/internal/comref[com]|/book/internals/internal/fileref[file]|/book/internals/internal/appref[app]"/> + <!--xsl:with-param name="genFuncMenu" select="true"/--> + <xsl:with-param name="curModule" select="$curModule"/> + </xsl:call-template> + </ul> + </div> + </div> + </xsl:template> <!--Users Guide --> diff --git a/lib/erl_docgen/src/docgen_edoc_xml_cb.erl b/lib/erl_docgen/src/docgen_edoc_xml_cb.erl index b065c18cda..5342d02947 100644 --- a/lib/erl_docgen/src/docgen_edoc_xml_cb.erl +++ b/lib/erl_docgen/src/docgen_edoc_xml_cb.erl @@ -182,7 +182,7 @@ chapter_title(#xmlElement{content=Es}) -> % name = h3 | h4 %% otp_xmlify(Es1) -> Es2 %% Es1 = Es2 = [#xmlElement{} | #xmlText{}] %% Fix things that are allowed in XHTML but not in chapter/erlref DTDs. -%% 1) lists (<ul>, <ol>, <dl>) and code snippets (<pre>) can not occur +%% 1) lists (<ul>, <ol>, <dl>) and code snippets (<pre>) cannot occur %% within a <p>, such a <p> must be splitted into a sequence of <p>, %% <ul>, <ol>, <dl> and <pre>. %% 2) <a> must only have either a href attribute (corresponds to a @@ -1260,11 +1260,14 @@ get_text(#xmlElement{content=[E]}) -> %% text_and_name_only(Es) -> {N, Ts} text_and_a_name_only(Es) -> - [Name|_] = [Name || - #xmlElement{ - name = a, - attributes = [#xmlAttribute{name=name}]}=Name <- Es], - {Name#xmlElement{content = []}, text_only(Es)}. + case [Name || #xmlElement{ + name = a, + attributes = [#xmlAttribute{name=name}]}=Name <- Es] of + [Name|_] -> + {Name#xmlElement{content = []}, text_only(Es)}; + [] -> + {"", text_only(Es)} + end. %% text_only(Es) -> Ts %% Takes a list of xmlElement and xmlText and return a lists of xmlText. diff --git a/lib/erl_docgen/src/docgen_xmerl_xml_cb.erl b/lib/erl_docgen/src/docgen_xmerl_xml_cb.erl index 59d4dccfb7..9d69143c3c 100644 --- a/lib/erl_docgen/src/docgen_xmerl_xml_cb.erl +++ b/lib/erl_docgen/src/docgen_xmerl_xml_cb.erl @@ -87,6 +87,7 @@ convert_tag(underline, Attrs) -> {em, Attrs}; convert_tag(Tag, Attrs) -> {Tag, Attrs}. is_url("http:"++_) -> true; +is_url("https:"++_) -> true; is_url("../"++_) -> true; is_url(FileRef) -> case filename:extension(FileRef) of diff --git a/lib/erl_interface/configure.in b/lib/erl_interface/configure.in index 747750c1fb..f0e9b2eb3f 100644 --- a/lib/erl_interface/configure.in +++ b/lib/erl_interface/configure.in @@ -29,11 +29,6 @@ dnl m4_define(EI_VERSION,regexp(m4_include(VERSION),[version \([-.0-9A-Za-z]+\)] AC_INIT() -if test "x$no_recursion" != "xyes" -a "x$OVERRIDE_CONFIG_CACHE" = "x"; then - # We do not want to use a common cache! - cache_file=/dev/null -fi - dnl How to set srcdir absolute is taken from the GNU Emacs distribution #### Make srcdir absolute, if it isn't already. It's important to #### avoid running the path through pwd unnecessary, since pwd can @@ -365,6 +360,7 @@ LDFLAGS="$LDFLAGS $sanitizers" # XXX # --------------------------------------------------------------------------- + AC_OUTPUT( src/$host/Makefile:src/Makefile.in src/$host/eidefs.mk:src/eidefs.mk.in diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml index 179dbbd966..7808bfd94f 100644 --- a/lib/erl_interface/doc/src/ei.xml +++ b/lib/erl_interface/doc/src/ei.xml @@ -35,6 +35,9 @@ <lib>ei</lib> <libsummary>Routines for handling the Erlang binary term format.</libsummary> <description> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <p>The library <c>ei</c> contains macros and functions to encode and decode the Erlang binary term format.</p> @@ -180,6 +183,43 @@ typedef enum { </func> <func> + <name since="OTP @OTP-15712@"><ret>int</ret><nametext>ei_decode_bitstring(const char *buf, int *index, const char **pp, unsigned int *bitoffsp, size_t *nbitsp)</nametext></name> + <fsummary>Decode a bitstring.</fsummary> + <desc> + <p>Decodes a bit string from the binary format.</p> + <taglist> + <tag><c>pp</c></tag> + <item><p>Either <c>NULL</c> or <c>*pp</c> returns a pointer to + the first byte of the bit string. The returned bit string is + readable as long as the buffer pointed to by <c>buf</c> is + readable and not written to.</p> + </item> + <tag><c>bitoffsp</c></tag> + <item><p>Either <c>NULL</c> or <c>*bitoffsp</c> returns the + number of unused bits in the first byte pointed to by + <c>*pp</c>. The value of <c>*bitoffsp</c> is between 0 and 7. + Unused bits in the first byte are the most significant bits.</p> + </item> + <tag><c>nbitsp</c></tag> + <item><p>Either <c>NULL</c> or <c>*nbitsp</c> returns the length + of the bit string in <em>bits</em>.</p> + </item> + </taglist> + <p>Returns <c>0</c> if it was a bit string term.</p> + <p>The number of <em>bytes</em> pointed to by <c>*pp</c>, which are + part of the bit string, is <c>(*bitoffsp + *nbitsp + 7)/8</c>. If + <c>(*bitoffsp + *bitsp)%8 > 0</c> then only <c>(*bitoffsp + + *bitsp)%8</c> bits of the last byte are used. Unused bits in + the last byte are the least significant bits.</p> + <p>The values of unused bits in the first and last byte are undefined + and cannot be relied on.</p> + <p>Number of bits may be divisible by 8, which means a binary + decodable by <c>ei_decode_binary</c> is also decodable by + <c>ei_decode_bitstring</c>.</p> + </desc> + </func> + + <func> <name since=""><ret>int</ret><nametext>ei_decode_boolean(const char *buf, int *index, int *p)</nametext></name> <fsummary>Decode a boolean.</fsummary> <desc> @@ -346,8 +386,10 @@ typedef enum { <c>t</c> is actually an <c>ETERM**</c> (see <seealso marker="erl_eterm"><c>erl_eterm</c></seealso>). The term is later to be deallocated.</p> - <p>Notice that this function is located in the <c>Erl_Interface</c> - library.</p> + <note><p>This function is deprecated as of OTP 22 and will be removed in + OTP 23 together with the old legacy <c>erl_interface</c> library (functions + with prefix <c>erl_</c>).</p> + </note> </desc> </func> @@ -456,6 +498,28 @@ typedef enum { </func> <func> + <name since="OTP @OTP-15712@"><ret>int</ret> + <nametext>ei_encode_bitstring(char *buf, int *index, const char *p, size_t bitoffs, size_t nbits)</nametext></name> + <name since="OTP @OTP-15712@"><ret>int</ret> + <nametext>ei_x_encode_bitstring(ei_x_buff* x, const char *p, size_t bitoffs, size_t nbits)</nametext></name> + <fsummary>Encode a bitstring.</fsummary> + <desc> + <p>Encodes a bit string in the binary format.</p> + <p>The data is at <c>p</c>. The length of the bit string is <c>nbits</c> + bits. The first <c>bitoffs</c> bits of the data at <c>p</c> are unused. + The first byte which is part of the bit string is + <c>p[bitoffs/8]</c>. The <c>bitoffs%8</c> most significant bits of + the first byte <c>p[bitoffs/8]</c> are unused.</p> + <p>The number of bytes which is part of the bit string is <c>(bitoffs + + nbits + 7)/8</c>. If <c>(bitoffs + nbits)%8 > 0</c> then only <c>(bitoffs + + nbits)%8</c> bits of the last byte are used. Unused bits in + the last byte are the least significant bits.</p> + <p>The values of unused bits are disregarded and does not need to be + cleared.</p> + </desc> + </func> + + <func> <name since=""><ret>int</ret><nametext>ei_encode_boolean(char *buf, int *index, int p)</nametext></name> <name since=""><ret>int</ret><nametext>ei_x_encode_boolean(ei_x_buff* x, int p)</nametext></name> <fsummary>Encode a boolean.</fsummary> @@ -653,6 +717,10 @@ ei_x_encode_string(&x, "Banana");</pre> <c>erl_interface</c>. Parameter <c>t</c> is actually an <c>ETERM</c> pointer. This function does not free the <c>ETERM</c>.</p> + <note><p>These functions are deprecated as of OTP 22 and will be removed in + OTP 23 together with the old legacy <c>erl_interface</c> library + (functions with prefix <c>erl_</c>).</p> + </note> </desc> </func> <func> @@ -722,12 +790,12 @@ ei_encode_tuple_header(buf, &i, 0);</pre> <name since=""><ret>int</ret><nametext>ei_get_type(const char *buf, const int *index, int *type, int *size)</nametext></name> <fsummary>Fetch the type and size of an encoded term.</fsummary> <desc> - <p>Returns the type in <c>type</c> and size in - <c>size</c> of the encoded term. For strings and atoms, + <p>Returns the type in <c>*type</c> and size in + <c>*size</c> of the encoded term. For strings and atoms, size is the number of characters <em>not</em> including the - terminating <c>NULL</c>. For binaries, <c>size</c> is the number of - bytes. For lists and tuples, <c>size</c> is the arity of - the object. For other types, <c>size</c> is 0. In all + terminating <c>NULL</c>. For binaries and bitstrings, <c>*size</c> is + the number of bytes. For lists, tuples and maps, <c>*size</c> is the + arity of the object. For other types, <c>*size</c> is 0. In all cases, <c>index</c> is left unchanged.</p> </desc> </func> diff --git a/lib/erl_interface/doc/src/ei_connect.xml b/lib/erl_interface/doc/src/ei_connect.xml index 2dd0a285fe..795f1249b3 100644 --- a/lib/erl_interface/doc/src/ei_connect.xml +++ b/lib/erl_interface/doc/src/ei_connect.xml @@ -34,6 +34,9 @@ <lib>ei_connect</lib> <libsummary>Communicate with distributed Erlang.</libsummary> <description> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <p>This module enables C-programs to communicate with Erlang nodes, using the Erlang distribution over TCP/IP.</p> diff --git a/lib/erl_interface/doc/src/ei_users_guide.xml b/lib/erl_interface/doc/src/ei_users_guide.xml index 2dfd99e35a..7ca10d1a99 100644 --- a/lib/erl_interface/doc/src/ei_users_guide.xml +++ b/lib/erl_interface/doc/src/ei_users_guide.xml @@ -34,6 +34,18 @@ </header> <section> + <title>Deprecation and Removal</title> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <note><p>The old legacy <c>erl_interface</c> library (functions + with prefix <c>erl_</c>) is deprecated as of OTP 22, and will be + removed in OTP 23. This does not apply to the <c>ei</c> + library. Reasonably new <c>gcc</c> compilers will issue deprecation + warnings. In order to disable these warnings, define the macro + <c>EI_NO_DEPR_WARN</c>.</p></note> + </section> + + <section> <title>Introduction</title> <p>The <c>Erl_Interface</c> library contains functions that help you integrate programs written in C and Erlang. The functions in diff --git a/lib/erl_interface/doc/src/erl_connect.xml b/lib/erl_interface/doc/src/erl_connect.xml index 139ac9e2f0..9492a82864 100644 --- a/lib/erl_interface/doc/src/erl_connect.xml +++ b/lib/erl_interface/doc/src/erl_connect.xml @@ -35,6 +35,15 @@ <lib>erl_connect</lib> <libsummary>Communicate with distributed Erlang.</libsummary> <description> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <note><p>The old legacy <c>erl_interface</c> library (functions + with prefix <c>erl_</c>) is deprecated as of OTP 22, and will be + removed in OTP 23. This does not apply to the <c>ei</c> + library. Reasonably new <c>gcc</c> compilers will issue deprecation + warnings. In order to disable these warnings, define the macro + <c>EI_NO_DEPR_WARN</c>.</p></note> + <p>This module provides support for communication between distributed Erlang nodes and C-nodes, in a manner that is transparent to Erlang processes.</p> diff --git a/lib/erl_interface/doc/src/erl_eterm.xml b/lib/erl_interface/doc/src/erl_eterm.xml index 070ed30dfe..295760b4e6 100644 --- a/lib/erl_interface/doc/src/erl_eterm.xml +++ b/lib/erl_interface/doc/src/erl_eterm.xml @@ -35,6 +35,15 @@ <lib>erl_eterm</lib> <libsummary>Functions for Erlang term construction.</libsummary> <description> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <note><p>The old legacy <c>erl_interface</c> library (functions + with prefix <c>erl_</c>) is deprecated as of OTP 22, and will be + removed in OTP 23. This does not apply to the <c>ei</c> + library. Reasonably new <c>gcc</c> compilers will issue deprecation + warnings. In order to disable these warnings, define the macro + <c>EI_NO_DEPR_WARN</c>.</p></note> + <p>This module provides functions for creating and manipulating Erlang terms.</p> diff --git a/lib/erl_interface/doc/src/erl_global.xml b/lib/erl_interface/doc/src/erl_global.xml index 72d43e81d5..39085b46f0 100644 --- a/lib/erl_interface/doc/src/erl_global.xml +++ b/lib/erl_interface/doc/src/erl_global.xml @@ -35,6 +35,15 @@ <lib>erl_global</lib> <libsummary>Access globally registered names.</libsummary> <description> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <note><p>The old legacy <c>erl_interface</c> library (functions + with prefix <c>erl_</c>) is deprecated as of OTP 22, and will be + removed in OTP 23. This does not apply to the <c>ei</c> + library. Reasonably new <c>gcc</c> compilers will issue deprecation + warnings. In order to disable these warnings, define the macro + <c>EI_NO_DEPR_WARN</c>.</p></note> + <p>This module provides support for registering, looking up, and unregistering names in the <c>global</c> module. For more information, see diff --git a/lib/erl_interface/doc/src/erl_interface.xml b/lib/erl_interface/doc/src/erl_interface.xml index 4e66655b39..decd66046a 100644 --- a/lib/erl_interface/doc/src/erl_interface.xml +++ b/lib/erl_interface/doc/src/erl_interface.xml @@ -58,6 +58,18 @@ </list> <section> + <title>Deprecation and Removal</title> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <note><p>The old legacy <c>erl_interface</c> library (functions + with prefix <c>erl_</c>) is deprecated as of OTP 22, and will be + removed in OTP 23. This does not apply to the <c>ei</c> + library. Reasonably new <c>gcc</c> compilers will issue deprecation + warnings. In order to disable these warnings, define the macro + <c>EI_NO_DEPR_WARN</c>.</p></note> + </section> + + <section> <title>Compiling and Linking Your Code</title> <p>In order to use any of the Erl_Interface functions, include the following lines in your code:</p> diff --git a/lib/erl_interface/doc/src/erl_malloc.xml b/lib/erl_interface/doc/src/erl_malloc.xml index aae3b7e078..6650620064 100644 --- a/lib/erl_interface/doc/src/erl_malloc.xml +++ b/lib/erl_interface/doc/src/erl_malloc.xml @@ -35,6 +35,14 @@ <lib>erl_malloc</lib> <libsummary>Memory allocation functions.</libsummary> <description> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <note><p>The old legacy <c>erl_interface</c> library (functions + with prefix <c>erl_</c>) is deprecated as of OTP 22, and will be + removed in OTP 23. This does not apply to the <c>ei</c> + library. Reasonably new <c>gcc</c> compilers will issue deprecation + warnings. In order to disable these warnings, define the macro + <c>EI_NO_DEPR_WARN</c>.</p></note> <p>This module provides functions for allocating and deallocating memory.</p> </description> diff --git a/lib/erl_interface/doc/src/erl_marshal.xml b/lib/erl_interface/doc/src/erl_marshal.xml index 1a6d3bb43c..33d359d871 100644 --- a/lib/erl_interface/doc/src/erl_marshal.xml +++ b/lib/erl_interface/doc/src/erl_marshal.xml @@ -35,6 +35,14 @@ <lib>erl_marshal</lib> <libsummary>Encoding and decoding of Erlang terms.</libsummary> <description> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <note><p>The old legacy <c>erl_interface</c> library (functions + with prefix <c>erl_</c>) is deprecated as of OTP 22, and will be + removed in OTP 23. This does not apply to the <c>ei</c> + library. Reasonably new <c>gcc</c> compilers will issue deprecation + warnings. In order to disable these warnings, define the macro + <c>EI_NO_DEPR_WARN</c>.</p></note> <p>This module contains functions for encoding Erlang terms into a sequence of bytes, and for decoding Erlang terms from a sequence of bytes.</p> diff --git a/lib/erl_interface/doc/src/ref_man.xml b/lib/erl_interface/doc/src/ref_man.xml index 1e20637cb7..a4f947c79f 100644 --- a/lib/erl_interface/doc/src/ref_man.xml +++ b/lib/erl_interface/doc/src/ref_man.xml @@ -29,6 +29,14 @@ <file>ref_man.xml</file> </header> <description> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <note><p>The old legacy <c>erl_interface</c> library (functions + with prefix <c>erl_</c>) is deprecated as of OTP 22, and will be + removed in OTP 23. This does not apply to the <c>ei</c> + library. Reasonably new <c>gcc</c> compilers will issue deprecation + warnings. In order to disable these warnings, define the macro + <c>EI_NO_DEPR_WARN</c>.</p></note> </description> <xi:include href="ei.xml"/> <xi:include href="ei_connect.xml"/> diff --git a/lib/erl_interface/doc/src/ref_man_ei.xml b/lib/erl_interface/doc/src/ref_man_ei.xml index 92ff9ed328..d8d1deaea1 100644 --- a/lib/erl_interface/doc/src/ref_man_ei.xml +++ b/lib/erl_interface/doc/src/ref_man_ei.xml @@ -30,8 +30,14 @@ <file>ref_man_ei.xml</file> </header> <description> - <p>The <c>ei</c> library is a <c>C</c> interface library for - communication with <c>Erlang</c>.</p> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <note><p>The old legacy <c>erl_interface</c> library (functions + with prefix <c>erl_</c>) is deprecated as of OTP 22, and will be + removed in OTP 23. This does not apply to the <c>ei</c> + library. Reasonably new <c>gcc</c> compilers will issue deprecation + warnings. In order to disable these warnings, define the macro + <c>EI_NO_DEPR_WARN</c>.</p></note> <note> <p>By default, the <c>ei</c> library is only guaranteed to be compatible with other Erlang/OTP components from the same diff --git a/lib/erl_interface/doc/src/ref_man_erl_interface.xml b/lib/erl_interface/doc/src/ref_man_erl_interface.xml index 4b1d0e9981..2b69d0fa74 100644 --- a/lib/erl_interface/doc/src/ref_man_erl_interface.xml +++ b/lib/erl_interface/doc/src/ref_man_erl_interface.xml @@ -30,6 +30,14 @@ <file>ref_man_erl_interface.xml</file> </header> <description> + <note><p>The support for VxWorks is deprecated as of OTP 22, and + will be removed in OTP 23.</p></note> + <note><p>The old legacy <c>erl_interface</c> library (functions + with prefix <c>erl_</c>) is deprecated as of OTP 22, and will be + removed in OTP 23. This does not apply to the <c>ei</c> + library. Reasonably new <c>gcc</c> compilers will issue deprecation + warnings. In order to disable these warnings, define the macro + <c>EI_NO_DEPR_WARN</c>.</p></note> <p>The <c>erl_interface</c> library is a <c>C</c> interface library for communication with <c>Erlang</c>.</p> <note> diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h index 947726208e..b138118f04 100644 --- a/lib/erl_interface/include/ei.h +++ b/lib/erl_interface/include/ei.h @@ -47,6 +47,25 @@ typedef LONG_PTR ssize_t; /* Sigh... */ # include <netdb.h> #endif +#ifdef __has_attribute +#if __has_attribute(deprecated) +#define EI_HAVE_DEPRECATED_ATTR__ 1 +#else +#undef EI_HAVE_DEPRECATED_ATTR__ +#endif +#endif + +#ifdef EI_NO_DEPR_WARN +#undef EI_HAVE_DEPRECATED_ATTR__ +#endif + +#ifdef EI_HAVE_DEPRECATED_ATTR__ +#define EI_DEPRECATED_ATTR_NAME deprecated +#define EI_DEPRECATED_ATTR __attribute__((EI_DEPRECATED_ATTR_NAME)) +#else +#define EI_DEPRECATED_ATTR_NAME +#define EI_DEPRECATED_ATTR +#endif /* -------------------------------------------------------------------- */ /* Defines part of API */ @@ -135,11 +154,14 @@ typedef LONG_PTR ssize_t; /* Sigh... */ #define ERL_STRING_EXT 'k' #define ERL_LIST_EXT 'l' #define ERL_BINARY_EXT 'm' +#define ERL_BIT_BINARY_EXT 'M' #define ERL_SMALL_BIG_EXT 'n' #define ERL_LARGE_BIG_EXT 'o' #define ERL_NEW_FUN_EXT 'p' #define ERL_MAP_EXT 't' #define ERL_FUN_EXT 'u' +#define ERL_EXPORT_EXT 'q' + #define ERL_NEW_CACHE 'N' /* c nodes don't know these two */ #define ERL_CACHED_ATOM 'C' @@ -250,15 +272,23 @@ typedef struct { typedef struct { long arity; char module[MAXATOMLEN_UTF8]; - erlang_char_encoding module_org_enc; - char md5[16]; - long index; - long old_index; - long uniq; - long n_free_vars; - erlang_pid pid; - long free_var_len; - char* free_vars; + enum { EI_FUN_CLOSURE, EI_FUN_EXPORT } type; + union { + struct { + char md5[16]; + long index; + long old_index; + long uniq; + long n_free_vars; + erlang_pid pid; + long free_var_len; + char* free_vars; + } closure; + struct { + char* func; + int func_allocated; + } exprt; + } u; } erlang_fun; /* a big */ @@ -496,7 +526,9 @@ int ei_x_encode_atom_len(ei_x_buff* x, const char* s, int len); int ei_x_encode_atom_len_as(ei_x_buff* x, const char* s, int len, erlang_char_encoding from, erlang_char_encoding to); int ei_encode_binary(char *buf, int *index, const void *p, long len); +int ei_encode_bitstring(char *buf, int *index, const char *p, size_t bitoffs, size_t bits); int ei_x_encode_binary(ei_x_buff* x, const void* s, int len); +int ei_x_encode_bitstring(ei_x_buff* x, const char* p, size_t bitoffs, size_t bits); int ei_encode_pid(char *buf, int *index, const erlang_pid *p); int ei_x_encode_pid(ei_x_buff* x, const erlang_pid* pid); int ei_encode_fun(char* buf, int* index, const erlang_fun* p); @@ -505,8 +537,8 @@ int ei_encode_port(char *buf, int *index, const erlang_port *p); int ei_x_encode_port(ei_x_buff* x, const erlang_port *p); int ei_encode_ref(char *buf, int *index, const erlang_ref *p); int ei_x_encode_ref(ei_x_buff* x, const erlang_ref *p); -int ei_encode_term(char *buf, int *index, void *t); /* ETERM* actually */ -int ei_x_encode_term(ei_x_buff* x, void* t); +int ei_encode_term(char *buf, int *index, void *t) EI_DEPRECATED_ATTR; +int ei_x_encode_term(ei_x_buff* x, void* t) EI_DEPRECATED_ATTR; int ei_encode_trace(char *buf, int *index, const erlang_trace *p); int ei_x_encode_trace(ei_x_buff* x, const erlang_trace *p); int ei_encode_tuple_header(char *buf, int *index, int arity); @@ -528,8 +560,6 @@ int ei_x_encode_map_header(ei_x_buff* x, long n); */ int ei_get_type(const char *buf, const int *index, int *type, int *size); -int ei_get_type_internal(const char *buf, const int *index, int *type, - int *size); /* Step through buffer, decoding the given type into the buffer * provided. On success, 0 is returned and index is updated to point @@ -548,12 +578,15 @@ int ei_decode_string(const char *buf, int *index, char *p); int ei_decode_atom(const char *buf, int *index, char *p); int ei_decode_atom_as(const char *buf, int *index, char *p, int destlen, erlang_char_encoding want, erlang_char_encoding* was, erlang_char_encoding* result); int ei_decode_binary(const char *buf, int *index, void *p, long *len); +int ei_decode_bitstring(const char *buf, int *index, const char** pp, + unsigned int* bitoffsp, size_t *nbitsp); + int ei_decode_fun(const char* buf, int* index, erlang_fun* p); void free_fun(erlang_fun* f); int ei_decode_pid(const char *buf, int *index, erlang_pid *p); int ei_decode_port(const char *buf, int *index, erlang_port *p); int ei_decode_ref(const char *buf, int *index, erlang_ref *p); -int ei_decode_term(const char *buf, int *index, void *t); /* ETERM** actually */ +int ei_decode_term(const char *buf, int *index, void *t) EI_DEPRECATED_ATTR; int ei_decode_trace(const char *buf, int *index, erlang_trace *p); int ei_decode_tuple_header(const char *buf, int *index, int *arity); int ei_decode_list_header(const char *buf, int *index, int *arity); diff --git a/lib/erl_interface/include/erl_interface.h b/lib/erl_interface/include/erl_interface.h index c22f21af2b..7c87223a38 100644 --- a/lib/erl_interface/include/erl_interface.h +++ b/lib/erl_interface/include/erl_interface.h @@ -25,8 +25,6 @@ /* Note: the 'ei' interface is the prefered C API. */ /************************************************************************/ -/* FIXME only include if needed? */ - #include "ei.h" /* ei is the base */ /* -------------------------------------------------------------------- */ @@ -195,11 +193,11 @@ typedef struct { int lenL; } Erl_Atom_data; -char* erl_atom_ptr_latin1(Erl_Atom_data*); -char* erl_atom_ptr_utf8(Erl_Atom_data*); -int erl_atom_size_latin1(Erl_Atom_data*); -int erl_atom_size_utf8(Erl_Atom_data*); -char* erl_atom_init_latin1(Erl_Atom_data*, const char*); +char* erl_atom_ptr_latin1(Erl_Atom_data*) EI_DEPRECATED_ATTR; +char* erl_atom_ptr_utf8(Erl_Atom_data*) EI_DEPRECATED_ATTR; +int erl_atom_size_latin1(Erl_Atom_data*) EI_DEPRECATED_ATTR; +int erl_atom_size_utf8(Erl_Atom_data*) EI_DEPRECATED_ATTR; +char* erl_atom_init_latin1(Erl_Atom_data*, const char*) EI_DEPRECATED_ATTR; typedef struct { Erl_Header h; @@ -324,110 +322,117 @@ typedef unsigned char Erl_Heap; /* The functions */ /* -------------------------------------------------------------------- */ -void erl_init(void *x, long y); -void erl_set_compat_rel(unsigned); -int erl_connect_init(int, char*,short); -int erl_connect_xinit(char*,char*,char*,struct in_addr*,char*,short); -int erl_connect(char*); -int erl_xconnect(struct in_addr*,char *); -int erl_close_connection(int); -int erl_receive(int, unsigned char*, int); -int erl_receive_msg(int, unsigned char*, int, ErlMessage*); -int erl_xreceive_msg(int, unsigned char**, int*, ErlMessage*); -int erl_send(int, ETERM*, ETERM*); -int erl_reg_send(int, char*, ETERM*); -ETERM *erl_rpc(int,char*,char*,ETERM*); -int erl_rpc_to(int,char*,char*,ETERM*); -int erl_rpc_from(int,int,ErlMessage*); +void erl_init(void *x, long y) EI_DEPRECATED_ATTR; +void erl_set_compat_rel(unsigned) EI_DEPRECATED_ATTR; +int erl_connect_init(int, char*,short) EI_DEPRECATED_ATTR; +int erl_connect_xinit(char*,char*,char*,struct in_addr*,char*,short) EI_DEPRECATED_ATTR; +int erl_connect(char*) EI_DEPRECATED_ATTR; +int erl_xconnect(struct in_addr*,char *) EI_DEPRECATED_ATTR; +int erl_close_connection(int) EI_DEPRECATED_ATTR; +int erl_receive(int, unsigned char*, int) EI_DEPRECATED_ATTR; +int erl_receive_msg(int, unsigned char*, int, ErlMessage*) EI_DEPRECATED_ATTR; +int erl_xreceive_msg(int, unsigned char**, int*, ErlMessage*) EI_DEPRECATED_ATTR; +int erl_send(int, ETERM*, ETERM*) EI_DEPRECATED_ATTR; +int erl_reg_send(int, char*, ETERM*) EI_DEPRECATED_ATTR; +ETERM *erl_rpc(int,char*,char*,ETERM*) EI_DEPRECATED_ATTR; +int erl_rpc_to(int,char*,char*,ETERM*) EI_DEPRECATED_ATTR; +int erl_rpc_from(int,int,ErlMessage*) EI_DEPRECATED_ATTR; /* erl_publish returns open descriptor on success, or -1 */ -int erl_publish(int port); -int erl_accept(int,ErlConnect*); +int erl_publish(int port) EI_DEPRECATED_ATTR; +int erl_accept(int,ErlConnect*) EI_DEPRECATED_ATTR; -const char *erl_thiscookie(void); -const char *erl_thisnodename(void); -const char *erl_thishostname(void); -const char *erl_thisalivename(void); -short erl_thiscreation(void); +const char *erl_thiscookie(void) EI_DEPRECATED_ATTR; +const char *erl_thisnodename(void) EI_DEPRECATED_ATTR; +const char *erl_thishostname(void) EI_DEPRECATED_ATTR; +const char *erl_thisalivename(void) EI_DEPRECATED_ATTR; +short erl_thiscreation(void) EI_DEPRECATED_ATTR; /* returns 0 on success, -1 if node not known to epmd or epmd not reached */ -int erl_unpublish(const char *alive); +int erl_unpublish(const char *alive) EI_DEPRECATED_ATTR; + +#ifdef EI_HAVE_DEPRECATED_ATTR__ +#define EI_DEPR_ATTR_EXTRA , EI_DEPRECATED_ATTR_NAME +#else +#define EI_DEPR_ATTR_EXTRA +#endif + /* Report generic error to stderr. */ void erl_err_msg(const char * __template, ...) - __attribute__ ((__format__ (printf, 1, 2))); + __attribute__ ((__format__ (printf, 1, 2) EI_DEPR_ATTR_EXTRA)) ; /* Report generic error to stderr and die. */ void erl_err_quit(const char * __template, ...) - __attribute__ ((__format__ (printf, 1, 2), __noreturn__)); + __attribute__ ((__format__ (printf, 1, 2), __noreturn__ EI_DEPR_ATTR_EXTRA)); /* Report system/libc error to stderr. */ void erl_err_ret(const char * __template, ...) - __attribute__ ((__format__ (printf, 1, 2))); + __attribute__ ((__format__ (printf, 1, 2) EI_DEPR_ATTR_EXTRA)); /* Report system/libc error to stderr and die. */ void erl_err_sys(const char * __template, ...) - __attribute__ ((__format__ (printf, 1, 2), __noreturn__)); - -ETERM *erl_cons(ETERM*,ETERM*); -ETERM *erl_copy_term(const ETERM*); -ETERM *erl_element(int,const ETERM*); - -ETERM *erl_hd(const ETERM*); -ETERM* erl_iolist_to_binary(const ETERM* term); -char* erl_iolist_to_string(const ETERM* term); -int erl_iolist_length(const ETERM*); -int erl_length(const ETERM*); -ETERM *erl_mk_atom(const char*); -ETERM *erl_mk_binary(const char*,int); -ETERM *erl_mk_empty_list(void); -ETERM *erl_mk_estring(const char*, int); -ETERM *erl_mk_float(double); -ETERM *erl_mk_int(int); -ETERM *erl_mk_longlong(long long); -ETERM *erl_mk_list(ETERM**,int); -ETERM *erl_mk_pid(const char*,unsigned int,unsigned int,unsigned char); -ETERM *erl_mk_port(const char*,unsigned int,unsigned char); -ETERM *erl_mk_ref(const char*,unsigned int,unsigned char); + __attribute__ ((__format__ (printf, 1, 2), __noreturn__ EI_DEPR_ATTR_EXTRA)); + +ETERM *erl_cons(ETERM*,ETERM*) EI_DEPRECATED_ATTR; +ETERM *erl_copy_term(const ETERM*) EI_DEPRECATED_ATTR; +ETERM *erl_element(int,const ETERM*) EI_DEPRECATED_ATTR; + +ETERM *erl_hd(const ETERM*) EI_DEPRECATED_ATTR; +ETERM* erl_iolist_to_binary(const ETERM* term) EI_DEPRECATED_ATTR; +char* erl_iolist_to_string(const ETERM* term) EI_DEPRECATED_ATTR; +int erl_iolist_length(const ETERM*) EI_DEPRECATED_ATTR; +int erl_length(const ETERM*) EI_DEPRECATED_ATTR; +ETERM *erl_mk_atom(const char*) EI_DEPRECATED_ATTR; +ETERM *erl_mk_binary(const char*,int) EI_DEPRECATED_ATTR; +ETERM *erl_mk_empty_list(void) EI_DEPRECATED_ATTR; +ETERM *erl_mk_estring(const char*, int) EI_DEPRECATED_ATTR; +ETERM *erl_mk_float(double) EI_DEPRECATED_ATTR; +ETERM *erl_mk_int(int) EI_DEPRECATED_ATTR; +ETERM *erl_mk_longlong(long long) EI_DEPRECATED_ATTR; +ETERM *erl_mk_list(ETERM**,int) EI_DEPRECATED_ATTR; +ETERM *erl_mk_pid(const char*,unsigned int,unsigned int,unsigned char) EI_DEPRECATED_ATTR; +ETERM *erl_mk_port(const char*,unsigned int,unsigned char) EI_DEPRECATED_ATTR; +ETERM *erl_mk_ref(const char*,unsigned int,unsigned char) EI_DEPRECATED_ATTR; ETERM *erl_mk_long_ref(const char*,unsigned int,unsigned int, - unsigned int,unsigned char); -ETERM *erl_mk_string(const char*); -ETERM *erl_mk_tuple(ETERM**,int); -ETERM *erl_mk_uint(unsigned int); -ETERM *erl_mk_ulonglong(unsigned long long); -ETERM *erl_mk_var(const char*); -int erl_print_term(FILE*,const ETERM*); -/* int erl_sprint_term(char*,const ETERM*); */ -int erl_size(const ETERM*); -ETERM *erl_tl(const ETERM*); -ETERM *erl_var_content(const ETERM*, const char*); - -ETERM *erl_format(char*, ... ); -int erl_match(ETERM*, ETERM*); - -char **erl_global_names(int fd, int *count); -int erl_global_register(int fd, const char *name, ETERM *pid); -int erl_global_unregister(int fd, const char *name); -ETERM *erl_global_whereis(int fd, const char *name, char *node); - -void erl_init_malloc(Erl_Heap*,long); -ETERM *erl_alloc_eterm(unsigned char); -void erl_eterm_release(void); -void erl_eterm_statistics(unsigned long*,unsigned long*); -void erl_free_array(ETERM**,int); -void erl_free_term(ETERM*); -void erl_free_compound(ETERM*); -void *erl_malloc(long); -void erl_free(void*); - -int erl_compare_ext(unsigned char*, unsigned char*); -ETERM *erl_decode(unsigned char*); -ETERM *erl_decode_buf(unsigned char**); -int erl_encode(ETERM*,unsigned char*t); -int erl_encode_buf(ETERM*,unsigned char**); -int erl_ext_size(unsigned char*); -unsigned char erl_ext_type(unsigned char*); /* Note: returned 'char' before R9C */ -unsigned char *erl_peek_ext(unsigned char*,int); -int erl_term_len(ETERM*); - -int cmp_latin1_vs_utf8(const char* sL, int lenL, const char* sU, int lenU); + unsigned int,unsigned char) EI_DEPRECATED_ATTR; +ETERM *erl_mk_string(const char*) EI_DEPRECATED_ATTR; +ETERM *erl_mk_tuple(ETERM**,int) EI_DEPRECATED_ATTR; +ETERM *erl_mk_uint(unsigned int) EI_DEPRECATED_ATTR; +ETERM *erl_mk_ulonglong(unsigned long long) EI_DEPRECATED_ATTR; +ETERM *erl_mk_var(const char*) EI_DEPRECATED_ATTR; +int erl_print_term(FILE*,const ETERM*) EI_DEPRECATED_ATTR; +/* int erl_sprint_term(char*,const ETERM*) EI_DEPRECATED_ATTR; */ +int erl_size(const ETERM*) EI_DEPRECATED_ATTR; +ETERM *erl_tl(const ETERM*) EI_DEPRECATED_ATTR; +ETERM *erl_var_content(const ETERM*, const char*) EI_DEPRECATED_ATTR; + +ETERM *erl_format(char*, ... ) EI_DEPRECATED_ATTR; +int erl_match(ETERM*, ETERM*) EI_DEPRECATED_ATTR; + +char **erl_global_names(int fd, int *count) EI_DEPRECATED_ATTR; +int erl_global_register(int fd, const char *name, ETERM *pid) EI_DEPRECATED_ATTR; +int erl_global_unregister(int fd, const char *name) EI_DEPRECATED_ATTR; +ETERM *erl_global_whereis(int fd, const char *name, char *node) EI_DEPRECATED_ATTR; + +void erl_init_malloc(Erl_Heap*,long) EI_DEPRECATED_ATTR; +ETERM *erl_alloc_eterm(unsigned char) EI_DEPRECATED_ATTR; +void erl_eterm_release(void) EI_DEPRECATED_ATTR; +void erl_eterm_statistics(unsigned long*,unsigned long*) EI_DEPRECATED_ATTR; +void erl_free_array(ETERM**,int) EI_DEPRECATED_ATTR; +void erl_free_term(ETERM*) EI_DEPRECATED_ATTR; +void erl_free_compound(ETERM*) EI_DEPRECATED_ATTR; +void *erl_malloc(long) EI_DEPRECATED_ATTR; +void erl_free(void*) EI_DEPRECATED_ATTR; + +int erl_compare_ext(unsigned char*, unsigned char*) EI_DEPRECATED_ATTR; +ETERM *erl_decode(unsigned char*) EI_DEPRECATED_ATTR; +ETERM *erl_decode_buf(unsigned char**) EI_DEPRECATED_ATTR; +int erl_encode(ETERM*,unsigned char*t) EI_DEPRECATED_ATTR; +int erl_encode_buf(ETERM*,unsigned char**) EI_DEPRECATED_ATTR; +int erl_ext_size(unsigned char*) EI_DEPRECATED_ATTR; +unsigned char erl_ext_type(unsigned char*) EI_DEPRECATED_ATTR; /* Note: returned 'char' before R9C */ +unsigned char *erl_peek_ext(unsigned char*,int) EI_DEPRECATED_ATTR; +int erl_term_len(ETERM*) EI_DEPRECATED_ATTR; + +int cmp_latin1_vs_utf8(const char* sL, int lenL, const char* sU, int lenU) EI_DEPRECATED_ATTR; /* -------------------------------------------------------------------- */ /* Wrappers around ei functions */ @@ -437,29 +442,29 @@ int cmp_latin1_vs_utf8(const char* sL, int lenL, const char* sU, int lenU); * Undocumented before R9C, included for compatibility with old code */ -struct hostent *erl_gethostbyname(const char *name); -struct hostent *erl_gethostbyaddr(const char *addr, int len, int type); +struct hostent *erl_gethostbyname(const char *name) EI_DEPRECATED_ATTR; +struct hostent *erl_gethostbyaddr(const char *addr, int len, int type) EI_DEPRECATED_ATTR; struct hostent *erl_gethostbyname_r(const char *name, struct hostent *hostp, char *buffer, int buflen, - int *h_errnop); + int *h_errnop) EI_DEPRECATED_ATTR; struct hostent *erl_gethostbyaddr_r(const char *addr, int length, int type, struct hostent *hostp, char *buffer, int buflen, - int *h_errnop); + int *h_errnop) EI_DEPRECATED_ATTR; /* * Undocumented, included for compatibility with old code */ -void erl_init_resolve(void); -int erl_distversion(int fd); -int erl_epmd_connect(struct in_addr *inaddr); -int erl_epmd_port(struct in_addr *inaddr, const char *alive, int *dist); +void erl_init_resolve(void) EI_DEPRECATED_ATTR; +int erl_distversion(int fd) EI_DEPRECATED_ATTR; +int erl_epmd_connect(struct in_addr *inaddr) EI_DEPRECATED_ATTR; +int erl_epmd_port(struct in_addr *inaddr, const char *alive, int *dist) EI_DEPRECATED_ATTR; #ifdef __cplusplus } diff --git a/lib/erl_interface/src/INSTALL b/lib/erl_interface/src/INSTALL index b42a17ac46..bf3ca8b6a5 100644 --- a/lib/erl_interface/src/INSTALL +++ b/lib/erl_interface/src/INSTALL @@ -122,10 +122,10 @@ you can use the `configure' options `--x-includes=DIR' and Specifying the System Type ========================== - There may be some features `configure' can not figure out + There may be some features `configure' cannot figure out automatically, but needs to determine by the type of host the package will run on. Usually `configure' can figure that out, but if it prints -a message saying it can not guess the host type, give it the +a message saying it cannot guess the host type, give it the `--host=TYPE' option. TYPE can either be a short name for the system type, such as `sun4', or a canonical name with three fields: CPU-COMPANY-SYSTEM diff --git a/lib/erl_interface/src/Makefile b/lib/erl_interface/src/Makefile index 800557fbfe..00c49f1622 100644 --- a/lib/erl_interface/src/Makefile +++ b/lib/erl_interface/src/Makefile @@ -27,7 +27,9 @@ include $(ERL_TOP)/make/output.mk include $(ERL_TOP)/make/target.mk debug opt shared purify quantify purecov gcov: +ifndef TERTIARY_BOOTSTRAP $(make_verbose)$(MAKE) -f $(TARGET)/Makefile TYPE=$@ +endif clean depend docs release release_docs tests release_tests check xmllint: $(make_verbose)$(MAKE) -f $(TARGET)/Makefile $@ diff --git a/lib/erl_interface/src/Makefile.in b/lib/erl_interface/src/Makefile.in index 2c1a8252ae..6e0d3476c7 100644 --- a/lib/erl_interface/src/Makefile.in +++ b/lib/erl_interface/src/Makefile.in @@ -31,6 +31,8 @@ .PHONY : debug opt release clean distclean depend +include $(ERL_TOP)/make/target.mk + # ---------------------------------------------------- # Application version and release dir specification # ---------------------------------------------------- @@ -125,6 +127,8 @@ else WARNFLAGS = @WFLAGS@ endif +WARNFLAGS += -DEI_NO_DEPR_WARN + CFLAGS = @LIB_CFLAGS@ $(WARNFLAGS) $(INCFLAGS) $(TYPE_FLAGS) PROG_CFLAGS = @CFLAGS@ $(WARNFLAGS) $(INCFLAGS) $(TYPE_FLAGS) -Ilegacy diff --git a/lib/erl_interface/src/README.internal b/lib/erl_interface/src/README.internal index c1f2d6863f..42c45b46a9 100644 --- a/lib/erl_interface/src/README.internal +++ b/lib/erl_interface/src/README.internal @@ -167,12 +167,12 @@ NOTE!!!! Sending a "char" to macros like isupper(), isalpha() where the character is > 127 will cause serios problems on some machines/OS. The reason is that - 'char' may be unsigned, i.e. the Swedish char '�' will + 'char' may be unsigned, i.e. the Swedish char 'ä' will as a number be negativ. The implementation of isupper() and others will on some machines use an array that is indexed with the incoming - character code. The Swedish '�' will then create an access + character code. The Swedish 'ä' will then create an access on memory outside the array! This may give a random value as a result or a segmentation @@ -219,7 +219,7 @@ There are some functions in the 'ei' library that uses the GCC and VC++ "long long" type. Unfortunately this can lead to some trouble. When user code is linked with the "libei.a" the linker will extract -all objects files needed for resolving all symbol referenses +all objects files needed for resolving all symbol references found. This means that you want to follow the rule that * To reduce executable code size we use resonably small C source @@ -252,7 +252,7 @@ example is that in plain R9C the ei_x_encode_longlong() function is located in the file "ei_x_encode.c". So if any "long long" ei_x function is used we have an unessesary dependency on "ei_encode_longlong.o" and then need to link with GNU ld on with the -user code or explicitely link with "libgcc.a". The situation can be +user code or explicitly link with "libgcc.a". The situation can be visible in in plain R9C using % nm -A erl_interface-3.4/lib/libei.a | \ diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c index 95cb10a20c..1b1479d2e9 100644 --- a/lib/erl_interface/src/connect/ei_connect.c +++ b/lib/erl_interface/src/connect/ei_connect.c @@ -1854,6 +1854,7 @@ static int send_name_or_challenge(ei_socket_callbacks *cbs, const char* function[] = {"SEND_NAME", "SEND_CHALLENGE"}; int err; ssize_t len; + unsigned int flags; if (f_chall) siz += 4; @@ -1875,7 +1876,7 @@ static int send_name_or_challenge(ei_socket_callbacks *cbs, } put8(s, 'n'); put16be(s, version); - put32be(s, (DFLAG_EXTENDED_REFERENCES + flags = (DFLAG_EXTENDED_REFERENCES | DFLAG_DIST_MONITOR | DFLAG_EXTENDED_PIDS_PORTS | DFLAG_FUN_TAGS @@ -1884,7 +1885,14 @@ static int send_name_or_challenge(ei_socket_callbacks *cbs, | DFLAG_SMALL_ATOM_TAGS | DFLAG_UTF8_ATOMS | DFLAG_MAP_TAG - | DFLAG_BIG_CREATION)); + | DFLAG_BIG_CREATION + | DFLAG_EXPORT_PTR_TAG + | DFLAG_BIT_BINARIES); + if (ei_internal_use_21_bitstr_expfun()) { + flags &= ~(DFLAG_EXPORT_PTR_TAG + | DFLAG_BIT_BINARIES); + } + put32be(s, flags); if (f_chall) put32be(s, challenge); memcpy(s, nodename, strlen(nodename)); @@ -1949,8 +1957,7 @@ static int recv_challenge(ei_socket_callbacks *cbs, void *ctx, goto error; } - if (!(*flags & DFLAG_EXTENDED_PIDS_PORTS) - && !ei_internal_use_r9_pids_ports()) { + if (!(*flags & DFLAG_EXTENDED_PIDS_PORTS)) { EI_TRACE_ERR0("recv_challenge","<- RECV_CHALLENGE peer cannot " "handle extended pids and ports"); erl_errno = EIO; @@ -2244,8 +2251,7 @@ static int recv_name(ei_socket_callbacks *cbs, void *ctx, goto error; } - if (!(*flags & DFLAG_EXTENDED_PIDS_PORTS) - && !ei_internal_use_r9_pids_ports()) { + if (!(*flags & DFLAG_EXTENDED_PIDS_PORTS)) { EI_TRACE_ERR0("recv_name","<- RECV_NAME peer cannot " "handle extended pids and ports"); erl_errno = EIO; diff --git a/lib/erl_interface/src/connect/ei_connect_int.h b/lib/erl_interface/src/connect/ei_connect_int.h index 0bcccaa84b..b41a5f2b23 100644 --- a/lib/erl_interface/src/connect/ei_connect_int.h +++ b/lib/erl_interface/src/connect/ei_connect_int.h @@ -102,6 +102,8 @@ extern int h_errno; #define DFLAG_FUN_TAGS 16 #define DFLAG_NEW_FUN_TAGS 0x80 #define DFLAG_EXTENDED_PIDS_PORTS 0x100 +#define DFLAG_EXPORT_PTR_TAG 0x200 +#define DFLAG_BIT_BINARIES 0x400 #define DFLAG_NEW_FLOATS 0x800 #define DFLAG_SMALL_ATOM_TAGS 0x4000 #define DFLAG_UTF8_ATOMS 0x10000 diff --git a/lib/erl_interface/src/decode/decode_binary.c b/lib/erl_interface/src/decode/decode_binary.c index 5b8d234984..0d28c67230 100644 --- a/lib/erl_interface/src/decode/decode_binary.c +++ b/lib/erl_interface/src/decode/decode_binary.c @@ -40,4 +40,41 @@ int ei_decode_binary(const char *buf, int *index, void *p, long *lenp) return 0; } +int ei_decode_bitstring(const char *buf, int *index, + const char** pp, + unsigned int* bitoffsp, + size_t *nbitsp) +{ + const char *s = buf + *index; + const char *s0 = s; + unsigned char last_bits; + const unsigned char tag = get8(s); + size_t len = get32be(s); + + switch(tag) { + case ERL_BINARY_EXT: + if (nbitsp) + *nbitsp = len * 8; + break; + case ERL_BIT_BINARY_EXT: + last_bits = get8(s); + if (((last_bits==0) != (len==0)) || last_bits > 8) + return -1; + + if (nbitsp) + *nbitsp = (len == 0) ? 0 : ((len-1) * 8) + last_bits; + break; + default: + return -1; + } + + if (pp) + *pp = s; + if (bitoffsp) + *bitoffsp = 0; + + s += len; + *index += s-s0; + return 0; +} diff --git a/lib/erl_interface/src/decode/decode_fun.c b/lib/erl_interface/src/decode/decode_fun.c index f944c028af..3a7a2b01c1 100644 --- a/lib/erl_interface/src/decode/decode_fun.c +++ b/lib/erl_interface/src/decode/decode_fun.c @@ -33,22 +33,20 @@ int ei_decode_fun(const char *buf, int *index, erlang_fun *p) int i, ix, ix0, n; erlang_pid* p_pid; char* p_module; - erlang_char_encoding* p_module_org_enc; long* p_index; long* p_uniq; long* p_old_index; if (p != NULL) { - p_pid = &p->pid; + p_pid = &p->u.closure.pid; p_module = &p->module[0]; - p_module_org_enc = &p->module_org_enc; - p_index = &p->index; - p_uniq = &p->uniq; - p_old_index = &p->old_index; + p_index = &p->u.closure.index; + p_uniq = &p->u.closure.uniq; + p_old_index = &p->u.closure.old_index; } else { - p_pid = NULL; p_module = NULL; p_module_org_enc = NULL; p_index = NULL; p_uniq = NULL; p_old_index = NULL; + p_pid = NULL; p_module = NULL; } switch (get8(s)) { @@ -63,7 +61,7 @@ int ei_decode_fun(const char *buf, int *index, erlang_fun *p) return -1; /* then the module (atom) */ if (ei_decode_atom_as(s, &ix, p_module, MAXATOMLEN_UTF8, ERLANG_UTF8, - p_module_org_enc, NULL) < 0) + NULL, NULL) < 0) return -1; /* then the index */ if (ei_decode_long(s, &ix, p_index) < 0) @@ -78,11 +76,11 @@ int ei_decode_fun(const char *buf, int *index, erlang_fun *p) return -1; } if (p != NULL) { - p->n_free_vars = n; - p->free_var_len = ix - ix0; - p->free_vars = ei_malloc(ix - ix0); - if (!(p->free_vars)) return -1; - memcpy(p->free_vars, s + ix0, ix - ix0); + p->u.closure.n_free_vars = n; + p->u.closure.free_var_len = ix - ix0; + p->u.closure.free_vars = ei_malloc(ix - ix0); + if (!(p->u.closure.free_vars)) return -1; + memcpy(p->u.closure.free_vars, s + ix0, ix - ix0); } s += ix; *index += s-s0; @@ -93,20 +91,23 @@ int ei_decode_fun(const char *buf, int *index, erlang_fun *p) n = get32be(s); /* then the arity */ i = get8(s); - if (p != NULL) p->arity = i; - /* then md5 */ - if (p != NULL) memcpy(p->md5, s, 16); + if (p != NULL) { + p->type = EI_FUN_CLOSURE; + p->arity = i; + /* then md5 */ + memcpy(p->u.closure.md5, s, 16); + } s += 16; /* then index */ i = get32be(s); - if (p != NULL) p->index = i; + if (p != NULL) p->u.closure.index = i; /* then the number of free vars (environment) */ i = get32be(s); - if (p != NULL) p->n_free_vars = i; + if (p != NULL) p->u.closure.n_free_vars = i; /* then the module (atom) */ ix = 0; if (ei_decode_atom_as(s, &ix, p_module, MAXATOMLEN_UTF8, ERLANG_UTF8, - p_module_org_enc, NULL) < 0) + NULL, NULL) < 0) return -1; /* then the old_index */ if (ei_decode_long(s, &ix, p_old_index) < 0) @@ -122,17 +123,56 @@ int ei_decode_fun(const char *buf, int *index, erlang_fun *p) n = n - (s - s0) + 1; if (n < 0) return -1; if (p != NULL) { - p->free_var_len = n; + p->u.closure.free_var_len = n; if (n > 0) { - p->free_vars = malloc(n); - if (!(p->free_vars)) return -1; - memcpy(p->free_vars, s, n); + p->u.closure.free_vars = malloc(n); + if (!(p->u.closure.free_vars)) return -1; + memcpy(p->u.closure.free_vars, s, n); } } s += n; *index += s-s0; return 0; break; + case ERL_EXPORT_EXT: { + char* p_func; + long* p_arity; + int used; + + if (p) { + p->type = EI_FUN_EXPORT; + p_arity = &p->arity; + } + else { + p_arity = NULL; + } + if (ei_decode_atom_as(s, &ix, p_module, MAXATOMLEN_UTF8, ERLANG_UTF8, + NULL, NULL) < 0) + return -1; + if (p) { + /* try use module buffer for function name */ + used = strlen(p->module) + 1; + p_func = p->module + used; + p->u.exprt.func = p_func; + p->u.exprt.func_allocated = 0; + } + else { + used = 0; + p_func = NULL; + } + while (ei_decode_atom_as(s, &ix, p_func, MAXATOMLEN_UTF8-used, + ERLANG_UTF8, NULL, NULL) < 0) { + if (!used) + return -1; + p_func = malloc(MAXATOMLEN_UTF8); + p->u.exprt.func = p_func; + p->u.exprt.func_allocated = 1; + used = 0; + } + if (ei_decode_long(s, &ix, p_arity) < 0) + return -1; + return 0; + } default: return -1; } @@ -140,6 +180,14 @@ int ei_decode_fun(const char *buf, int *index, erlang_fun *p) void free_fun(erlang_fun* f) { - if (f->free_var_len > 0) - ei_free(f->free_vars); + switch (f->type) { + case EI_FUN_CLOSURE: + if (f->u.closure.free_var_len > 0) + ei_free(f->u.closure.free_vars); + break; + case EI_FUN_EXPORT: + if (f->u.exprt.func_allocated) + ei_free(f->u.exprt.func); + break; + } } diff --git a/lib/erl_interface/src/decode/decode_skip.c b/lib/erl_interface/src/decode/decode_skip.c index 0db315f09b..736c00e074 100644 --- a/lib/erl_interface/src/decode/decode_skip.c +++ b/lib/erl_interface/src/decode/decode_skip.c @@ -27,7 +27,7 @@ int ei_skip_term(const char* buf, int* index) /* ASSERT(ep != NULL); */ - ei_get_type_internal(buf, index, &ty, &n); + ei_get_type(buf, index, &ty, &n); switch (ty) { case ERL_ATOM_EXT: /* FIXME: what if some weird locale is in use? */ @@ -54,7 +54,7 @@ int ei_skip_term(const char* buf, int* index) if (ei_decode_list_header(buf, index, &n) < 0) return -1; for (i = 0; i < n; ++i) ei_skip_term(buf, index); - if (ei_get_type_internal(buf, index, &ty, &n) < 0) return -1; + if (ei_get_type(buf, index, &ty, &n) < 0) return -1; if (ty != ERL_NIL_EXT) ei_skip_term(buf, index); else @@ -79,6 +79,10 @@ int ei_skip_term(const char* buf, int* index) if (ei_decode_binary(buf, index, NULL, NULL) < 0) return -1; break; + case ERL_BIT_BINARY_EXT: + if (ei_decode_bitstring(buf, index, NULL, NULL, NULL) < 0) + return -1; + break; case ERL_SMALL_INTEGER_EXT: case ERL_INTEGER_EXT: if (ei_decode_long(buf, index, NULL) < 0) return -1; diff --git a/lib/erl_interface/src/encode/encode_binary.c b/lib/erl_interface/src/encode/encode_binary.c index 4471c51769..0562979417 100644 --- a/lib/erl_interface/src/encode/encode_binary.c +++ b/lib/erl_interface/src/encode/encode_binary.c @@ -22,6 +22,10 @@ #include "eiext.h" #include "putget.h" +static void copy_bits(const unsigned char* src, size_t soffs, + unsigned char* dst, size_t n); + + int ei_encode_binary(char *buf, int *index, const void *p, long len) { char *s = buf + *index; @@ -40,3 +44,106 @@ int ei_encode_binary(char *buf, int *index, const void *p, long len) return 0; } +int ei_encode_bitstring(char *buf, int *index, + const char *p, + size_t bitoffs, + size_t bits) +{ + char *s = buf + *index; + char *s0 = s; + size_t bytes = (bits + 7) / 8; + char last_bits = bits % 8; + + if (!buf) s += last_bits ? 6 : 5; + else { + char* tagp = s++; + put32be(s, bytes); + if (last_bits) { + *tagp = ERL_BIT_BINARY_EXT; + put8(s, last_bits); + } + else + *tagp = ERL_BINARY_EXT; + + copy_bits((const unsigned char*)p, bitoffs, (unsigned char*)s, bits); + } + s += bytes; + + *index += s-s0; + + return 0; +} + + +/* + * MAKE_MASK(n) constructs a mask with n bits. + * Example: MAKE_MASK(3) returns the binary number 00000111. + */ +#define MAKE_MASK(n) ((((unsigned) 1) << (n))-1) + + +static +void copy_bits(const unsigned char* src, /* Base pointer to source. */ + size_t soffs, /* Bit offset for source relative to src. */ + unsigned char* dst, /* Destination. */ + size_t n) /* Number of bits to copy. */ +{ + unsigned rmask; + unsigned count; + unsigned deoffs; + unsigned bits; + unsigned bits1; + unsigned rshift; + + if (n == 0) + return; + + deoffs = n & 7; + rmask = deoffs ? (MAKE_MASK(deoffs) << (8-deoffs)) : 0; + + if (soffs == 0) { + unsigned nbytes = (n + 7) / 8; + memcpy(dst, src, nbytes); + if (rmask) + dst[nbytes-1] &= rmask; + return; + } + + src += soffs / 8; + soffs &= 7; + + if (n < 8) { /* Less than one byte */ + bits = (*src << soffs); + if (soffs+n > 8) { + src++; + bits |= (*src >> (8 - soffs)); + } + *dst = bits & rmask; + return; + } + + count = n >> 3; + + rshift = 8 - soffs; + bits = *src; + if (soffs + n > 8) { + src++; + } + + while (count--) { + bits1 = bits << soffs; + bits = *src; + src++; + *dst = bits1 | (bits >> rshift); + dst++; + } + + if (rmask) { + bits1 = bits << soffs; + if ((rmask << rshift) & 0xff) { + bits = *src; + bits1 |= (bits >> rshift); + } + *dst = bits1 & rmask; + } +} diff --git a/lib/erl_interface/src/encode/encode_fun.c b/lib/erl_interface/src/encode/encode_fun.c index 3bfc7530d1..e29424f9f4 100644 --- a/lib/erl_interface/src/encode/encode_fun.c +++ b/lib/erl_interface/src/encode/encode_fun.c @@ -26,56 +26,72 @@ int ei_encode_fun(char *buf, int *index, const erlang_fun *p) { int ix = *index; - if (p->arity == -1) { - /* ERL_FUN_EXT */ - if (buf != NULL) { - char* s = buf + ix; - put8(s, ERL_FUN_EXT); - put32be(s, p->n_free_vars); - } - ix += sizeof(char) + 4; - if (ei_encode_pid(buf, &ix, &p->pid) < 0) - return -1; - if (ei_encode_atom_as(buf, &ix, p->module, ERLANG_UTF8, p->module_org_enc) < 0) - return -1; - if (ei_encode_long(buf, &ix, p->index) < 0) - return -1; - if (ei_encode_long(buf, &ix, p->uniq) < 0) - return -1; - if (buf != NULL) - memcpy(buf + ix, p->free_vars, p->free_var_len); - ix += p->free_var_len; - } else { - char *size_p; - /* ERL_NEW_FUN_EXT */ - if (buf != NULL) { - char* s = buf + ix; - put8(s, ERL_NEW_FUN_EXT); - size_p = s; - s += 4; - put8(s, p->arity); - memcpy(s, p->md5, sizeof(p->md5)); - s += sizeof(p->md5); - put32be(s, p->index); - put32be(s, p->n_free_vars); - } else - size_p = NULL; - ix += 1 + 4 + 1 + sizeof(p->md5) + 4 + 4; - if (ei_encode_atom_as(buf, &ix, p->module, ERLANG_UTF8, p->module_org_enc) < 0) - return -1; - if (ei_encode_long(buf, &ix, p->old_index) < 0) - return -1; - if (ei_encode_long(buf, &ix, p->uniq) < 0) - return -1; - if (ei_encode_pid(buf, &ix, &p->pid) < 0) - return -1; - if (buf != NULL) - memcpy(buf + ix, p->free_vars, p->free_var_len); - ix += p->free_var_len; - if (size_p != NULL) { - int sz = buf + ix - size_p; - put32be(size_p, sz); + switch (p->type) { + case EI_FUN_CLOSURE: + if (p->arity == -1) { + /* ERL_FUN_EXT */ + if (buf != NULL) { + char* s = buf + ix; + put8(s, ERL_FUN_EXT); + put32be(s, p->u.closure.n_free_vars); + } + ix += sizeof(char) + 4; + if (ei_encode_pid(buf, &ix, &p->u.closure.pid) < 0) + return -1; + if (ei_encode_atom_as(buf, &ix, p->module, ERLANG_UTF8, ERLANG_UTF8) < 0) + return -1; + if (ei_encode_long(buf, &ix, p->u.closure.index) < 0) + return -1; + if (ei_encode_long(buf, &ix, p->u.closure.uniq) < 0) + return -1; + if (buf != NULL) + memcpy(buf + ix, p->u.closure.free_vars, p->u.closure.free_var_len); + ix += p->u.closure.free_var_len; + } else { + char *size_p; + if (buf != NULL) { + char* s = buf + ix; + put8(s, ERL_NEW_FUN_EXT); + size_p = s; + s += 4; + put8(s, p->arity); + memcpy(s, p->u.closure.md5, sizeof(p->u.closure.md5)); + s += sizeof(p->u.closure.md5); + put32be(s, p->u.closure.index); + put32be(s, p->u.closure.n_free_vars); + } else + size_p = NULL; + ix += 1 + 4 + 1 + sizeof(p->u.closure.md5) + 4 + 4; + if (ei_encode_atom_as(buf, &ix, p->module, ERLANG_UTF8, ERLANG_UTF8) < 0) + return -1; + if (ei_encode_long(buf, &ix, p->u.closure.old_index) < 0) + return -1; + if (ei_encode_long(buf, &ix, p->u.closure.uniq) < 0) + return -1; + if (ei_encode_pid(buf, &ix, &p->u.closure.pid) < 0) + return -1; + if (buf != NULL) + memcpy(buf + ix, p->u.closure.free_vars, p->u.closure.free_var_len); + ix += p->u.closure.free_var_len; + if (size_p != NULL) { + int sz = buf + ix - size_p; + put32be(size_p, sz); + } } + break; + case EI_FUN_EXPORT: + if (buf != NULL) { + char* s = buf + ix; + put8(s, ERL_EXPORT_EXT); + } + ix++; + if (ei_encode_atom_as(buf, &ix, p->module, ERLANG_UTF8, ERLANG_UTF8) < 0) + return -1; + if (ei_encode_atom_as(buf, &ix, p->u.exprt.func, ERLANG_UTF8, ERLANG_UTF8) < 0) + return -1; + if (ei_encode_long(buf, &ix, p->arity) < 0) + return -1; + break; } *index = ix; return 0; diff --git a/lib/erl_interface/src/legacy/erl_eterm.c b/lib/erl_interface/src/legacy/erl_eterm.c index 7ed2bdbc93..7ecea83b1a 100644 --- a/lib/erl_interface/src/legacy/erl_eterm.c +++ b/lib/erl_interface/src/legacy/erl_eterm.c @@ -299,12 +299,7 @@ void erl_mk_pid_helper(ETERM *ep, unsigned int number, unsigned int serial, unsigned int creation) { ERL_PID_NUMBER(ep) = number & 0x7fff; /* 15 bits */ - if (ei_internal_use_r9_pids_ports()) { - ERL_PID_SERIAL(ep) = serial & 0x07; /* 3 bits */ - } - else { - ERL_PID_SERIAL(ep) = serial & 0x1fff; /* 13 bits */ - } + ERL_PID_SERIAL(ep) = serial & 0x1fff; /* 13 bits */ ERL_PID_CREATION(ep) = creation; /* 32 bits */ } @@ -334,12 +329,7 @@ ETERM *erl_mk_port(const char *node, void erl_mk_port_helper(ETERM* ep, unsigned number, unsigned int creation) { - if (ei_internal_use_r9_pids_ports()) { - ERL_PORT_NUMBER(ep) = number & 0x3ffff; /* 18 bits */ - } - else { - ERL_PORT_NUMBER(ep) = number & 0x0fffffff; /* 18 bits */ - } + ERL_PORT_NUMBER(ep) = number & 0x0fffffff; /* 18 bits */ ERL_PORT_CREATION(ep) = creation; /* 32 bits */ } diff --git a/lib/erl_interface/src/misc/ei_compat.c b/lib/erl_interface/src/misc/ei_compat.c index 93d7dbfb83..787895992e 100644 --- a/lib/erl_interface/src/misc/ei_compat.c +++ b/lib/erl_interface/src/misc/ei_compat.c @@ -22,19 +22,22 @@ #include "ei.h" #include "ei_internal.h" -#define EI_COMPAT_NO_REL (~((unsigned) 0)) +#include <limits.h> -static unsigned compat_rel = EI_COMPAT_NO_REL; +#ifndef EI_COMPAT +# define EI_COMPAT UINT_MAX +#endif + +static unsigned compat_rel = EI_COMPAT; void ei_set_compat_rel(unsigned rel) { - if (compat_rel == EI_COMPAT_NO_REL) - compat_rel = rel; + compat_rel = rel; } -int -ei_internal_use_r9_pids_ports(void) +int ei_internal_use_21_bitstr_expfun(void) { - return compat_rel < 10; + return compat_rel < 22; } + diff --git a/lib/erl_interface/src/misc/ei_decode_term.c b/lib/erl_interface/src/misc/ei_decode_term.c index 63a7034508..8a4f7cc30d 100644 --- a/lib/erl_interface/src/misc/ei_decode_term.c +++ b/lib/erl_interface/src/misc/ei_decode_term.c @@ -87,6 +87,14 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term) case ERL_BINARY_EXT: term->size = get32be(s); return 0; + case ERL_BIT_BINARY_EXT: { + int bytes = get32be(s); + int last_bits = get8(s); + if (((last_bits==0) != (bytes==0)) || last_bits > 8) + return -1; + term->size = bytes; + return 0; + } case ERL_SMALL_BIG_EXT: if ((term->arity = get8(s)) != 4) return -1; sign = get8(s); diff --git a/lib/erl_interface/src/misc/ei_internal.h b/lib/erl_interface/src/misc/ei_internal.h index f28dd6d668..ab12597c86 100644 --- a/lib/erl_interface/src/misc/ei_internal.h +++ b/lib/erl_interface/src/misc/ei_internal.h @@ -157,7 +157,7 @@ int ei_init_connect(void); void ei_trace_printf(const char *name, int level, const char *format, ...); -int ei_internal_use_r9_pids_ports(void); +int ei_internal_use_21_bitstr_expfun(void); int ei_get_cbs_ctx__(ei_socket_callbacks **cbs, void **ctx, int fd); diff --git a/lib/erl_interface/src/misc/ei_printterm.c b/lib/erl_interface/src/misc/ei_printterm.c index 058de00de5..5c40fb7747 100644 --- a/lib/erl_interface/src/misc/ei_printterm.c +++ b/lib/erl_interface/src/misc/ei_printterm.c @@ -131,7 +131,7 @@ static int print_term(FILE* fp, ei_x_buff* x, if (fp == NULL && x == NULL) return -1; doquote = 0; - ei_get_type_internal(buf, index, &ty, &n); + ei_get_type(buf, index, &ty, &n); switch (ty) { case ERL_ATOM_EXT: case ERL_ATOM_UTF8_EXT: @@ -189,7 +189,7 @@ static int print_term(FILE* fp, ei_x_buff* x, xputs(", ", fp, x); ch_written += 2; } } - if (ei_get_type_internal(buf, &tindex, &ty, &n) < 0) goto err; + if (ei_get_type(buf, &tindex, &ty, &n) < 0) goto err; if (ty != ERL_NIL_EXT) { xputs(" | ", fp, x); ch_written += 3; r = print_term(fp, x, buf, &tindex); @@ -249,6 +249,34 @@ static int print_term(FILE* fp, ei_x_buff* x, xputc('>', fp, x); ++ch_written; ei_free(p); break; + case ERL_BIT_BINARY_EXT: { + const char* cp; + size_t bits; + unsigned int bitoffs; + int trunc = 0; + + if (ei_decode_bitstring(buf, index, &cp, &bitoffs, &bits) < 0 + || bitoffs != 0) { + goto err; + } + ch_written += xprintf(fp, x, "#Bits<"); + m = (bits+7) / 8; + if (m > BINPRINTSIZE) { + m = BINPRINTSIZE; + trunc = 1; + } + --m; + for (i = 0; i < m; ++i) { + ch_written += xprintf(fp, x, "%d,", cp[i]); + } + ch_written += xprintf(fp, x, "%d", cp[i]); + if (trunc) + ch_written += xprintf(fp, x, ",..."); + else if (bits % 8 != 0) + ch_written += xprintf(fp, x, ":%u", (unsigned)(bits % 8)); + xputc('>', fp, x); ++ch_written; + break; + } case ERL_SMALL_INTEGER_EXT: case ERL_INTEGER_EXT: if (ei_decode_long(buf, index, &l) < 0) goto err; diff --git a/lib/erl_interface/src/misc/ei_pthreads.c b/lib/erl_interface/src/misc/ei_pthreads.c index ec1c8d956f..c6d07a9a0a 100644 --- a/lib/erl_interface/src/misc/ei_pthreads.c +++ b/lib/erl_interface/src/misc/ei_pthreads.c @@ -78,7 +78,7 @@ static void tls_init_once(void) errno_tls_index = TlsAlloc(); if (errno_tls_index == TLS_OUT_OF_INDEXES) { fprintf(stderr, - "FATAL ERROR: can not allocate TLS index for " + "FATAL ERROR: cannot allocate TLS index for " "erl_errno (error code = %d)!\n",GetLastError()); exit(1); } diff --git a/lib/erl_interface/src/misc/ei_x_encode.c b/lib/erl_interface/src/misc/ei_x_encode.c index 4ff5974663..8e77679d2a 100644 --- a/lib/erl_interface/src/misc/ei_x_encode.c +++ b/lib/erl_interface/src/misc/ei_x_encode.c @@ -117,6 +117,16 @@ int ei_x_encode_binary(ei_x_buff* x, const void* p, int len) return ei_encode_binary(x->buff, &x->index, p, len); } +int ei_x_encode_bitstring(ei_x_buff* x, const char* p, size_t bitoffs, size_t bits) +{ + int i = x->index; + if (ei_encode_bitstring(NULL, &i, p, bitoffs, bits) == -1) + return -1; + if (!x_fix_buff(x, i)) + return -1; + return ei_encode_bitstring(x->buff, &x->index, p, bitoffs, bits); +} + int ei_x_encode_long(ei_x_buff* x, long n) { int i = x->index; diff --git a/lib/erl_interface/src/misc/get_type.c b/lib/erl_interface/src/misc/get_type.c index aa69cd4d60..eef58a9363 100644 --- a/lib/erl_interface/src/misc/get_type.c +++ b/lib/erl_interface/src/misc/get_type.c @@ -27,17 +27,8 @@ /* for types with meaningful length attributes, return the length too. In other cases, return length 0 */ -/* FIXME working on this one.... */ - int ei_get_type(const char *buf, const int *index, int *type, int *len) { - return ei_get_type_internal(buf, index, type, len); -} - - -int ei_get_type_internal(const char *buf, const int *index, - int *type, int *len) -{ const char *s = buf + *index; *type = get8(s); @@ -64,7 +55,9 @@ int ei_get_type_internal(const char *buf, const int *index, case ERL_LARGE_TUPLE_EXT: case ERL_LIST_EXT: + case ERL_MAP_EXT: case ERL_BINARY_EXT: + case ERL_BIT_BINARY_EXT: *len = get32be(s); break; diff --git a/lib/erl_interface/src/misc/show_msg.c b/lib/erl_interface/src/misc/show_msg.c index 5868cccba6..805d69e9b3 100644 --- a/lib/erl_interface/src/misc/show_msg.c +++ b/lib/erl_interface/src/misc/show_msg.c @@ -342,7 +342,7 @@ static void show_term(const char *termbuf, int *index, FILE *stream) int i, len; char *s; - ei_get_type_internal(termbuf,index,&type,&len); + ei_get_type(termbuf,index,&type,&len); switch (type) { case ERL_VERSION_MAGIC: @@ -455,6 +455,12 @@ static void show_term(const char *termbuf, int *index, FILE *stream) fprintf(stream,"#Bin<%ld>",num); break; + case ERL_BIT_BINARY_EXT: { + size_t bits; + ei_decode_bitstring(termbuf, index, NULL, NULL, &bits); + fprintf(stream, "#Bits<%lu>", (unsigned long)bits); + break; + } case ERL_LARGE_BIG_EXT: /* doesn't actually decode - just skip over it */ /* FIXME if GMP, what to do here?? */ diff --git a/lib/erl_interface/src/prog/ei_fake_prog.c b/lib/erl_interface/src/prog/ei_fake_prog.c index 158464b385..6f58c9833d 100644 --- a/lib/erl_interface/src/prog/ei_fake_prog.c +++ b/lib/erl_interface/src/prog/ei_fake_prog.c @@ -186,7 +186,6 @@ int main(void) ei_x_encode_empty_list(&eix); ei_get_type(charp, intp, intp, intp); - ei_get_type_internal(charp, intp, intp, intp); ei_decode_version(charp, intp, intp); ei_decode_long(charp, intp, longp); diff --git a/lib/erl_interface/src/prog/erl_start.c b/lib/erl_interface/src/prog/erl_start.c index c766f4780e..b7aa451946 100644 --- a/lib/erl_interface/src/prog/erl_start.c +++ b/lib/erl_interface/src/prog/erl_start.c @@ -97,7 +97,7 @@ #endif #ifndef RSH -#define RSH "/usr/bin/rsh" +#define RSH "/usr/bin/ssh" #endif #ifndef HAVE_SOCKLEN_T diff --git a/lib/erl_interface/src/registry/reg_dump.c b/lib/erl_interface/src/registry/reg_dump.c index 43c9824433..da0413e6e6 100644 --- a/lib/erl_interface/src/registry/reg_dump.c +++ b/lib/erl_interface/src/registry/reg_dump.c @@ -90,7 +90,7 @@ static int mn_start_dump(int fd, const erlang_pid *self, || (arity != 2) || ei_decode_atom(buf,&index,tmpbuf) || strcmp(tmpbuf,"rex") - || ei_get_type_internal(buf,&index,&type,&arity) + || ei_get_type(buf,&index,&type,&arity) || (type != ERL_PID_EXT)) return -1; /* bad response from other side */ diff --git a/lib/erl_interface/test/all_SUITE_data/ei_runner.h b/lib/erl_interface/test/all_SUITE_data/ei_runner.h index 2608661303..7c874ac82e 100644 --- a/lib/erl_interface/test/all_SUITE_data/ei_runner.h +++ b/lib/erl_interface/test/all_SUITE_data/ei_runner.h @@ -53,6 +53,7 @@ void free_packet(char*); #define fail(reason) do_fail(__FILE__, __LINE__, reason) #define fail1(reason, a1) do_fail(__FILE__, __LINE__, reason, a1) +#define fail2(reason, a1, a2) do_fail(__FILE__, __LINE__, reason, a1, a2) #define report(ok) do_report(__FILE__, __LINE__, ok) void do_report(char* file, int line, int ok); diff --git a/lib/erl_interface/test/ei_accept_SUITE.erl b/lib/erl_interface/test/ei_accept_SUITE.erl index 9c9c3f86b6..f40c67375b 100644 --- a/lib/erl_interface/test/ei_accept_SUITE.erl +++ b/lib/erl_interface/test/ei_accept_SUITE.erl @@ -43,8 +43,12 @@ init_per_testcase(Case, Config) -> runner:init_per_testcase(?MODULE, Case, Config). ei_accept(Config) when is_list(Config) -> + ei_accept_do(Config, 0), % default + ei_accept_do(Config, 21). % ei_set_compat_rel + +ei_accept_do(Config, CompatRel) -> P = runner:start(Config, ?interpret), - 0 = ei_connect_init(P, 42, erlang:get_cookie(), 0), + 0 = ei_connect_init(P, 42, erlang:get_cookie(), 0, CompatRel), Myname = hd(tl(string:tokens(atom_to_list(node()), "@"))), io:format("Myname ~p ~n", [Myname]), @@ -52,15 +56,18 @@ ei_accept(Config) when is_list(Config) -> io:format("EINode ~p ~n", [EINode]), %% We take this opportunity to also test export-funs and bit-strings - %% with (ugly) tuple fallbacks. + %% with (ugly) tuple fallbacks in OTP 21 and older. %% Test both toward pending connection and established connection. RealTerms = [<<1:1>>, fun lists:map/2], - Fallbacks = [{<<128>>,1}, {lists,map}], + EncTerms = case CompatRel of + 0 -> RealTerms; + 21 -> [{<<128>>,1}, {lists,map}] + end, Self = self(), Funny = fun() -> hello end, TermToSend = {call, Self, "Test", Funny, RealTerms}, - TermToGet = {call, Self, "Test", Funny, Fallbacks}, + TermToGet = {call, Self, "Test", Funny, EncTerms}, Port = 6543, {ok, ListenFd} = ei_publish(P, Port), {any, EINode} ! TermToSend, @@ -94,7 +101,7 @@ ei_threaded_accept(Config) when is_list(Config) -> %% Test erlang:monitor toward erl_interface "processes" monitor_ei_process(Config) when is_list(Config) -> P = runner:start(Config, ?interpret), - 0 = ei_connect_init(P, 42, erlang:get_cookie(), 0), + 0 = ei_connect_init(P, 42, erlang:get_cookie(), 0, 0), Myname = hd(tl(string:tokens(atom_to_list(node()), "@"))), io:format("Myname ~p ~n", [Myname]), @@ -167,8 +174,8 @@ start_einode(Einode, N, Host) -> %%% Interface functions for ei (erl_interface) functions. -ei_connect_init(P, Num, Cookie, Creation) -> - send_command(P, ei_connect_init, [Num,Cookie,Creation]), +ei_connect_init(P, Num, Cookie, Creation, Compat) -> + send_command(P, ei_connect_init, [Num,Cookie,Creation,Compat]), case get_term(P) of {term,Int} when is_integer(Int) -> Int end. diff --git a/lib/erl_interface/test/ei_accept_SUITE_data/ei_accept_test.c b/lib/erl_interface/test/ei_accept_SUITE_data/ei_accept_test.c index c209f506b1..09b0b5440b 100644 --- a/lib/erl_interface/test/ei_accept_SUITE_data/ei_accept_test.c +++ b/lib/erl_interface/test/ei_accept_SUITE_data/ei_accept_test.c @@ -58,7 +58,7 @@ static struct { int num_args; /* Number of arguments. */ void (*func)(char* buf, int len); } commands[] = { - "ei_connect_init", 3, cmd_ei_connect_init, + "ei_connect_init", 4, cmd_ei_connect_init, "ei_publish", 1, cmd_ei_publish, "ei_accept", 1, cmd_ei_accept, "ei_receive", 1, cmd_ei_receive, @@ -106,21 +106,25 @@ TESTCASE(interpret) static void cmd_ei_connect_init(char* buf, int len) { int index = 0, r = 0; - int type, size; - long l; - char b[100]; + long num, creation; + unsigned long compat; + char node_name[100]; char cookie[MAXATOMLEN], * cp = cookie; ei_x_buff res; - if (ei_decode_long(buf, &index, &l) < 0) + if (ei_decode_long(buf, &index, &num) < 0) fail("expected int"); - sprintf(b, "c%d", l); - /* FIXME don't use internal and maybe use skip?! */ - ei_get_type_internal(buf, &index, &type, &size); + sprintf(node_name, "c%d", num); if (ei_decode_atom(buf, &index, cookie) < 0) fail("expected atom (cookie)"); if (cookie[0] == '\0') cp = NULL; - r = ei_connect_init(&ec, b, cp, 0); + if (ei_decode_long(buf, &index, &creation) < 0) + fail("expected int"); + if (ei_decode_long(buf, &index, &compat) < 0) + fail("expected uint"); + if (compat) + ei_set_compat_rel(compat); + r = ei_connect_init(&ec, node_name, cp, creation); ei_x_new_with_version(&res); ei_x_encode_long(&res, r); send_bin_term(&res); diff --git a/lib/erl_interface/test/ei_connect_SUITE.erl b/lib/erl_interface/test/ei_connect_SUITE.erl index 75b6bf18da..6184ce801b 100644 --- a/lib/erl_interface/test/ei_connect_SUITE.erl +++ b/lib/erl_interface/test/ei_connect_SUITE.erl @@ -79,9 +79,10 @@ ei_send_funs(Config) when is_list(Config) -> {ok,Fd} = ei_connect(P, node()), Fun1 = fun ei_send/1, - Fun2 = fun(X) -> P, X, Fd, Fun1 end, + Fun2 = fun(X) -> {P, X, Fd, Fun1} end, + Bits = <<1,2,3:5>>, - AMsg={Fun1,Fun2}, + AMsg={Fun1,Fun2,Bits}, %%AMsg={wait_with_funs, new_dist_format}, ok = ei_send_funs(P, Fd, self(), AMsg), EIMsg = receive M -> M end, diff --git a/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c b/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c index 58c0c7f8d8..385bcdd422 100644 --- a/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c +++ b/lib/erl_interface/test/ei_connect_SUITE_data/ei_connect_test.c @@ -107,7 +107,6 @@ TESTCASE(interpret) static void cmd_ei_connect_init(char* buf, int len) { int index = 0, r = 0; - int type, size; long l; char b[100]; char cookie[MAXATOMLEN], * cp = cookie; @@ -115,8 +114,6 @@ static void cmd_ei_connect_init(char* buf, int len) if (ei_decode_long(buf, &index, &l) < 0) fail("expected int"); sprintf(b, "c%ld", l); - /* FIXME don't use internal and maybe use skip?! */ - ei_get_type_internal(buf, &index, &type, &size); if (ei_decode_atom(buf, &index, cookie) < 0) fail("expected atom (cookie)"); if (cookie[0] == '\0') @@ -212,6 +209,9 @@ static void cmd_ei_send_funs(char* buf, int len) erlang_pid pid; ei_x_buff x; erlang_fun fun1, fun2; + char* bitstring; + size_t bits; + int bitoffs; if (ei_decode_long(buf, &index, &fd) < 0) fail("expected long"); @@ -219,20 +219,24 @@ static void cmd_ei_send_funs(char* buf, int len) fail("expected pid (node)"); if (ei_decode_tuple_header(buf, &index, &n) < 0) fail("expected tuple"); - if (n != 2) + if (n != 3) fail("expected tuple"); if (ei_decode_fun(buf, &index, &fun1) < 0) fail("expected Fun1"); if (ei_decode_fun(buf, &index, &fun2) < 0) fail("expected Fun2"); + if (ei_decode_bitstring(buf, &index, &bitstring, &bitoffs, &bits) < 0) + fail("expected bitstring"); if (ei_x_new_with_version(&x) < 0) fail("ei_x_new_with_version"); - if (ei_x_encode_tuple_header(&x, 2) < 0) + if (ei_x_encode_tuple_header(&x, 3) < 0) fail("encode tuple header"); if (ei_x_encode_fun(&x, &fun1) < 0) fail("encode fun1"); if (ei_x_encode_fun(&x, &fun2) < 0) fail("encode fun2"); + if (ei_x_encode_bitstring(&x, bitstring, bitoffs, bits) < 0) + fail("encode bitstring"); free_fun(&fun1); free_fun(&fun2); send_errno_result(ei_send(fd, &pid, x.buff, x.index)); diff --git a/lib/erl_interface/test/ei_decode_SUITE.erl b/lib/erl_interface/test/ei_decode_SUITE.erl index 75560ea7c9..e005ec89c7 100644 --- a/lib/erl_interface/test/ei_decode_SUITE.erl +++ b/lib/erl_interface/test/ei_decode_SUITE.erl @@ -194,6 +194,9 @@ test_ei_decode_misc(Config) when is_list(Config) -> send_term_as_binary(P,<<>>), send_term_as_binary(P,<<"ÅÄÖåäö">>), + send_term_as_binary(P,<<1, 2, 3:5>>), + send_term_as_binary(P,<<1:1>>), + % send_term_as_binary(P,{}), % send_term_as_binary(P,[]), diff --git a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c index e516f310b6..46d6b8f2af 100644 --- a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c +++ b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c @@ -256,66 +256,134 @@ int ei_decode_my_string(const char *buf, int *index, char *to, //#define EI_DECODE_UTF8_STRING(FUNC,SIZE,VAL) -#define EI_DECODE_BIN(FUNC,SIZE,VAL,LEN) \ - { \ - char p[1024]; \ - char *buf; \ - long len; \ - int size1 = 0; \ - int size2 = 0; \ - int err; \ - message("ei_" #FUNC " should be " #VAL); \ - buf = read_packet(NULL); \ - err = ei_ ## FUNC(buf+1, &size1, NULL, &len); \ +static void decode_bin(int exp_size, const char* val, int exp_len) +{ + char p[1024]; + char *buf; + long len; + int size1 = 0; + int size2 = 0; + int err; + message("ei_decode_binary should be %s", val); + buf = read_packet(NULL); + err = ei_decode_binary(buf+1, &size1, NULL, &len); message("err = %d, size = %d, len = %d, expected size = %d, expected len = %d\n",\ - err,size1,len,SIZE,LEN); \ - if (err != 0) { \ - if (err != -1) { \ - fail("returned non zero but not -1 if NULL pointer"); \ - } else { \ - fail("returned non zero"); \ - } \ - return; \ - } \ -\ - if (len != LEN) { \ - fail("size is not correct"); \ - return; \ - } \ -\ - err = ei_ ## FUNC(buf+1, &size2, p, &len); \ + err,size1,len, exp_size, exp_len); + if (err != 0) { + if (err != -1) { + fail("returned non zero but not -1 if NULL pointer"); + } else { + fail("returned non zero"); + } + return; + } + + if (len != exp_len) { + fail("size is not correct"); + return; + } + + err = ei_decode_binary(buf+1, &size2, p, &len); message("err = %d, size = %d, len = %d, expected size = %d, expected len = %d\n",\ - err,size2,len,SIZE,LEN); \ - if (err != 0) { \ - if (err != -1) { \ - fail("returned non zero but not -1 if NULL pointer"); \ - } else { \ - fail("returned non zero"); \ - } \ - return; \ - } \ -\ - if (len != LEN) { \ - fail("size is not correct"); \ - return; \ - } \ -\ - if (strncmp(p,VAL,LEN) != 0) { \ - fail("value is not correct"); \ - return; \ - } \ -\ - if (size1 != size2) { \ - fail("size with and without pointer differs"); \ - return; \ - } \ -\ - if (size1 != SIZE) { \ - fail("size of encoded data is incorrect"); \ - return; \ - } \ - free_packet(buf); \ - } \ + err,size2,len, exp_size, exp_len); + if (err != 0) { + if (err != -1) { + fail("returned non zero but not -1 if NULL pointer"); + } else { + fail("returned non zero"); + } + return; + } + + if (len != exp_len) { + fail("size is not correct"); + return; + } + + if (strncmp(p,val,exp_len) != 0) { + fail("value is not correct"); + return; + } + + if (size1 != size2) { + fail("size with and without pointer differs"); + return; + } + + if (size1 != exp_size) { + fail("size of encoded data is incorrect"); + return; + } + free_packet(buf); +} + +static void decode_bits(int exp_size, const char* val, size_t exp_bits) +{ + const char* p; + char *buf; + size_t bits; + int bitoffs; + int size1 = 0; + int size2 = 0; + int err; + message("ei_decode_bitstring should be %d bits", (int)exp_bits); + buf = read_packet(NULL); + err = ei_decode_bitstring(buf+1, &size1, NULL, &bitoffs, &bits); + message("err = %d, size = %d, bitoffs = %d, bits = %d, expected size = %d, expected bits = %d\n",\ + err,size1, bitoffs, (int)bits, exp_size, (int)exp_bits); + + if (err != 0) { + if (err != -1) { + fail("returned non zero but not -1 if NULL pointer"); + } else { + fail("returned non zero"); + } + return; + } + + if (bits != exp_bits) { + fail("number of bits is not correct"); + return; + } + if (bitoffs != 0) { + fail("non zero bit offset"); + return; + } + + err = ei_decode_bitstring(buf+1, &size2, &p, NULL, &bits); + message("err = %d, size = %d, len = %d, expected size = %d, expected len = %d\n",\ + err,size2, (int)bits, exp_size, (int)exp_bits); + if (err != 0) { + if (err != -1) { + fail("returned non zero but not -1 if NULL pointer"); + } else { + fail("returned non zero"); + } + return; + } + + if (bits != exp_bits) { + fail("bits is not correct"); + return; + } + + if (memcmp(p, val, (exp_bits+7)/8) != 0) { + fail("value is not correct"); + return; + } + + if (size1 != size2) { + fail("size with and without pointer differs"); + return; + } + + if (size1 != exp_size) { + fail2("size of encoded data is incorrect %d != %d", size1, exp_size); + return; + } + free_packet(buf); +} + /* ******************************************************************** */ @@ -644,9 +712,17 @@ TESTCASE(test_ei_decode_misc) EI_DECODE_STRING(decode_my_string, 1, ""); EI_DECODE_STRING(decode_my_string, 9, "������"); - EI_DECODE_BIN(decode_binary, 8, "foo", 3); - EI_DECODE_BIN(decode_binary, 5, "", 0); - EI_DECODE_BIN(decode_binary, 11, "������", 6); + decode_bin(8, "foo", 3); + decode_bin(5, "", 0); + decode_bin(11, "������", 6); + +#define LAST_BYTE(V, BITS) ((V) << (8-(BITS))) + { + unsigned char bits1[] = {1, 2, LAST_BYTE(3,5) }; + unsigned char bits2[] = {LAST_BYTE(1,1) }; + decode_bits(9, bits1, 21); + decode_bits(7, bits2, 1); + } /* FIXME check \0 in strings and atoms? */ /* diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE.erl b/lib/erl_interface/test/ei_decode_encode_SUITE.erl index 0f23cdfbb9..3451d9f503 100644 --- a/lib/erl_interface/test/ei_decode_encode_SUITE.erl +++ b/lib/erl_interface/test/ei_decode_encode_SUITE.erl @@ -120,9 +120,31 @@ test_ei_decode_encode(Config) when is_list(Config) -> send_rec(P, #{key => value}), send_rec(P, maps:put(Port, Ref, #{key => value, key2 => Pid})), + [send_rec(P, <<16#dec0deb175:B/little>>) || B <- lists:seq(0,48)], + + % And last an ugly duckling to test ei_encode_bitstring with bitoffs != 0 + encode_bitstring(P), + runner:recv_eot(P), ok. +encode_bitstring(P) -> + %% Send one bitstring to c-node + Bits = <<16#18f6d4b2907e5c3a1:66>>, + P ! {self(), {command, term_to_binary(Bits, [{minor_version, 2}])}}, + + %% and then receive and verify a number of different sub-bitstrings + receive_sub_bitstring(P, Bits, 0, bit_size(Bits)). + +receive_sub_bitstring(_, _, _, NBits) when NBits < 0 -> + ok; +receive_sub_bitstring(P, Bits, BitOffs, NBits) -> + <<_:BitOffs, Sub:NBits/bits, _/bits>> = Bits, + %%io:format("expecting term_to_binary(~p) = ~p\n", [Sub, term_to_binary(Sub)]), + {_B,Sub} = get_buf_and_term(P), + receive_sub_bitstring(P, Bits, BitOffs+1, NBits - ((NBits div 20)+1)). + + %% ######################################################################## %% diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c index 55d9ed1b1a..85ca6c56e9 100644 --- a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c +++ b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c @@ -40,6 +40,13 @@ typedef struct erlang_char_encoding enc; }my_atom; +typedef struct +{ + const char* bytes; + unsigned int bitoffs; + size_t nbits; +}my_bitstring; + struct my_obj { union { erlang_fun fun; @@ -49,6 +56,7 @@ struct my_obj { erlang_trace trace; erlang_big big; my_atom atom; + my_bitstring bits; int arity; }u; @@ -119,6 +127,26 @@ struct Type my_atom_type = { (encodeFT*)ei_encode_my_atom, (x_encodeFT*)ei_x_encode_my_atom }; +int ei_decode_my_bits(const char *buf, int *index, my_bitstring* a) +{ + return ei_decode_bitstring(buf, index, (a ? &a->bytes : NULL), + (a ? &a->bitoffs : NULL), + (a ? &a->nbits : NULL)); +} +int ei_encode_my_bits(char *buf, int *index, my_bitstring* a) +{ + return ei_encode_bitstring(buf, index, a->bytes, a->bitoffs, a->nbits); +} +int ei_x_encode_my_bits(ei_x_buff* x, my_bitstring* a) +{ + return ei_x_encode_bitstring(x, a->bytes, a->bitoffs, a->nbits); +} + +struct Type my_bitstring_type = { + "bits", "my_bitstring", (decodeFT*)ei_decode_my_bits, + (encodeFT*)ei_encode_my_bits, (x_encodeFT*)ei_x_encode_my_bits +}; + int my_decode_tuple_header(const char *buf, int *index, struct my_obj* obj) { @@ -237,11 +265,7 @@ void decode_encode(struct Type** tv, int nobj) size1 = 0; err = t->ei_decode_fp(inp, &size1, NULL); if (err != 0) { - if (err != -1) { - fail("decode returned non zero but not -1"); - } else { - fail1("decode '%s' returned non zero", t->name); - } + fail2("decode '%s' returned non zero %d", t->name, err); return; } if (size1 < 1) { @@ -470,6 +494,66 @@ void decode_encode_big(struct Type* t) } +void encode_bitstring(void) +{ + char* packet; + char* inp; + char out_buf[BUFSZ]; + int size; + int err, i; + ei_x_buff arg; + const char* p; + unsigned int bitoffs; + size_t nbits, org_nbits; + + packet = read_packet(NULL); + inp = packet+1; + + size = 0; + err = ei_decode_bitstring(inp, &size, &p, &bitoffs, &nbits); + if (err != 0) { + fail1("ei_decode_bitstring returned non zero %d", err); + return; + } + + /* + * Now send a bunch of different sub-bitstrings back + * encoded both with ei_encode_ and ei_x_encode_. + */ + org_nbits = nbits; + do { + size = 0; + err = ei_encode_bitstring(out_buf, &size, p, bitoffs, nbits); + if (err != 0) { + fail1("ei_encode_bitstring returned non zero %d", err); + return; + } + + ei_x_new(&arg); + err = ei_x_encode_bitstring(&arg, p, bitoffs, nbits); + if (err != 0) { + fail1("ei_x_encode_bitstring returned non zero %d", err); + ei_x_free(&arg); + return; + } + + if (arg.index < 1) { + fail("size is < 1"); + ei_x_free(&arg); + return; + } + + send_buffer(out_buf, size); + send_buffer(arg.buff, arg.index); + ei_x_free(&arg); + + bitoffs++; + nbits -= (nbits / 20) + 1; + } while (nbits < org_nbits); + + free_packet(packet); +} + /* ******************************************************************** */ @@ -537,6 +621,12 @@ TESTCASE(test_ei_decode_encode) decode_encode(map, 7); } + for (i=0; i <= 48; i++) { + decode_encode_one(&my_bitstring_type); + } + + encode_bitstring(); + report(1); } diff --git a/lib/hipe/cerl/Makefile b/lib/hipe/cerl/Makefile index f653dce36f..5c367b5b77 100644 --- a/lib/hipe/cerl/Makefile +++ b/lib/hipe/cerl/Makefile @@ -66,7 +66,10 @@ DOC_FILES= $(MODULES:%=$(DOCS)/%.html) include ../native.mk -ERL_COMPILE_FLAGS += +inline -Werror +warn_export_vars +warn_unused_import +warn_missing_spec #+warn_untyped_record +ERL_COMPILE_FLAGS += +inline +warn_export_vars +warn_unused_import +warn_missing_spec #+warn_untyped_record +ifneq ($(NATIVE_LIBS_ENABLED),yes) +ERL_COMPILE_FLAGS += -Werror +endif # ---------------------------------------------------- # Targets diff --git a/lib/hipe/doc/src/hipe_app.xml b/lib/hipe/doc/src/hipe_app.xml index 63bc6ea2d7..61d92fdffe 100644 --- a/lib/hipe/doc/src/hipe_app.xml +++ b/lib/hipe/doc/src/hipe_app.xml @@ -62,6 +62,11 @@ and the runtime system that have limited or no support for HiPE compiled modules. </p> <taglist> + <tag>Binary matching</tag> + <item><p>The HiPE compiler will crash on modules containing binary + matching.</p> + </item> + <tag>Stack traces</tag> <item><p>Stack traces returned from <seealso marker="erts:erlang#get_stacktrace/0"> <c>erlang:get_stacktrace/0</c></seealso> or as part of <c>'EXIT'</c> terms @@ -78,12 +83,12 @@ </item> <tag>NIFs</tag> - <item><p>Modules compiled with HiPE can not call <seealso marker="erts:erlang#load_nif-2"> + <item><p>Modules compiled with HiPE cannot call <seealso marker="erts:erlang#load_nif-2"> <c>erlang:load_nif/2</c></seealso> to load NIFs.</p> </item> <tag>-on_load</tag> - <item><p>Modules compiled with HiPE can not use + <item><p>Modules compiled with HiPE cannot use <seealso marker="doc/reference_manual:code_loading#on_load"><c>-on_load()</c></seealso> directives.</p> </item> diff --git a/lib/hipe/icode/hipe_beam_to_icode.erl b/lib/hipe/icode/hipe_beam_to_icode.erl index f429d40272..8e7e56b6c4 100644 --- a/lib/hipe/icode/hipe_beam_to_icode.erl +++ b/lib/hipe/icode/hipe_beam_to_icode.erl @@ -647,6 +647,13 @@ trans_fun([{put_tuple,_Size,Reg}|Instructions], Env) -> Primop = hipe_icode:mk_primop(Dest,mktuple,Src), Moves ++ [Primop | trans_fun(Instructions2,Env2)]; %%--- put --- SHOULD NOT REALLY EXIST HERE; put INSTRUCTIONS ARE HANDLED ABOVE. +%%--- put_tuple2 --- +trans_fun([{put_tuple2,Reg,{list,Elements}}|Instructions], Env) -> + Dest = [mk_var(Reg)], + {Moves,Vars,Env2} = trans_elements(Elements, [], [], Env), + Src = lists:reverse(Vars), + Primop = hipe_icode:mk_primop(Dest, mktuple, Src), + Moves ++ [Primop | trans_fun(Instructions, Env2)]; %%--- badmatch --- trans_fun([{badmatch,Arg}|Instructions], Env) -> BadVar = trans_arg(Arg), @@ -1139,9 +1146,10 @@ trans_fun([{test,has_map_fields,{f,Lbl},Map,{list,Keys}}|Instructions], Env) -> lists:flatten([[K, {r, 0}] || K <- Keys])), [MapMove, TestInstructions | trans_fun(Instructions, Env2)]; trans_fun([{get_map_elements,{f,Lbl},Map,{list,KVPs}}|Instructions], Env) -> + KVPs1 = overwrite_map_last(Map, KVPs), {MapMove, MapVar, Env1} = mk_move_and_var(Map, Env), {TestInstructions, GetInstructions, Env2} = - trans_map_query(MapVar, map_label(Lbl), Env1, KVPs), + trans_map_query(MapVar, map_label(Lbl), Env1, KVPs1), [MapMove, TestInstructions, GetInstructions | trans_fun(Instructions, Env2)]; %%--- put_map_assoc --- trans_fun([{put_map_assoc,{f,Lbl},Map,Dst,_N,{list,Pairs}}|Instructions], Env) -> @@ -1181,6 +1189,21 @@ trans_fun([raw_raise|Instructions], Env) -> [hipe_icode:mk_primop(Dst,raw_raise,Vars) | trans_fun(Instructions, Env)]; %%-------------------------------------------------------------------- +%% New binary matching added in OTP 22. +%%-------------------------------------------------------------------- +%%--- bs_get_tail --- +trans_fun([{bs_get_tail=Name,_,_,_}|_Instructions], _Env) -> + nyi(Name); +%%--- bs_start_match3 --- +trans_fun([{bs_start_match3=Name,_,_,_,_}|_Instructions], _Env) -> + nyi(Name); +%%--- bs_get_position --- +trans_fun([{bs_get_position=Name,_,_,_}|_Instructions], _Env) -> + nyi(Name); +%%--- bs_set_position --- +trans_fun([{bs_set_position=Name,_,_}|_Instructions], _Env) -> + nyi(Name); +%%-------------------------------------------------------------------- %%--- ERROR HANDLING --- %%-------------------------------------------------------------------- trans_fun([X|_], _) -> @@ -1188,6 +1211,9 @@ trans_fun([X|_], _) -> trans_fun([], _) -> []. +nyi(Name) -> + throw({unimplemented_instruction,Name}). + %%-------------------------------------------------------------------- %% trans_call and trans_enter generate correct Icode calls/tail-calls, %% recognizing explicit fails. @@ -1563,6 +1589,21 @@ trans_type_test2(function2, Lbl, Arg, Arity, Env) -> hipe_icode:label_name(True), map_label(Lbl)), {[Move1,Move2,I,True],Env2}. + +%% +%% Makes sure that if a get_map_elements instruction will overwrite +%% the map source, it will be done last. +%% +overwrite_map_last(Map, KVPs) -> + overwrite_map_last2(Map, KVPs, []). + +overwrite_map_last2(Map, [Key,Map|KVPs], _Last) -> + overwrite_map_last2(Map, KVPs, [Key,Map]); +overwrite_map_last2(Map, [Key,Val|KVPs], Last) -> + [Key,Val|overwrite_map_last2(Map, KVPs, Last)]; +overwrite_map_last2(_Map, [], Last) -> + Last. + %% %% Handles the get_map_elements instruction and the has_map_fields %% test instruction. @@ -1683,6 +1724,19 @@ trans_puts([{put,X}|Code], Vars, Moves, Env) -> trans_puts(Code, Vars, Moves, Env) -> %% No more put operations {Moves, Code, Vars, Env}. +trans_elements([X|Code], Vars, Moves, Env) -> + case type(X) of + var -> + Var = mk_var(X), + trans_elements(Code, [Var|Vars], Moves, Env); + #beam_const{value=C} -> + Var = mk_var(new), + Move = hipe_icode:mk_move(Var, hipe_icode:mk_const(C)), + trans_elements(Code, [Var|Vars], [Move|Moves], Env) + end; +trans_elements([], Vars, Moves, Env) -> + {Moves, Vars, Env}. + %%----------------------------------------------------------------------- %% The code for this instruction is a bit large because we are treating %% different cases differently. We want to use the icode `type' diff --git a/lib/hipe/icode/hipe_icode_range.erl b/lib/hipe/icode/hipe_icode_range.erl index 34b18acccd..098a7a8d8c 100644 --- a/lib/hipe/icode/hipe_icode_range.erl +++ b/lib/hipe/icode/hipe_icode_range.erl @@ -1633,7 +1633,7 @@ inf_bsl(_, pos_inf) -> neg_inf; inf_bsl(Number, neg_inf) when is_integer(Number), Number >= 0 -> 0; inf_bsl(_Number, neg_inf) -> -1; inf_bsl(Number1, Number2) when is_integer(Number1), is_integer(Number2) -> - %% We can not shift left with a number which is not a fixnum. We + %% We cannot shift left with a number which is not a fixnum. We %% don't have enough memory. Bits = ?BITS, if Number2 > (Bits bsl 1) -> inf_bsl(Number1, pos_inf); diff --git a/lib/hipe/llvm/Makefile b/lib/hipe/llvm/Makefile index 817ff67dcd..9f7a2def6d 100644 --- a/lib/hipe/llvm/Makefile +++ b/lib/hipe/llvm/Makefile @@ -70,7 +70,10 @@ TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) include ../native.mk -ERL_COMPILE_FLAGS += -Werror +inline +warn_export_vars #+warn_missing_spec +ERL_COMPILE_FLAGS += +inline +warn_export_vars #+warn_missing_spec +ifneq ($(NATIVE_LIBS_ENABLED),yes) +ERL_COMPILE_FLAGS += -Werror +endif # if in 32 bit backend define BIT32 symbol ifneq ($(BITS64),yes) diff --git a/lib/hipe/llvm/hipe_llvm_main.erl b/lib/hipe/llvm/hipe_llvm_main.erl index 54c435c127..44f0566379 100644 --- a/lib/hipe/llvm/hipe_llvm_main.erl +++ b/lib/hipe/llvm/hipe_llvm_main.erl @@ -526,8 +526,8 @@ unique_folder(FunName, Arity, Options) -> case proplists:get_bool(llvm_save_temps, Options) of true -> %% Store folder in current directory DirName; - false -> %% Temporarily store folder in tempfs (/dev/shm/) - "/dev/shm/" ++ DirName + false -> %% Temporarily store folder in tempfs or tmp dir + tmpfs_folder() ++ DirName end, %% Make sure it does not exist case dir_exists(Dir) of @@ -537,6 +537,14 @@ unique_folder(FunName, Arity, Options) -> Dir end. +tmpfs_folder() -> + case os:type() of + {unix, linux} -> + "/dev/shm/"; + {unix, _} -> %% Fallback to tmp dir. e.g. FreeBSD + "/tmp/" + end. + %% @doc Function that checks that a given Filename is an existing Directory %% Name (from http://rosettacode.org/wiki/Ensure_that_a_file_exists#Erlang) dir_exists(Filename) -> diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl index e2cb9c0f0b..094b7bc508 100644 --- a/lib/hipe/main/hipe.erl +++ b/lib/hipe/main/hipe.erl @@ -196,7 +196,7 @@ file/1, file/2, get_llvm_version/0, - llvm_support_available/0, + erllvm_is_supported/0, load/1, help/0, help_hiper/0, @@ -218,12 +218,11 @@ %% Basic type declaration for exported functions of the 'hipe' module %%------------------------------------------------------------------- --type mod() :: atom(). --type f_unit() :: mod() | binary(). +-type mod() :: module(). +-type file_or_bin() :: file:filename() | binary(). -type ret_rtl() :: [_]. -type c_ret() :: {'ok', mod()} | {'error', term()} | {'ok', mod(), ret_rtl()}. %% The last for debugging only --type compile_file() :: atom() | string() | binary(). -type compile_ret() :: {hipe_architecture(), binary()} | list(). %%------------------------------------------------------------------- @@ -233,26 +232,26 @@ %%------------------------------------------------------------------- -%% @spec load(Mod) -> {module, Mod} | {error, Reason} -%% Mod = mod() +%% @spec load(Module) -> {module, Module} | {error, Reason} +%% Module = mod() %% Reason = term() %% %% @doc Like load/2, but tries to locate a BEAM file automatically. %% %% @see load/2 --spec load(Mod) -> {'module', Mod} | {'error', term()} when Mod :: mod(). +-spec load(Module) -> {'module', Module} | {'error', Reason :: term()} + when Module :: mod(). -load(Mod) -> - load(Mod, beam_file(Mod)). +load(Module) -> + load(Module, beam_file(Module)). -%% @spec load(Mod, BeamFileName) -> {module, Mod} | {error, Reason} -%% Mod = mod() +%% @spec load(Module, BeamFileName) -> {module, Module} | {error, Reason} +%% Module = mod() +%% BeamFileName = file:filename() %% Reason = term() -%% BeamFileName = string() -%% filename() = term() %% -%% @type mod() = atom(). A module name. +%% @type mod() = module(). A module name. %% %% @doc User interface for loading code into memory. The code can be %% given as a native code binary or as the file name of a BEAM file @@ -262,8 +261,8 @@ load(Mod) -> %% %% @see load/1 --spec load(Mod, string()) -> {'module', Mod} | {'error', term()} - when Mod :: mod(). +-spec load(Module, file:filename()) -> {'module', Module} | {'error', term()} + when Module :: mod(). load(Mod, BeamFileName) when is_list(BeamFileName) -> Architecture = erlang:system_info(hipe_architecture), @@ -273,26 +272,22 @@ load(Mod, BeamFileName) when is_list(BeamFileName) -> Error -> {error, Error} end. -%% @spec c(Name) -> {ok, Name} | {error, Reason} -%% Name = mod() +%% @spec c(Mod) -> {ok, Mod} | {error, Reason} +%% Mod = mod() %% Reason = term() %% -%% @equiv c(Name, []) +%% @equiv c(Mod, []) -spec c(mod()) -> c_ret(). -c(Name) -> - c(Name, []). +c(Mod) -> + c(Mod, []). -%% @spec c(Name, options()) -> {ok, Name} | {error, Reason} -%% Name = mod() +%% @spec c(Module, options()) -> {ok, Module} | {error, Reason} +%% Module = mod() %% options() = [option()] %% option() = term() %% Reason = term() -%% -%% @type fun() = atom(). A function identifier. -%% -%% @type arity() = integer(). A function arity; always nonnegative. %% %% @doc User-friendly native code compiler interface. Reads BEAM code %% from the corresponding "Module<code>.beam</code>" file in the @@ -307,12 +302,12 @@ c(Name) -> -spec c(mod(), comp_options()) -> c_ret(). -c(Name, Options) -> - c(Name, beam_file(Name), Options). +c(Module, Options) -> + c(Module, beam_file(Module), Options). -%% @spec c(Name, File, options()) -> {ok, Name} | {error, Reason} -%% Name = mod() -%% File = filename() | binary() +%% @spec c(Module, File, options()) -> {ok, Module} | {error, Reason} +%% Module = mod() +%% File = file:filename() | binary() %% Reason = term() %% %% @doc Like <code>c/2</code>, but reads BEAM code from the specified @@ -321,32 +316,32 @@ c(Name, Options) -> %% @see c/2 %% @see f/2 -c(Name, File, Opts) -> +c(Module, File, Opts) -> Opts1 = user_compile_opts(Opts), - case compile(Name, File, Opts1) of + case compile(Module, File, Opts1) of {ok, Res} -> case proplists:get_bool(to_rtl, Opts1) of - true -> {ok, Name, Res}; - false -> {ok, Name} + true -> {ok, Module, Res}; + false -> {ok, Module} end; Other -> Other end. %% @spec f(File) -> {ok, Name} | {error, Reason} -%% File = filename() | binary() +%% File = file:filename() | binary() %% Name = mod() %% Reason = term() %% %% @equiv f(File, []) --spec f(f_unit()) -> {'ok', mod()} | {'error', term()}. +-spec f(file_or_bin()) -> {'ok', mod()} | {'error', term()}. f(File) -> f(File, []). %% @spec f(File, options()) -> {ok, Name} | {error, Reason} -%% File = filename() | binary() +%% File = file:filename() | binary() %% Name = mod() %% Reason = term() %% @@ -355,7 +350,7 @@ f(File) -> %% %% @see c/3 --spec f(f_unit(), comp_options()) -> {'ok', mod()} | {'error', term()}. +-spec f(file_or_bin(), comp_options()) -> {'ok', mod()} | {'error', term()}. f(File, Opts) -> case file(File, user_compile_opts(Opts)) of @@ -371,20 +366,20 @@ user_compile_opts(Opts) -> Opts ++ ?USER_DEFAULTS. -%% @spec compile(Name) -> {ok, {Target,Binary}} | {error, Reason} -%% Name = mod() +%% @spec compile(Module) -> {ok, {Target,Binary}} | {error, Reason} +%% Module = mod() %% Binary = binary() %% Reason = term() %% -%% @equiv compile(Name, []) +%% @equiv compile(Module, []) -spec compile(mod()) -> {'ok', compile_ret()} | {'error', term()}. -compile(Name) -> - compile(Name, []). +compile(Module) -> + compile(Module, []). -%% @spec compile(Name, options()) -> {ok, {Target,Binary}} | {error, Reason} -%% Name = mod() +%% @spec compile(Module, options()) -> {ok, {Target,Binary}} | {error, Reason} +%% Module = mod() %% Binary = binary() %% Reason = term() %% @@ -403,26 +398,26 @@ compile(Name) -> %% @see file/2 %% @see load/2 --spec compile(mod(), comp_options()) -> {'ok', compile_ret()} | {'error', _}. +-spec compile(mod(), comp_options()) -> {'ok', compile_ret()} | {'error', term()}. -compile(Name, Options) -> - compile(Name, beam_file(Name), Options). +compile(Module, Options) -> + compile(Module, beam_file(Module), Options). --spec beam_file(mod()) -> string(). +-spec beam_file(mod()) -> file:filename(). beam_file(Module) when is_atom(Module) -> case code:which(Module) of non_existing -> - ?error_msg("Cannot find ~w.beam file.",[Module]), + ?error_msg("Cannot find ~w.beam file.", [Module]), ?EXIT({cant_find_beam_file,Module}); - File -> % string() + File when is_list(File) -> File end. %% @spec compile(Name, File, options()) -> %% {ok, {Target, Binary}} | {error, Reason} %% Name = mod() -%% File = filename() | binary() +%% File = file:filename() | binary() %% Binary = binary() %% Reason = term() %% @@ -431,7 +426,7 @@ beam_file(Module) when is_atom(Module) -> %% %% @see compile/2 --spec compile(mod(), compile_file(), comp_options()) -> +-spec compile(mod(), file_or_bin(), comp_options()) -> {'ok', compile_ret()} | {'error', term()}. compile(Name, File, Opts0) when is_atom(Name) -> @@ -475,18 +470,18 @@ compile(Name, File, Opts0) when is_atom(Name) -> run_compiler(Name, DisasmFun, IcodeFun, Opts) end. --spec compile_core(mod(), cerl:c_module(), compile_file(), comp_options()) -> +-spec compile_core(mod(), cerl:c_module(), file_or_bin(), comp_options()) -> {'ok', compile_ret()} | {'error', term()}. compile_core(Name, Core0, File, Opts) -> Core = cerl:from_records(Core0), compile(Name, Core, File, Opts). -%% @spec compile(Name, Core, File, options()) -> +%% @spec compile(Module, Core, File, options()) -> %% {ok, {Target, Binary}} | {error, Reason} -%% Name = mod() +%% Module = mod() %% Core = coreErlang() | [] -%% File = filename() | binary() +%% File = file:filename() | binary() %% Binary = binary() %% Reason = term() %% @@ -499,7 +494,7 @@ compile_core(Name, Core0, File, Opts) -> %% %% @see compile/3 --spec compile(mod(), cerl:c_module() | [], compile_file(), comp_options()) -> +-spec compile(mod(), cerl:c_module() | [], file_or_bin(), comp_options()) -> {'ok', compile_ret()} | {'error', term()}. compile(Name, [], File, Opts) -> @@ -511,38 +506,36 @@ compile(Name, Core, File, Opts) when is_atom(Name) -> end, run_compiler(Name, DisasmFun, IcodeFun, Opts). -%% @spec file(File) -> {ok, Name, {Target, Binary}} | {error, Reason} -%% File = filename() | binary() -%% Name = mod() | mfa() +%% @spec file(File) -> {ok, Mod, {Target, Binary}} | {error, Reason} +%% File = file:filename() +%% Mod = mod() %% Binary = binary() %% Reason = term() %% %% @equiv file(File, []) --spec file(Mod) -> {'ok', Mod, compile_ret()} | {'error', term()} - when Mod :: mod(). +-spec file(file:filename()) -> {'ok', mod(), compile_ret()} | {'error', term()}. file(File) -> file(File, []). -%% @spec file(File, options()) -> {ok, Name, {Target,Binary}} | {error, Reason} -%% File = filename() -%% Name = mod() | mfa() +%% @spec file(File, options()) -> {ok, Mod, {Target, Binary}} | {error, Reason} +%% File = file:filename() +%% Mod = mod() %% Binary = binary() %% Reason = term() %% %% @doc Like <code>compile/2</code>, but takes the module name from the -%% specified <code>File</code>. Returns both the name and the final +%% specified <code>File</code>. Returns both the module name and the final %% binary if successful. %% %% @see file/1 %% @see compile/2 --spec file(Mod, comp_options()) -> {'ok', Mod, compile_ret()} - | {'error', term()} - when Mod :: mod(). -file(File, Options) when is_atom(File) -> - case beam_lib:info(atom_to_list(File)) of +-spec file(file:filename(), comp_options()) -> {'ok', mod(), compile_ret()} + | {'error', Reason :: term()}. +file(File, Options) when is_list(File) -> + case beam_lib:info(File) of L when is_list(L) -> {module, Mod} = lists:keyfind(module, 1, L), case compile(Mod, File, Options) of @@ -590,9 +583,8 @@ fix_beam_exports([], Exports) -> Exports. get_beam_icode(Mod, {BeamCode, Exports}, File, Options) -> - {ok, Icode} = - ?option_time((catch {ok, hipe_beam_to_icode:module(BeamCode, Options)}), - "BEAM-to-Icode", Options), + Icode = ?option_time(hipe_beam_to_icode:module(BeamCode, Options), + "BEAM-to-Icode", Options), BeamBin = get_beam_code(File), {{Mod, Exports, Icode}, BeamBin}. @@ -653,7 +645,7 @@ run_compiler_1(Name, DisasmFun, IcodeFun, Options) -> get(hipe_target_arch)), Opts = case proplists:get_bool(to_llvm, Opts0) andalso - not llvm_support_available() of + not llvm_version_is_OK() of true -> ?error_msg("No LLVM version 3.9 or greater " "found in $PATH; aborting " @@ -669,9 +661,12 @@ run_compiler_1(Name, DisasmFun, IcodeFun, Options) -> {Icode, WholeModule} = IcodeFun(Code, Opts), CompRes = compile_finish(Icode, WholeModule, Opts), compiler_return(CompRes, Parent) - catch error:Error:StackTrace -> + catch + error:Error:StackTrace -> print_crash_message(Name, Error, StackTrace), - exit(Error) + exit(Error); + throw:{unimplemented_instruction,_Instruction}=Error -> + exit(Error) end end), Timeout = case proplists:get_value(timeout, Options) of @@ -1607,9 +1602,15 @@ check_options(Opts) -> ok end. --spec llvm_support_available() -> boolean(). +-spec erllvm_is_supported() -> boolean(). +erllvm_is_supported() -> + %% XXX: The test should really check the _target_ architecture, + %% (hipe_target_arch), but there's no guarantee it's set. + Arch = erlang:system_info(hipe_architecture), + lists:member(Arch, [amd64, x86]) andalso llvm_version_is_OK(). -llvm_support_available() -> +-spec llvm_version_is_OK() -> boolean(). +llvm_version_is_OK() -> get_llvm_version() >= {3,9}. -type llvm_version() :: {Major :: integer(), Minor :: integer()}. diff --git a/lib/hipe/rtl/Makefile b/lib/hipe/rtl/Makefile index becdd0b7d8..0c0f6e24f5 100644 --- a/lib/hipe/rtl/Makefile +++ b/lib/hipe/rtl/Makefile @@ -75,7 +75,10 @@ TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) include ../native.mk -ERL_COMPILE_FLAGS += -Werror +inline +warn_unused_import +warn_export_vars +ERL_COMPILE_FLAGS += +inline +warn_unused_import +warn_export_vars +ifneq ($(NATIVE_LIBS_ENABLED),yes) +ERL_COMPILE_FLAGS += -Werror +endif # ---------------------------------------------------- # Targets diff --git a/lib/hipe/rtl/hipe_icode2rtl.erl b/lib/hipe/rtl/hipe_icode2rtl.erl index 6da8a76d34..1ab41f4deb 100644 --- a/lib/hipe/rtl/hipe_icode2rtl.erl +++ b/lib/hipe/rtl/hipe_icode2rtl.erl @@ -215,7 +215,7 @@ gen_enter(I, VarMap, ConstTab) -> {Code1, ConstTab2} = case hipe_icode:enter_type(I) of primop -> - IsGuard = false, % enter can not happen in a guard + IsGuard = false, % enter cannot happen in a guard hipe_rtl_primops:gen_enter_primop({Fun, Args}, IsGuard, ConstTab1); Type -> Call = gen_enter_1(Fun, Args, Type), diff --git a/lib/hipe/rtl/hipe_rtl_arith.inc b/lib/hipe/rtl/hipe_rtl_arith.inc index c05b7aa160..575f10b542 100644 --- a/lib/hipe/rtl/hipe_rtl_arith.inc +++ b/lib/hipe/rtl/hipe_rtl_arith.inc @@ -118,8 +118,8 @@ eval_alu(Op, Arg1, Arg2) -> %% Björn & Bjarni: %% We need to be able to do evaluations based only on the bits, since -%% there are cases where we can evaluate a subset of the bits, but can -%% not do a full eval-alub call (eg. a + 0 gives no carry) +%% there are cases where we can evaluate a subset of the bits, but +%% cannot do a full eval-alub call (eg. a + 0 gives no carry) %% -spec eval_cond_bits(hipe_rtl:alub_cond(), boolean(), boolean(), boolean(), boolean()) -> boolean(). diff --git a/lib/hipe/rtl/hipe_rtl_ssa_const_prop.erl b/lib/hipe/rtl/hipe_rtl_ssa_const_prop.erl index cad43e2df5..72373e536d 100644 --- a/lib/hipe/rtl/hipe_rtl_ssa_const_prop.erl +++ b/lib/hipe/rtl/hipe_rtl_ssa_const_prop.erl @@ -31,11 +31,11 @@ %% %% Some things to note: %% -%% 1. All precoloured registers are assumed to contain bottom. We can not +%% 1. All precoloured registers are assumed to contain bottom. We cannot %% do anything with them since they are not in SSA-form. This might be %% possible to resolve in some way, but we decided to not go there. %% -%% 2. const_labels are assumed to be bottom, we can not find the address +%% 2. const_labels are assumed to be bottom, we cannot find the address %% in any nice way (that I know of, maybe someone can help ?). I %% suppose they don't get a value until linking (or some step that %% resembles it). They are only affecting bignums and floats (at least @@ -579,7 +579,7 @@ visit_multimove(Inst, Env) -> %% Procedure : visit_call/2 %% Purpose : execute a call-instruction. All calls return bottom. We make %% this assumption since the icode-leel have taken care of BIF's -%% and we belive that we are left with the things that can not be +%% and we belive that we are left with the things that cannot be %% done att compile time. %% Arguments : Inst - The instruction %% Env - The environment diff --git a/lib/hipe/test/Makefile b/lib/hipe/test/Makefile index efeb0887ab..3650cda663 100644 --- a/lib/hipe/test/Makefile +++ b/lib/hipe/test/Makefile @@ -13,7 +13,6 @@ MODULES= \ # .erl files for these modules are automatically generated GEN_MODULES= \ basic_SUITE \ - bs_SUITE \ maps_SUITE \ sanity_SUITE diff --git a/lib/hipe/test/basic_SUITE_data/basic_inline_function.erl b/lib/hipe/test/basic_SUITE_data/basic_inline_function.erl deleted file mode 100644 index 4c08064670..0000000000 --- a/lib/hipe/test/basic_SUITE_data/basic_inline_function.erl +++ /dev/null @@ -1,73 +0,0 @@ -%%% -*- erlang-indent-level: 2 -*- -%%%------------------------------------------------------------------- -%%% Author: Kostis Sagonas -%%% -%%% Contains tests that depend on the compiler inliner being turned on. -%%%------------------------------------------------------------------- --module(basic_inline_function). - --export([test/0]). - --compile({inline, [{to_objects, 3}]}). - -test() -> - ok = test_inline_match(), - ok. - -%%-------------------------------------------------------------------- - -test_inline_match() -> - bad_object = test1(a, {binary, foo, set}, c), - bad_object = test2(a, {binary, foo, set}, c), - bad_object = test3(a, {binary, foo, set}, c), - ok. - -%% Inlined -test1(KeysObjs, C, Ts) -> - case catch to_objects(KeysObjs, C, Ts) of - {'EXIT', _} -> - bad_object; - ok -> - ok - end. - -%% "Inlined" by hand -test2(KeysObjs, C, _Ts) -> - case catch (case C of - {binary, _, set} -> - <<_ObjSz0:32, _T/binary>> = KeysObjs; - _ -> ok - end) of - {'EXIT', _} -> - bad_object; - ok -> - ok - end. - -%% Not inlined -test3(KeysObjs, C, Ts) -> - case catch fto_objects(KeysObjs, C, Ts) of - {'EXIT', _} -> - bad_object; - ok -> - ok - end. - -%% Inlined. -to_objects(Bin, {binary, _, set}, _Ts) -> - <<_ObjSz0:32, _T/binary>> = Bin, - ok; -to_objects(<<_ObjSz0:32, _T/binary>> ,_, _) -> - ok; -to_objects(_Bin, _, _Ts) -> - ok. - -%% Not Inlined. -fto_objects(Bin, {binary, _, set}, _Ts) -> - <<_ObjSz0:32, _T/binary>> = Bin, - ok; -fto_objects(<<_ObjSz0:32, _T/binary>> ,_,_) -> - ok; -fto_objects(_Bin, _, _Ts) -> - ok. - diff --git a/lib/hipe/test/hipe_testsuite_driver.erl b/lib/hipe/test/hipe_testsuite_driver.erl index 8813af5dfc..c506dd5e1d 100644 --- a/lib/hipe/test/hipe_testsuite_driver.erl +++ b/lib/hipe/test/hipe_testsuite_driver.erl @@ -170,7 +170,7 @@ run(TestCase, Dir, _OutDir) -> {ok, TestCase} = hipe:c(TestCase, [o0|HiPEOpts]), ok = TestCase:test(), ToLLVM = try TestCase:to_llvm() catch error:undef -> true end, - case ToLLVM andalso hipe:llvm_support_available() of + case ToLLVM andalso hipe:erllvm_is_supported() of true -> {ok, TestCase} = hipe:c(TestCase, [to_llvm|HiPEOpts]), ok = TestCase:test(); diff --git a/lib/inets/doc/src/httpd.xml b/lib/inets/doc/src/httpd.xml index 66369e8df9..1750078db0 100644 --- a/lib/inets/doc/src/httpd.xml +++ b/lib/inets/doc/src/httpd.xml @@ -173,7 +173,7 @@ <item> <p>For <c>ip_comm</c> configuration options, see <seealso marker="kernel:gen_tcp#listen-2">gen_tcp:listen/2</seealso>, some options - that are used internally by httpd can not be set.</p> + that are used internally by httpd cannot be set.</p> <p>For <c>SSL</c> configuration options, see <seealso marker="ssl:ssl#listen-2">ssl:listen/2</seealso>.</p> <p>Default is <c>ip_comm</c>.</p> diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index a1b5dfc309..2710ea2f2f 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -1399,7 +1399,7 @@ Add option {ftp_extension, boolean} to enable use of extended commands EPSV and EPRT, as specified in RFC 2428, for IPv4 instead of using the legacy commands. Ipv6 - can not be supported without the extended commands.</p> + cannot be supported without the extended commands.</p> <p> Own Id: OTP-12255</p> </item> diff --git a/lib/inets/doc/src/notes_history.xml b/lib/inets/doc/src/notes_history.xml index c12899e614..1523827db9 100644 --- a/lib/inets/doc/src/notes_history.xml +++ b/lib/inets/doc/src/notes_history.xml @@ -1149,8 +1149,8 @@ <list type="bulleted"> <item> <p>When further testing the functionality of https requests - that goes through a proxy. We realised that alas this can - not currently be supported as it requires features from + that goes through a proxy. We realised that alas this + cannot currently be supported as it requires features from the ssl implementation that is not currently available. So for now an error message will be returned when trying to use this functionality.</p> diff --git a/lib/inets/examples/httpd_load_test/hdlt_slave.erl b/lib/inets/examples/httpd_load_test/hdlt_slave.erl index 5ee005629d..30ddfd76af 100644 --- a/lib/inets/examples/httpd_load_test/hdlt_slave.erl +++ b/lib/inets/examples/httpd_load_test/hdlt_slave.erl @@ -44,18 +44,17 @@ %% this to work is that the 'erl' program can be found in PATH. %% %% If the master and slave are on different hosts, start/N uses -%% the 'rsh' program to spawn an Erlang node on the other host. +%% the 'ssh' program to spawn an Erlang node on the other host. %% Alternative, if the master was started as %% 'erl -sname xxx -rsh my_rsh...', then 'my_rsh' will be used instead -%% of 'rsh' (this is useful for systems where the rsh program is named -%% 'remsh'). +%% of 'ssh' (this is useful for systems still using rsh or remsh). %% %% For this to work, the following conditions must be fulfilled: %% -%% 1. There must be an Rsh program on computer; if not an error +%% 1. There must be an ssh program on computer; if not an error %% is returned. %% -%% 2. The hosts must be configured to allowed 'rsh' access without +%% 2. The hosts must be configured to allow 'ssh' access without %% prompts for password. %% %% The slave node will have its filer and user server redirected @@ -244,7 +243,7 @@ register_unique_name(Number) -> %% Makes up the command to start the nodes. %% If the node should run on the local host, there is -%% no need to use rsh. +%% no need to use ssh. mk_cmd(Host, Name, Paths, Args, Waiter, Prog) -> PaPaths = [[" -pa ", Path] || Path <- Paths], diff --git a/lib/inets/src/http_client/httpc_cookie.erl b/lib/inets/src/http_client/httpc_cookie.erl index cbf428ab3e..2e647a1438 100644 --- a/lib/inets/src/http_client/httpc_cookie.erl +++ b/lib/inets/src/http_client/httpc_cookie.erl @@ -271,7 +271,7 @@ lookup_cookies(CookieDb, Host, Path) -> lookup_domain_cookies(_CookieDb, [], AccCookies) -> lists:flatten(AccCookies); -%% Top domains can not have cookies +%% Top domains cannot have cookies lookup_domain_cookies(_CookieDb, [_], AccCookies) -> lists:flatten(AccCookies); diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl index 0333442bf2..0dc0483fa9 100644 --- a/lib/inets/src/http_client/httpc_manager.erl +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -863,7 +863,7 @@ select_session(Candidates, _, Max, pipeline) -> select_session(Candidates, Max). select_session([] = _Candidates, _Max) -> - ?hcrd("select session - no candicate", []), + ?hcrd("select session - no candidate", []), no_connection; select_session(Candidates, Max) -> NewCandidates = diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl index 78d6b4ed24..bb6b76da89 100644 --- a/lib/inets/src/http_client/httpc_response.erl +++ b/lib/inets/src/http_client/httpc_response.erl @@ -398,7 +398,7 @@ redirect(Response = {_, Headers, _}, Request) -> THost = http_util:maybe_add_brackets(maps:get(host, URIMap), Brackets), TPort = maps:get(port, URIMap), TPath = maps:get(path, URIMap), - TQuery = maps:get(query, URIMap, ""), + TQuery = add_question_mark(maps:get(query, URIMap, "")), NewURI = uri_string:normalize( uri_string:recompose(URIMap)), HostPort = http_request:normalize_host(TScheme, THost, TPort), @@ -417,6 +417,14 @@ redirect(Response = {_, Headers, _}, Request) -> end end. +add_question_mark(<<>>) -> + <<>>; +add_question_mark([]) -> + []; +add_question_mark(Comp) when is_binary(Comp) -> + <<$?, Comp/binary>>; +add_question_mark(Comp) when is_list(Comp) -> + [$?|Comp]. %% RFC3986 - 5.2.2. Transform References resolve_uri(Scheme, Host, Port, Path, Query, URI) -> diff --git a/lib/inets/src/http_server/mod_responsecontrol.erl b/lib/inets/src/http_server/mod_responsecontrol.erl index 07129940a5..a32ba65c22 100644 --- a/lib/inets/src/http_server/mod_responsecontrol.erl +++ b/lib/inets/src/http_server/mod_responsecontrol.erl @@ -71,7 +71,7 @@ do_responsecontrol(Info) -> %% If a client sends more then one of the if-XXXX fields in a request -%% The standard says it does not specify the behaviuor so I specified it :-) +%% The standard says it does not specify the behaviour so I specified it :-) %% The priority between the fields is %% 1.If-modified %% 2.If-Unmodified diff --git a/lib/inets/test/http_format_SUITE.erl b/lib/inets/test/http_format_SUITE.erl index d6b0e5f9f5..0a5aed67d5 100644 --- a/lib/inets/test/http_format_SUITE.erl +++ b/lib/inets/test/http_format_SUITE.erl @@ -435,7 +435,7 @@ http_request(Config) when is_list(Config) -> [<<>>, Length1], HttpBody1)). %%------------------------------------------------------------------------- validate_request_line() -> - [{doc, "Test httpd_request:validate/3. Makes sure you can not get past" + [{doc, "Test httpd_request:validate/3. Makes sure you cannot get past" " the server_root and that the request is recognized by the server" " and protcol version."}]. validate_request_line(Config) when is_list(Config) -> diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 8357e02014..d4b33ae2c6 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -339,14 +339,6 @@ end_per_testcase(Case, Config) end_per_testcase(_Case, _Config) -> ok. -is_ipv6_supported() -> - case gen_udp:open(0, [inet6]) of - {ok, Socket} -> - gen_udp:close(Socket), - true; - _ -> - false - end. %%-------------------------------------------------------------------- @@ -1612,7 +1604,8 @@ post_with_content_type(Config) when is_list(Config) -> %%-------------------------------------------------------------------- request_options() -> - [{doc, "Test http get request with socket options against local server (IPv6)"}]. + [{require, ipv6_hosts}, + {doc, "Test http get request with socket options against local server (IPv6)"}]. request_options(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, {ok, {{_,200,_}, [_ | _], _ = [_ | _]}} = httpc:request(get, Request, [], @@ -2945,3 +2938,12 @@ receive_stream_n(Ref, N) -> ct:pal("Data: ~p", [Data]), receive_stream_n(Ref, N-1) end. + +is_ipv6_supported() -> + {ok, Hostname0} = inet:gethostname(), + try + lists:member(list_to_atom(Hostname0), ct:get_config(ipv6_hosts)) + catch + _: _ -> + false + end. diff --git a/lib/inets/test/httpc_proxy_SUITE.erl b/lib/inets/test/httpc_proxy_SUITE.erl index 198b245399..3ee7981660 100644 --- a/lib/inets/test/httpc_proxy_SUITE.erl +++ b/lib/inets/test/httpc_proxy_SUITE.erl @@ -534,7 +534,7 @@ init_local_proxy(Config) -> ct:fail({local_proxy_start_failed,Error}) end; _ -> - {skip,"Platform can not run local proxy start script"} + {skip,"Platform cannot run local proxy start script"} end. init_local_proxy_string(String, Config) -> diff --git a/lib/inets/test/httpd_bench_SUITE.erl b/lib/inets/test/httpd_bench_SUITE.erl index 4b549dcb5b..087516f56c 100644 --- a/lib/inets/test/httpd_bench_SUITE.erl +++ b/lib/inets/test/httpd_bench_SUITE.erl @@ -37,7 +37,8 @@ %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- suite() -> - [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}]. + [{timetrap, {minutes, 1}}, + {ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}]. all() -> [ @@ -499,8 +500,8 @@ start_dummy("http"= Protocol, Config) -> Conf = [ %%{big, filename:join(DataDir, "1M_file")}, %%{small, filename:join(DataDir, "1k_file")}, - {big, {gen, crypto:rand_bytes(1000000)}}, - {small, {gen, crypto:rand_bytes(1000)}}, + {big, {gen, crypto:strong_rand_bytes(1000000)}}, + {small, {gen, crypto:strong_rand_bytes(1000)}}, {http_version, HTTPVersion}, {keep_alive, ?config(keep_alive, Config)} ], @@ -519,8 +520,8 @@ start_dummy("https" = Protocol, Config) -> Opts = [{active, true}, {nodelay, true}, {reuseaddr, true} | SSLOpts], Conf = [%%{big, filename:join(DataDir, "1M_file")}, %%{small, filename:join(DataDir, "1k_file")}, - {big, {gen, crypto:rand_bytes(1000000)}}, - {small, {gen, crypto:rand_bytes(1000)}}, + {big, {gen, crypto:strong_rand_bytes(1000000)}}, + {small, {gen, crypto:strong_rand_bytes(1000)}}, {http_version, HTTPVersion}, {keep_alive, ?config(keep_alive, Config)} ], diff --git a/lib/jinterface/doc/src/jinterface_users_guide.xml b/lib/jinterface/doc/src/jinterface_users_guide.xml index 56f88124bf..504c77f339 100644 --- a/lib/jinterface/doc/src/jinterface_users_guide.xml +++ b/lib/jinterface/doc/src/jinterface_users_guide.xml @@ -366,20 +366,20 @@ OtpNode node = new OtpNode("gurka"); </code> <seealso marker="java/com/ericsson/otp/erlang/OtpEpmd">OtpEpmd</seealso> class. Nodes wishing to contact other nodes must first request information from Epmd before a connection can be set up, however this is done automatically - by <seealso marker="java/com/ericsson/otp/erlang/OtpSelf#connect(com.ericsson.otp.erlang.OtpPeer)">OtpSelf.connect()</seealso> when necessary. </p> - <p>When you use <seealso marker="java/com/ericsson/otp/erlang/OtpSelf#connect(com.ericsson.otp.erlang.OtpPeer)">OtpSelf.connect()</seealso> to connect to an Erlang node, + by <seealso marker="java/com/ericsson/otp/erlang/OtpSelf#connect-com.ericsson.otp.erlang.OtpPeer-">OtpSelf.connect()</seealso> when necessary. </p> + <p>When you use <seealso marker="java/com/ericsson/otp/erlang/OtpSelf#connect-com.ericsson.otp.erlang.OtpPeer-">OtpSelf.connect()</seealso> to connect to an Erlang node, a connection is first made to epmd and, if the node is known, a connection is then made to the Erlang node.</p> <p>Java nodes can also register themselves with epmd if they want other nodes in the system to be able to find and connect to them. - This is done by call to method <seealso marker="java/com/ericsson/otp/erlang/OtpEpmd#publishPort(com.ericsson.otp.erlang.OtpLocalNode)">OtpEpmd.publishPort()</seealso>.</p> + This is done by call to method <seealso marker="java/com/ericsson/otp/erlang/OtpEpmd#publishPort-com.ericsson.otp.erlang.OtpLocalNode-">OtpEpmd.publishPort()</seealso>.</p> <p>Be aware that on some systems (such as VxWorks), a failed node will not be detected by this mechanism since the operating system does not automatically close descriptors that were left open when the node failed. If a node has failed in this way, epmd will prevent you from registering a new node with the old name, since it thinks that the old name is still in use. In this case, you must unregister the name - explicitly, by using <seealso marker="java/com/ericsson/otp/erlang/OtpEpmd#unPublishPort(com.ericsson.otp.erlang.OtpLocalNode)">OtpEpmd.unPublishPort()</seealso></p> + explicitly, by using <seealso marker="java/com/ericsson/otp/erlang/OtpEpmd#unPublishPort-com.ericsson.otp.erlang.OtpLocalNode-">OtpEpmd.unPublishPort()</seealso></p> <p>This will cause epmd to close the connection from the far end. Note that if the name was in fact still in use by a node, the results of this operation are unpredictable. Also, doing this does not cause the diff --git a/lib/jinterface/doc/src/notes.xml b/lib/jinterface/doc/src/notes.xml index 75a2364384..e4bfddcd17 100644 --- a/lib/jinterface/doc/src/notes.xml +++ b/lib/jinterface/doc/src/notes.xml @@ -720,7 +720,7 @@ <item> <p><c>OtpMbox.receive()</c> and <c>OtpMbox.receive(long timeout)</c> can now throw <c>OtpErlangDecodeException</c> if the received message - can not be decoded. <c>null</c> is now only returned from + cannot be decoded. <c>null</c> is now only returned from <c>OtpMbox.receive(long timeout)</c> if a timeout occurs. <c>OtpMbox.receive()</c> will never return <c>null</c>.</p> <p>*** POTENTIAL INCOMPATIBILITY ***</p> diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractNode.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractNode.java index 222330654a..c3f71a84f0 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractNode.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/AbstractNode.java @@ -89,7 +89,7 @@ public class AbstractNode implements OtpTransportFactory { static final int dFlagHiddenAtomCache = 0x40; // NOT SUPPORTED static final int dflagNewFunTags = 0x80; static final int dFlagExtendedPidsPorts = 0x100; - static final int dFlagExportPtrTag = 0x200; // NOT SUPPORTED + static final int dFlagExportPtrTag = 0x200; static final int dFlagBitBinaries = 0x400; static final int dFlagNewFloats = 0x800; static final int dFlagUnicodeIo = 0x1000; @@ -105,6 +105,7 @@ public class AbstractNode implements OtpTransportFactory { int flags = dFlagExtendedReferences | dFlagExtendedPidsPorts | dFlagBitBinaries | dFlagNewFloats | dFlagFunTags | dflagNewFunTags | dFlagUtf8Atoms | dFlagMapTag + | dFlagExportPtrTag | dFlagBigCreation; /* initialize hostname and default cookie */ diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpEpmd.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpEpmd.java index 363fdb950a..fffb8475d3 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpEpmd.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpEpmd.java @@ -102,7 +102,7 @@ public class OtpEpmd { /** * Set the port number to be used to contact the epmd process. Only needed * when the default port is not desired and system environment variable - * ERL_EPMD_PORT can not be read (applet). + * ERL_EPMD_PORT cannot be read (applet). */ public static void useEpmdPort(final int port) { EpmdPort.set(port); diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpInputStream.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpInputStream.java index ded8f6e1e5..6d81ce630b 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpInputStream.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpInputStream.java @@ -601,7 +601,7 @@ public class OtpInputStream extends ByteArrayInputStream { * @return the integer value. * * @exception OtpErlangDecodeException - * if the next term in the stream can not be represented as a + * if the next term in the stream cannot be represented as a * positive integer. */ public int read_uint() throws OtpErlangDecodeException { @@ -622,7 +622,7 @@ public class OtpInputStream extends ByteArrayInputStream { * @return the integer value. * * @exception OtpErlangDecodeException - * if the next term in the stream can not be represented as + * if the next term in the stream cannot be represented as * an integer. */ public int read_int() throws OtpErlangDecodeException { @@ -643,7 +643,7 @@ public class OtpInputStream extends ByteArrayInputStream { * @return the short value. * * @exception OtpErlangDecodeException - * if the next term in the stream can not be represented as a + * if the next term in the stream cannot be represented as a * positive short. */ public short read_ushort() throws OtpErlangDecodeException { @@ -664,7 +664,7 @@ public class OtpInputStream extends ByteArrayInputStream { * @return the short value. * * @exception OtpErlangDecodeException - * if the next term in the stream can not be represented as a + * if the next term in the stream cannot be represented as a * short. */ public short read_short() throws OtpErlangDecodeException { @@ -685,7 +685,7 @@ public class OtpInputStream extends ByteArrayInputStream { * @return the long value. * * @exception OtpErlangDecodeException - * if the next term in the stream can not be represented as a + * if the next term in the stream cannot be represented as a * positive long. */ public long read_ulong() throws OtpErlangDecodeException { @@ -698,7 +698,7 @@ public class OtpInputStream extends ByteArrayInputStream { * @return the long value. * * @exception OtpErlangDecodeException - * if the next term in the stream can not be represented as a + * if the next term in the stream cannot be represented as a * long. */ public long read_long() throws OtpErlangDecodeException { diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpMbox.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpMbox.java index 42e178c3f6..29a8bc1540 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpMbox.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpMbox.java @@ -156,7 +156,7 @@ public class OtpMbox { * of the next message waiting in this mailbox. * * @exception OtpErlangDecodeException - * if the message can not be decoded. + * if the message cannot be decoded. * * @exception OtpErlangExit * if a linked {@link OtpErlangPid pid} has exited or has @@ -184,7 +184,7 @@ public class OtpMbox { * of the next message waiting in this mailbox. * * @exception OtpErlangDecodeException - * if the message can not be decoded. + * if the message cannot be decoded. * * @exception OtpErlangExit * if a linked {@link OtpErlangPid pid} has exited or has diff --git a/lib/jinterface/test/nc_SUITE.erl b/lib/jinterface/test/nc_SUITE.erl index 4f225a396e..7833d070b7 100644 --- a/lib/jinterface/test/nc_SUITE.erl +++ b/lib/jinterface/test/nc_SUITE.erl @@ -142,7 +142,8 @@ fun_roundtrip(Config) when is_list(Config)-> do_echo([fun(A, B) -> A + B end, fun(A) -> lists:reverse(A) end, fun() -> ok end, - fun fun_roundtrip/1], + fun fun_roundtrip/1, + fun ?MODULE:fun_roundtrip/1], Config). port_roundtrip(doc) -> []; diff --git a/lib/kernel/doc/src/application.xml b/lib/kernel/doc/src/application.xml index 83a83ebad2..f4ec2c610f 100644 --- a/lib/kernel/doc/src/application.xml +++ b/lib/kernel/doc/src/application.xml @@ -582,7 +582,7 @@ Nodes = [cp1@cave, {cp2@cave, cp3@cave}]</code> the primary application) for the primary application and all included applications, for which the start phase is defined.</p> <p>For a description of <c>StartType</c>, see - <seealso marker="Module:start/2"><c>Module:start/2</c></seealso>.</p> + <seealso marker="#Module:start/2"><c>Module:start/2</c></seealso>.</p> </desc> </func> <func> diff --git a/lib/kernel/doc/src/auth.xml b/lib/kernel/doc/src/auth.xml index a57da18de9..c735d02fed 100644 --- a/lib/kernel/doc/src/auth.xml +++ b/lib/kernel/doc/src/auth.xml @@ -46,7 +46,7 @@ <fsummary>Magic cookie for local node (deprecated).</fsummary> <desc> <p>Use - <seealso marker="erts:erlang#erlang:get_cookie/0"><c>erlang:get_cookie()</c></seealso> + <seealso marker="erts:erlang#get_cookie/0"><c>erlang:get_cookie()</c></seealso> in ERTS instead.</p> </desc> </func> @@ -58,7 +58,7 @@ </type_desc> <desc> <p>Use - <seealso marker="erts:erlang#erlang:set_cookie/2"><c>erlang:set_cookie(node(), <anno>Cookie</anno>)</c> + <seealso marker="erts:erlang#set_cookie/2"><c>erlang:set_cookie(node(), <anno>Cookie</anno>)</c> in ERTS</seealso> instead.</p> </desc> </func> @@ -94,7 +94,7 @@ <p>Sets the magic cookie of <c><anno>Node</anno></c> to <c><anno>Cookie</anno></c> and verifies the status of the authorization. Equivalent to calling - <seealso marker="erts:erlang#erlang:set_cookie/2"><c>erlang:set_cookie(<anno>Node</anno>, <anno>Cookie</anno>)</c></seealso>, followed by + <seealso marker="erts:erlang#set_cookie/2"><c>erlang:set_cookie(<anno>Node</anno>, <anno>Cookie</anno>)</c></seealso>, followed by <seealso marker="#is_auth/1"><c>auth:is_auth(<anno>Node</anno>)</c></seealso>.</p> </desc> </func> diff --git a/lib/kernel/doc/src/code.xml b/lib/kernel/doc/src/code.xml index 85178da930..4aa9e8b9d2 100644 --- a/lib/kernel/doc/src/code.xml +++ b/lib/kernel/doc/src/code.xml @@ -538,7 +538,7 @@ zip:create("mnesia-4.4.7.ez", </item> <tag><c>not_purged</c></tag> <item> - <p>The object code can not be loaded because an old version + <p>The object code cannot be loaded because an old version of the code already exists.</p> </item> <tag><c>sticky_directory</c></tag> @@ -611,7 +611,7 @@ ok = code:finish_loading(Prepared), <taglist> <tag><c>not_purged</c></tag> <item> - <p>The object code can not be loaded because an old version + <p>The object code cannot be loaded because an old version of the code already exists.</p> </item> <tag><c>sticky_directory</c></tag> diff --git a/lib/kernel/doc/src/erl_ddll.xml b/lib/kernel/doc/src/erl_ddll.xml index f2d5e1b397..52d5bcd079 100644 --- a/lib/kernel/doc/src/erl_ddll.xml +++ b/lib/kernel/doc/src/erl_ddll.xml @@ -200,7 +200,7 @@ <fsummary>Remove a monitor for a driver.</fsummary> <desc> <p>Removes a driver monitor in much the same way as - <seealso marker="erts:erlang#erlang:demonitor/1"><c>erlang:demonitor/1</c></seealso> + <seealso marker="erts:erlang#demonitor/1"><c>erlang:demonitor/1</c></seealso> in ERTS does with process monitors. For details about how to create driver monitors, see @@ -430,7 +430,7 @@ <desc> <p>Creates a driver monitor and works in many ways as - <seealso marker="erts:erlang#erlang:monitor/2"><c>erlang:monitor/2</c></seealso> + <seealso marker="erts:erlang#monitor/2"><c>erlang:monitor/2</c></seealso> in ERTS, does for processes. When a driver changes state, the monitor results in a monitor message that is sent to the calling diff --git a/lib/kernel/doc/src/gen_sctp.xml b/lib/kernel/doc/src/gen_sctp.xml index f70d6c24db..61ac1485c1 100644 --- a/lib/kernel/doc/src/gen_sctp.xml +++ b/lib/kernel/doc/src/gen_sctp.xml @@ -253,7 +253,7 @@ connect(Socket, Ip, Port>, <desc> <p>Assigns a new controlling process <c><anno>Pid</anno></c> to <c><anno>Socket</anno></c>. Same implementation as - <seealso marker="gen_udp:controlling_process/2"><c>gen_udp:controlling_process/2</c></seealso>. + <seealso marker="gen_udp#controlling_process/2"><c>gen_udp:controlling_process/2</c></seealso>. </p> </desc> </func> diff --git a/lib/kernel/doc/src/gen_tcp.xml b/lib/kernel/doc/src/gen_tcp.xml index fc16473393..f8b41d24e2 100644 --- a/lib/kernel/doc/src/gen_tcp.xml +++ b/lib/kernel/doc/src/gen_tcp.xml @@ -259,6 +259,12 @@ do_recv(Sock, Bs) -> <p>The optional <c><anno>Timeout</anno></c> parameter specifies a time-out in milliseconds. Defaults to <c>infinity</c>.</p> <note> + <p>Keep in mind that if the underlying OS <c>connect()</c> call returns + a timeout, <c>gen_tcp:connect</c> will also return a timeout + (i.e. <c>{error, etimedout}</c>), even if a larger <c>Timeout</c> was + specified.</p> + </note> + <note> <p>The default values for options specified to <c>connect</c> can be affected by the Kernel configuration parameter <c>inet_default_connect_options</c>. For details, see diff --git a/lib/kernel/doc/src/inet.xml b/lib/kernel/doc/src/inet.xml index 5e33bbc3ff..d4678ca5db 100644 --- a/lib/kernel/doc/src/inet.xml +++ b/lib/kernel/doc/src/inet.xml @@ -1010,13 +1010,34 @@ get_tcpi_sacked(Sock) -> <marker id="option-linger"></marker> </item> <tag><c>{linger, {true|false, Seconds}}</c></tag> - <item> + <item> <p>Determines the time-out, in seconds, for flushing unsent data - in the <c>close/1</c> socket call. If the first component of - the value tuple is <c>false</c>, the second is ignored. This - means that <c>close/1</c> returns immediately, not waiting - for data to be flushed. Otherwise, the second component is - the flushing time-out, in seconds.</p> + in the <c>close/1</c> socket call. </p> + <p>The first component is if linger is enabled, the second component + is the flushing time-out, in seconds. There are 3 alternatives:</p> + <taglist> + <tag><c>{false, _}</c></tag> + <item> + <p>close/1 or shutdown/2 returns immediately, + not waiting for data to be flushed, with closing + happening in the background.</p> + </item> + <tag><c>{true, 0}</c></tag> + <item> + <p>Aborts the connection when it is closed. + Discards any data still remaining in the send buffers + and sends RST to the peer.</p> + <p>This avoids TCP's TIME_WAIT state, but leaves open + the possibility that another "incarnation" of this connection + being created.</p> + </item> + <tag><c>{true, Time} when Time > 0</c></tag> + <item> + <p>close/1 or shutdown/2 will not return until + all queued messages for the socket have been successfully + sent or the linger timeout (Time) has been reached.</p> + </item> + </taglist> </item> <tag><c>{low_msgq_watermark, Size}</c></tag> <item> diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml index dbd83e1a6e..7f9609d5c1 100644 --- a/lib/kernel/doc/src/kernel_app.xml +++ b/lib/kernel/doc/src/kernel_app.xml @@ -369,6 +369,14 @@ MaxT = TickTime + TickTime / 4</code> performed. This option ensures that <c>global</c> is synchronized.</p> </item> + <tag><c>start_distribution = true | false</c></tag> + <item> + <p>Starts all distribution services, such as <c>rpc</c>, + <c>global</c>, and <c>net_kernel</c> if the parameter is + <c>true</c>. This parameter is to be set to <c>false</c> + for systems who want to disable all distribution functionality.</p> + <p>Defaults to <c>true</c>.</p> + </item> <tag><c>start_dist_ac = true | false</c></tag> <item> <p>Starts the <c>dist_ac</c> server if the parameter is diff --git a/lib/kernel/doc/src/logger.xml b/lib/kernel/doc/src/logger.xml index 5bdfcf91db..2a060ce1f9 100644 --- a/lib/kernel/doc/src/logger.xml +++ b/lib/kernel/doc/src/logger.xml @@ -1351,5 +1351,3 @@ logger:set_proxy_config(maps:merge(Old, Config)). </p> </section> </erlref> - - diff --git a/lib/kernel/doc/src/logger_chapter.xml b/lib/kernel/doc/src/logger_chapter.xml index bfd0acf634..1aa4b7a3a2 100644 --- a/lib/kernel/doc/src/logger_chapter.xml +++ b/lib/kernel/doc/src/logger_chapter.xml @@ -212,13 +212,13 @@ <pre>fun((<seealso marker="logger#type-report"><c>logger:report()</c></seealso>,<seealso marker="logger#type-report_cb_config"><c>logger:report_cb_config()</c></seealso>) -> <seealso marker="stdlib:unicode#type-chardata"><c>unicode:chardata()</c></seealso>) </pre> <p>The fun must obey the <c>depth</c> and <c>chars_limit</c> - parameters provided in the second argument, as the formatter can - not do anything useful of these parameters with the returned - string. The extra data also contains a field named - <c>single_line</c>, indicating if the printed log message may - contain line breaks or not. This variant is used when the - formatting of the report depends on the size or single line - parameters.</p> + parameters provided in the second argument, as the formatter + cannot do anything useful of these parameters with the + returned string. The extra data also contains a field named + <c>single_line</c>, indicating if the printed log message may + contain line breaks or not. This variant is used when the + formatting of the report depends on the size or single line + parameters.</p> <p>Example, format string and arguments:</p> <code>logger:error("The file does not exist: ~ts",[Filename])</code> <p>Example, string:</p> @@ -718,7 +718,7 @@ logger:debug(#{got => connection_request, id => Id, state => State}, </seealso></pre> <p>For all other values of <c>HandlerId</c>, this entry adds a new handler, equivalent to calling</p> - <pre><seealso marker="logger:add_handler/3"> + <pre><seealso marker="logger#add_handler/3"> logger:add_handler(HandlerId, Module, HandlerConfig) </seealso></pre> <p>Multiple entries of this type are allowed.</p></item> diff --git a/lib/kernel/doc/src/logger_disk_log_h.xml b/lib/kernel/doc/src/logger_disk_log_h.xml index 5b2374690e..aa577f3c62 100644 --- a/lib/kernel/doc/src/logger_disk_log_h.xml +++ b/lib/kernel/doc/src/logger_disk_log_h.xml @@ -66,7 +66,7 @@ corresponds to the <c>name</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to the same name as the handler identity, in the current directory.</p> @@ -77,7 +77,7 @@ corresponds to the <c>type</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>wrap</c>.</p> </item> @@ -88,7 +88,7 @@ corresponds to the <c>MaxNoFiles</c> element in the <c>size</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>10</c>.</p> <p>The setting has no effect on a halt log.</p> @@ -101,7 +101,7 @@ corresponds to the <c>MaxNoBytes</c> element in the <c>size</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>1048576</c> bytes for a wrap log, and <c>infinity</c> for a halt log.</p> diff --git a/lib/kernel/doc/src/logger_std_h.xml b/lib/kernel/doc/src/logger_std_h.xml index 5ed1a2f210..5ac4f58d12 100644 --- a/lib/kernel/doc/src/logger_std_h.xml +++ b/lib/kernel/doc/src/logger_std_h.xml @@ -58,7 +58,7 @@ <tag><marker id="type"/><c>type = standard_io | standard_error | file</c></tag> <item> <p>Specifies the log destination.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>standard_io</c>, unless parameter <seealso marker="#file"><c>file</c></seealso> is @@ -68,7 +68,7 @@ <item> <p>This specifies the name of the log file when the handler is of type <c>file</c>.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to the same name as the handler identity, in the current directory.</p> @@ -93,9 +93,9 @@ or <c>{delayed_write,Size,Delay}</c> is found in the list, <c>delayed_write</c> is added.</item> </list> - <p>Log files are always UTF-8 encoded. The encoding can not be + <p>Log files are always UTF-8 encoded. The encoding cannot be changed by setting the mode <c>{encoding,Encoding}</c>.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>[raw,append,delayed_write]</c>.</p> </item> diff --git a/lib/kernel/examples/gen_tcp_dist/Makefile b/lib/kernel/examples/gen_tcp_dist/Makefile index 65513a1729..0c916835ea 100644 --- a/lib/kernel/examples/gen_tcp_dist/Makefile +++ b/lib/kernel/examples/gen_tcp_dist/Makefile @@ -2,9 +2,7 @@ RM=rm -f CP=cp EBIN=ebin ERLC=erlc -# Works if building in open source source tree -KERNEL_INCLUDE=$(ERL_TOP)/lib/kernel/include -ERLCFLAGS+= -W -I$(KERNEL_INCLUDE) +ERLCFLAGS+= -W MODULES=gen_tcp_dist diff --git a/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl b/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl index 98554ed805..cda4c470f9 100644 --- a/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl +++ b/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl @@ -53,10 +53,10 @@ -import(error_logger,[error_msg/2]). --include("net_address.hrl"). +-include_lib("kernel/include/net_address.hrl"). --include("dist.hrl"). --include("dist_util.hrl"). +-include_lib("kernel/include/dist.hrl"). +-include_lib("kernel/include/dist_util.hrl"). %% ------------------------------------------------------------ %% Select this protocol based on node name @@ -679,7 +679,14 @@ dist_cntrlr_setup_loop(Socket, TickHandler, Sup) -> %% From now on we execute on normal priority process_flag(priority, normal), erlang:dist_ctrl_get_data_notification(DHandle), - dist_cntrlr_output_loop(DHandle, Socket) + case init:get_argument(gen_tcp_dist_output_loop) of + error -> + dist_cntrlr_output_loop(DHandle, Socket); + {ok, [[ModStr, FuncStr]]} -> % For testing... + apply(list_to_atom(ModStr), + list_to_atom(FuncStr), + [DHandle, Socket]) + end end. %% We use active 10 for good throughput while still diff --git a/lib/kernel/include/dist.hrl b/lib/kernel/include/dist.hrl index 003852f1b0..f06fc328d7 100644 --- a/lib/kernel/include/dist.hrl +++ b/lib/kernel/include/dist.hrl @@ -42,6 +42,9 @@ -define(DFLAG_BIG_CREATION, 16#40000). -define(DFLAG_SEND_SENDER, 16#80000). -define(DFLAG_BIG_SEQTRACE_LABELS, 16#100000). +%% -define(DFLAG_NO_MAGIC, 16#200000). %% Used internally only +-define(DFLAG_EXIT_PAYLOAD, 16#400000). +-define(DFLAG_FRAGMENTS, 16#800000). %% Also update dflag2str() in ../src/dist_util.erl %% when adding flags... diff --git a/lib/kernel/src/Makefile b/lib/kernel/src/Makefile index 3d1506ea08..fcb599859b 100644 --- a/lib/kernel/src/Makefile +++ b/lib/kernel/src/Makefile @@ -123,7 +123,6 @@ MODULES = \ logger_server \ logger_simple_h \ logger_sup \ - net \ net_adm \ net_kernel \ os \ @@ -176,8 +175,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif -ERL_COMPILE_FLAGS += -I../include -Werror +ERL_COMPILE_FLAGS += -I../include # ---------------------------------------------------- # Targets diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index 9a8091fb2e..7715dca7c6 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -537,14 +537,12 @@ check_conf_data(ConfData) when is_list(ConfData) -> {AppName, List} when is_atom(AppName), is_list(List) -> case lists:keymember(AppName, 1, ConfDataRem) of true -> - ?LOG_WARNING("duplicate application config: " ++ atom_to_list(AppName)); + {error, "duplicate application config: " ++ atom_to_list(AppName)}; false -> - ok - end, - - case check_para(List, AppName) of - ok -> check_conf_data(ConfDataRem); - Error -> Error + case check_para(List, AppName) of + ok -> check_conf_data(ConfDataRem); + Error -> Error + end end; {AppName, List} when is_list(List) -> ErrMsg = "application: " @@ -570,15 +568,14 @@ check_para([], _AppName) -> check_para([{Para, Val} | ParaList], AppName) when is_atom(Para) -> case lists:keymember(Para, 1, ParaList) of true -> - ?LOG_WARNING("application: " ++ atom_to_list(AppName) ++ - "; duplicate parameter: " ++ atom_to_list(Para)); + ErrMsg = "application: " ++ atom_to_list(AppName) + ++ "; duplicate parameter: " ++ atom_to_list(Para), + {error, ErrMsg}; false -> - ok - end, - - case check_para_value(Para, Val, AppName) of - ok -> check_para(ParaList, AppName); - {error, _} = Error -> Error + case check_para_value(Para, Val, AppName) of + ok -> check_para(ParaList, AppName); + {error, _} = Error -> Error + end end; check_para([{Para, _Val} | _ParaList], AppName) -> {error, "application: " ++ atom_to_list(AppName) ++ "; invalid parameter name: " ++ diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl index ecc022b28d..09ed31f10c 100644 --- a/lib/kernel/src/dist_util.erl +++ b/lib/kernel/src/dist_util.erl @@ -116,6 +116,10 @@ dflag2str(?DFLAG_SEND_SENDER) -> "SEND_SENDER"; dflag2str(?DFLAG_BIG_SEQTRACE_LABELS) -> "BIG_SEQTRACE_LABELS"; +dflag2str(?DFLAG_EXIT_PAYLOAD) -> + "EXIT_PAYLOAD"; +dflag2str(?DFLAG_FRAGMENTS) -> + "FRAGMENTS"; dflag2str(_) -> "UNKNOWN". diff --git a/lib/kernel/src/erl_distribution.erl b/lib/kernel/src/erl_distribution.erl index 0bec78e938..cdb2d2f1f6 100644 --- a/lib/kernel/src/erl_distribution.erl +++ b/lib/kernel/src/erl_distribution.erl @@ -21,6 +21,8 @@ -behaviour(supervisor). +-include_lib("kernel/include/logger.hrl"). + -export([start_link/0,start_link/2,init/1,start/1,stop/0]). -define(DBG,erlang:display([?MODULE,?LINE])). @@ -83,6 +85,10 @@ do_start_link([{Arg,Flag}|T]) -> case init:get_argument(Arg) of {ok,[[Name]]} -> start_link([list_to_atom(Name),Flag|ticktime()], true); + {ok,[[Name]|_Rest]} -> + ?LOG_WARNING("Multiple -~p given to erl, using the first, ~p", + [Arg, Name]), + start_link([list_to_atom(Name),Flag|ticktime()], true); _ -> do_start_link(T) end; diff --git a/lib/kernel/src/erts_debug.erl b/lib/kernel/src/erts_debug.erl index 1270de4144..e6a30d0b92 100644 --- a/lib/kernel/src/erts_debug.erl +++ b/lib/kernel/src/erts_debug.erl @@ -33,10 +33,12 @@ -export([breakpoint/2, disassemble/1, display/1, dist_ext_to_term/2, flat_size/1, get_internal_state/1, instructions/0, + interpreter_size/0, map_info/1, same/2, set_internal_state/2, size_shared/1, copy_shared/1, dirty_cpu/2, dirty_io/2, dirty/3, lcnt_control/1, lcnt_control/2, lcnt_collect/0, lcnt_clear/0, - lc_graph/0, lc_graph_to_dot/2, lc_graph_merge/2]). + lc_graph/0, lc_graph_to_dot/2, lc_graph_merge/2, + alloc_blocks_size/1]). -spec breakpoint(MFA, Flag) -> non_neg_integer() when MFA :: {Module :: module(), @@ -117,6 +119,11 @@ get_internal_state(_) -> instructions() -> erlang:nif_error(undef). +-spec interpreter_size() -> pos_integer(). + +interpreter_size() -> + erlang:nif_error(undef). + -spec ic(F) -> Result when F :: function(), Result :: term(). @@ -495,3 +502,58 @@ lcg_print_locks(Out, [LastLock]) -> lcg_print_locks(Out, [Lock | Rest]) -> io:format(Out, "~w,\n", [Lock]), lcg_print_locks(Out, Rest). + + +%% Returns the amount of memory allocated by the given allocator type. +-spec alloc_blocks_size(Type) -> non_neg_integer() | undefined when + Type :: atom(). + +alloc_blocks_size(Type) -> + Allocs = erlang:system_info(alloc_util_allocators), + Sizes = erlang:system_info({allocator_sizes, Allocs}), + alloc_blocks_size_1(Sizes, Type, 0). + +alloc_blocks_size_1([], _Type, 0) -> + undefined; +alloc_blocks_size_1([{_Type, false} | Rest], Type, Acc) -> + alloc_blocks_size_1(Rest, Type, Acc); +alloc_blocks_size_1([{Type, Instances} | Rest], Type, Acc0) -> + F = fun ({instance, _, L}, Acc) -> + MBCSPool = case lists:keyfind(mbcs_pool, 1, L) of + {_, Pool} -> Pool; + false -> [] + end, + {_,MBCS} = lists:keyfind(mbcs, 1, L), + {_,SBCS} = lists:keyfind(sbcs, 1, L), + Acc + + sum_block_sizes(MBCSPool) + + sum_block_sizes(MBCS) + + sum_block_sizes(SBCS) + end, + alloc_blocks_size_1(Rest, Type, lists:foldl(F, Acc0, Instances)); +alloc_blocks_size_1([{_Type, Instances} | Rest], Type, Acc0) -> + F = fun ({instance, _, L}, Acc) -> + Acc + sum_foreign_sizes(Type, L) + end, + alloc_blocks_size_1(Rest, Type, lists:foldl(F, Acc0, Instances)); +alloc_blocks_size_1([], _Type, Acc) -> + Acc. + +sum_foreign_sizes(Type, L) -> + case lists:keyfind(mbcs_pool, 1, L) of + {_,Pool} -> + {_,ForeignBlocks} = lists:keyfind(foreign_blocks, 1, Pool), + case lists:keyfind(Type, 1, ForeignBlocks) of + {_,TypeSizes} -> sum_block_sizes(TypeSizes); + false -> 0 + end; + _ -> + 0 + end. + +sum_block_sizes(Blocks) -> + lists:foldl( + fun({blocks_size, Sz,_,_}, Sz0) -> Sz0+Sz; + ({blocks_size, Sz}, Sz0) -> Sz0+Sz; + (_, Sz) -> Sz + end, 0, Blocks). diff --git a/lib/kernel/src/gen_tcp.erl b/lib/kernel/src/gen_tcp.erl index 7f7833ec23..5d4764f8ff 100644 --- a/lib/kernel/src/gen_tcp.erl +++ b/lib/kernel/src/gen_tcp.erl @@ -156,7 +156,7 @@ connect(Address, Port, Opts) -> Options :: [connect_option()], Timeout :: timeout(), Socket :: socket(), - Reason :: inet:posix(). + Reason :: timeout | inet:posix(). connect(Address, Port, Opts, Time) -> Timer = inet:start_timer(Time), @@ -220,7 +220,7 @@ listen(Port, Opts0) -> -spec accept(ListenSocket) -> {ok, Socket} | {error, Reason} when ListenSocket :: socket(), Socket :: socket(), - Reason :: closed | timeout | system_limit | inet:posix(). + Reason :: closed | system_limit | inet:posix(). accept(S) -> case inet_db:lookup_socket(S) of @@ -312,7 +312,7 @@ recv(S, Length) when is_port(S) -> Length :: non_neg_integer(), Timeout :: timeout(), Packet :: string() | binary() | HttpPacket, - Reason :: closed | inet:posix(), + Reason :: closed | timeout | inet:posix(), HttpPacket :: term(). recv(S, Length, Time) when is_port(S) -> diff --git a/lib/kernel/src/gen_udp.erl b/lib/kernel/src/gen_udp.erl index d6e8652e77..fad7b2f887 100644 --- a/lib/kernel/src/gen_udp.erl +++ b/lib/kernel/src/gen_udp.erl @@ -95,7 +95,7 @@ -spec open(Port) -> {ok, Socket} | {error, Reason} when Port :: inet:port_number(), Socket :: socket(), - Reason :: inet:posix(). + Reason :: system_limit | inet:posix(). open(Port) -> open(Port, []). @@ -112,7 +112,7 @@ open(Port) -> | {bind_to_device, binary()} | option(), Socket :: socket(), - Reason :: inet:posix(). + Reason :: system_limit | inet:posix(). open(Port, Opts0) -> {Mod, Opts} = inet:udp_module(Opts0), @@ -186,7 +186,7 @@ recv(S,Len) when is_port(S), is_integer(Len) -> Port :: inet:port_number(), AncData :: inet:ancillary_data(), Packet :: string() | binary(), - Reason :: not_owner | inet:posix(). + Reason :: not_owner | timeout | inet:posix(). recv(S,Len,Time) when is_port(S) -> case inet_db:lookup_socket(S) of diff --git a/lib/kernel/src/global.erl b/lib/kernel/src/global.erl index a38522eb5c..3875074d74 100644 --- a/lib/kernel/src/global.erl +++ b/lib/kernel/src/global.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -50,10 +50,9 @@ %% This is for backward compatibility only; the functionality is broken. -define(WARN_DUPLICATED_NAME, global_multi_name_action). -%% Undocumented Kernel variable. Set this to 0 (zero) to get the old -%% behaviour. +%% Undocumented Kernel variable. -define(N_CONNECT_RETRIES, global_connect_retries). --define(DEFAULT_N_CONNECT_RETRIES, 5). +-define(DEFAULT_N_CONNECT_RETRIES, 0). %%% In certain places in the server, calling io:format hangs everything, %%% so we'd better use erlang:display/1. @@ -125,16 +124,12 @@ %%% There are also ETS tables used for bookkeeping of locks and names %%% (the first position is the key): %%% -%%% global_locks (set): {ResourceId, LockRequesterId, [{Pid,RPid,ref()]} +%%% global_locks (set): {ResourceId, LockRequesterId, [{Pid,ref()]} %%% Pid is locking ResourceId, ref() is the monitor ref. -%%% RPid =/= Pid if there is an extra process calling erlang:monitor(). -%%% global_names (set): {Name, Pid, Method, RPid, ref()} +%%% global_names (set): {Name, Pid, Method, ref()} %%% Registered names. ref() is the monitor ref. -%%% RPid =/= Pid if there is an extra process calling erlang:monitor(). %%% global_names_ext (set): {Name, Pid, RegNode} %%% External registered names (C-nodes). -%%% (The RPid:s can be removed when/if erlang:monitor() returns before -%%% trying to connect to the other node.) %%% %%% Helper tables: %%% global_pid_names (bag): {Pid, Name} | {ref(), Name} @@ -310,7 +305,7 @@ re_register_name(Name, Pid, Method0) when is_pid(Pid) -> -spec registered_names() -> [Name] when Name :: term(). registered_names() -> - MS = ets:fun2ms(fun({Name,_Pid,_M,_RP,_R}) -> Name end), + MS = ets:fun2ms(fun({Name,_Pid,_M,_R}) -> Name end), ets:select(global_names, MS). %%----------------------------------------------------------------- @@ -1235,7 +1230,7 @@ ins_name_ext(Name, Pid, Method, RegNode, FromPidOrNode, ExtraInfo, S0) -> where(Name) -> case ets:lookup(global_names, Name) of - [{_Name, Pid, _Method, _RPid, _Ref}] -> + [{_Name, Pid, _Method, _Ref}] -> if node(Pid) == node() -> case is_process_alive(Pid) of true -> Pid; @@ -1272,10 +1267,10 @@ can_set_lock({ResourceId, LockRequesterId}) -> end. insert_lock({ResourceId, LockRequesterId}=Id, Pid, PidRefs, S) -> - {RPid, Ref} = do_monitor(Pid), + Ref = erlang:monitor(process, Pid), true = ets:insert(global_pid_ids, {Pid, ResourceId}), true = ets:insert(global_pid_ids, {Ref, ResourceId}), - Lock = {ResourceId, LockRequesterId, [{Pid,RPid,Ref} | PidRefs]}, + Lock = {ResourceId, LockRequesterId, [{Pid,Ref} | PidRefs]}, true = ets:insert(global_locks, Lock), trace_message(S, {ins_lock, node(Pid)}, [Id, Pid]). @@ -1293,10 +1288,9 @@ handle_del_lock({ResourceId, LockReqId}, Pid, S0) -> _ -> S0 end. -remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,RPid,Ref}], Down, S0) -> +remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,Ref}], Down, S0) -> ?trace({remove_lock_1, {id,ResourceId},{pid,Pid}}), true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), true = ets:delete(global_locks, ResourceId), true = ets:delete_object(global_pid_ids, {Pid, ResourceId}), true = ets:delete_object(global_pid_ids, {Ref, ResourceId}), @@ -1309,9 +1303,8 @@ remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,RPid,Ref}], Down, S0) -> remove_lock(ResourceId, LockRequesterId, Pid, PidRefs0, _Down, S) -> ?trace({remove_lock_2, {id,ResourceId},{pid,Pid}}), PidRefs = case lists:keyfind(Pid, 1, PidRefs0) of - {Pid, RPid, Ref} -> + {Pid, Ref} -> true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), true = ets:delete_object(global_pid_ids, {Ref, ResourceId}), lists:keydelete(Pid, 1, PidRefs0); @@ -1324,11 +1317,6 @@ remove_lock(ResourceId, LockRequesterId, Pid, PidRefs0, _Down, S) -> trace_message(S, {rem_lock, node(Pid)}, [{ResourceId, LockRequesterId}, Pid]). -kill_monitor_proc(Pid, Pid) -> - ok; -kill_monitor_proc(RPid, _Pid) -> - exit(RPid, kill). - do_ops(Ops, ConnNode, Names_ext, ExtraInfo, S0) -> ?trace({do_ops, {ops,Ops}}), @@ -1394,8 +1382,8 @@ sync_other(Node, N) -> % exit(normal). insert_global_name(Name, Pid, Method, FromPidOrNode, ExtraInfo, S) -> - {RPid, Ref} = do_monitor(Pid), - true = ets:insert(global_names, {Name, Pid, Method, RPid, Ref}), + Ref = erlang:monitor(process, Pid), + true = ets:insert(global_names, {Name, Pid, Method, Ref}), true = ets:insert(global_pid_names, {Pid, Name}), true = ets:insert(global_pid_names, {Ref, Name}), case lock_still_set(FromPidOrNode, ExtraInfo, S) of @@ -1437,7 +1425,7 @@ extra_info(Tag, ExtraInfo) -> del_name(Ref, S) -> NameL = [Name || {_, Name} <- ets:lookup(global_pid_names, Ref), - {_, _Pid, _Method, _RPid, Ref1} <- + {_, _Pid, _Method, Ref1} <- ets:lookup(global_names, Name), Ref1 =:= Ref], case NameL of @@ -1450,24 +1438,23 @@ del_name(Ref, S) -> %% Keeps the entry in global_names for whereis_name/1. delete_global_name_keep_pid(Name, S) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, RPid, Ref}] -> - delete_global_name2(Name, Pid, RPid, Ref, S); + [{Name, Pid, _Method, Ref}] -> + delete_global_name2(Name, Pid, Ref, S); [] -> S end. delete_global_name2(Name, S) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, RPid, Ref}] -> + [{Name, Pid, _Method, Ref}] -> true = ets:delete(global_names, Name), - delete_global_name2(Name, Pid, RPid, Ref, S); + delete_global_name2(Name, Pid, Ref, S); [] -> S end. -delete_global_name2(Name, Pid, RPid, Ref, S) -> +delete_global_name2(Name, Pid, Ref, S) -> true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), delete_global_name(Name, Pid), ?trace({delete_global_name,{item,Name},{pid,Pid}}), true = ets:delete_object(global_pid_names, {Pid, Name}), @@ -1929,9 +1916,9 @@ reset_node_state(Node) -> %% from the same partition. exchange_names([{Name, Pid, Method} | Tail], Node, Ops, Res) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, _RPid2, _Ref2}] -> + [{Name, Pid, _Method, _Ref2}] -> exchange_names(Tail, Node, Ops, Res); - [{Name, Pid2, Method2, _RPid2, _Ref2}] when node() < Node -> + [{Name, Pid2, Method2, _Ref2}] when node() < Node -> %% Name clash! Add the result of resolving to Res(olved). %% We know that node(Pid) =/= node(), so we don't %% need to link/unlink to Pid. @@ -1960,7 +1947,7 @@ exchange_names([{Name, Pid, Method} | Tail], Node, Ops, Res) -> Op = {delete, Name}, exchange_names(Tail, Node, [Op | Ops], [Op | Res]) end; - [{Name, _Pid2, _Method, _RPid, _Ref}] -> + [{Name, _Pid2, _Method, _Ref}] -> %% The other node will solve the conflict. exchange_names(Tail, Node, Ops, Res); _ -> @@ -2036,7 +2023,7 @@ pid_is_locking(Pid, PidRefs) -> delete_lock(Ref, S0) -> Locks = pid_locks(Ref), F = fun({ResourceId, LockRequesterId, PidRefs}, S) -> - {Pid, _RPid, Ref} = lists:keyfind(Ref, 3, PidRefs), + {Pid, Ref} = lists:keyfind(Ref, 2, PidRefs), remove_lock(ResourceId, LockRequesterId, Pid, PidRefs, true, S) end, lists:foldl(F, S0, Locks). @@ -2046,10 +2033,10 @@ pid_locks(Ref) -> ets:lookup(global_locks, ResourceId) end, ets:lookup(global_pid_ids, Ref)), [Lock || Lock = {_Id, _Req, PidRefs} <- L, - rpid_is_locking(Ref, PidRefs)]. + ref_is_locking(Ref, PidRefs)]. -rpid_is_locking(Ref, PidRefs) -> - lists:keyfind(Ref, 3, PidRefs) =/= false. +ref_is_locking(Ref, PidRefs) -> + lists:keyfind(Ref, 2, PidRefs) =/= false. handle_nodedown(Node, S) -> %% DOWN signals from monitors have removed locks and registered names. @@ -2062,7 +2049,7 @@ handle_nodedown(Node, S) -> get_names() -> ets:select(global_names, - ets:fun2ms(fun({Name, Pid, Method, _RPid, _Ref}) -> + ets:fun2ms(fun({Name, Pid, Method, _Ref}) -> {Name, Pid, Method} end)). @@ -2205,24 +2192,6 @@ unexpected_message(Message, What) -> %%% Utilities -%% When/if erlang:monitor() returns before trying to connect to the -%% other node this function can be removed. -do_monitor(Pid) -> - case (node(Pid) =:= node()) orelse lists:member(node(Pid), nodes()) of - true -> - %% Assume the node is still up - {Pid, erlang:monitor(process, Pid)}; - false -> - F = fun() -> - Ref = erlang:monitor(process, Pid), - receive - {'DOWN', Ref, process, Pid, _Info} -> - exit(normal) - end - end, - erlang:spawn_monitor(F) - end. - intersection(_, []) -> []; intersection(L1, L2) -> diff --git a/lib/kernel/src/inet_config.erl b/lib/kernel/src/inet_config.erl index 9f76360b8b..e771461b65 100644 --- a/lib/kernel/src/inet_config.erl +++ b/lib/kernel/src/inet_config.erl @@ -98,7 +98,7 @@ init() -> {win32,WinType} -> win32_load_from_registry(WinType); _ -> - error("can not read win32 system registry~n", []) + error("cannot read win32 system registry~n", []) end end, CfgFiles), diff --git a/lib/kernel/src/inet_dns.erl b/lib/kernel/src/inet_dns.erl index f1f58bc872..6c98d2aab7 100644 --- a/lib/kernel/src/inet_dns.erl +++ b/lib/kernel/src/inet_dns.erl @@ -699,7 +699,7 @@ encode_labels(Bin, Comp0, Pos, [L|Ls]=Labels) none -> Comp = if Pos < (3 bsl 14) -> %% Just in case - compression - %% pointers can not reach further + %% pointers cannot reach further gb_trees:insert(Labels, Pos, Comp0); true -> Comp0 end, diff --git a/lib/kernel/src/inet_tcp_dist.erl b/lib/kernel/src/inet_tcp_dist.erl index c37212b0f9..c5a114a9ef 100644 --- a/lib/kernel/src/inet_tcp_dist.erl +++ b/lib/kernel/src/inet_tcp_dist.erl @@ -212,6 +212,7 @@ do_accept(Driver, Kernel, AcceptPid, Socket, MyNode, Allowed, SetupTime) -> [{active, true}, {deliver, port}, {packet, 4}, + binary, nodelay()]) end, f_getll = fun(S) -> diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 6bd5518a37..8fe6bdd1ca 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -74,7 +74,6 @@ logger_simple_h, logger_std_h, logger_sup, - net, net_adm, net_kernel, os, diff --git a/lib/kernel/src/kernel.erl b/lib/kernel/src/kernel.erl index c68d04e279..bfa091a036 100644 --- a/lib/kernel/src/kernel.erl +++ b/lib/kernel/src/kernel.erl @@ -68,7 +68,7 @@ config_change(Changed, New, Removed) -> %%% auth, ...) ...) %%% %%% The rectangular boxes are supervisors. All supervisors except -%%% for kernel_safe_sup terminates the enitre erlang node if any of +%%% for kernel_safe_sup terminates the entire erlang node if any of %%% their children dies. Any child that can't be restarted in case %%% of failure must be placed under one of these supervisors. Any %%% other child must be placed under safe_sup. These children may @@ -145,26 +145,11 @@ init([]) -> {ok, {SupFlags, [Code, File, StdError, User, LoggerSup, Config, RefC, SafeSup]}}; _ -> - Rpc = #{id => rex, - start => {rpc, start_link, []}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [rpc]}, - - Global = #{id => global_name_server, - start => {global, start_link, []}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [global]}, - - GlGroup = #{id => global_group, - start => {global_group,start_link,[]}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [global_group]}, + DistChildren = + case application:get_env(kernel, start_distribution) of + {ok, false} -> []; + _ -> start_distribution() + end, InetDb = #{id => inet_db, start => {inet_db, start_link, []}, @@ -173,13 +158,6 @@ init([]) -> type => worker, modules => [inet_db]}, - NetSup = #{id => net_sup, - start => {erl_distribution, start_link, []}, - restart => permanent, - shutdown => infinity, - type => supervisor, - modules => [erl_distribution]}, - SigSrv = #{id => erl_signal_server, start => {gen_event, start_link, [{local, erl_signal_server}]}, restart => permanent, @@ -187,14 +165,11 @@ init([]) -> type => worker, modules => dynamic}, - DistAC = start_dist_ac(), - Timer = start_timer(), {ok, {SupFlags, - [Code, Rpc, Global, InetDb | DistAC] ++ - [NetSup, GlGroup, File, SigSrv, - StdError, User, Config, RefC, SafeSup, LoggerSup] ++ Timer}} + [Code, InetDb | DistChildren] ++ + [File, SigSrv, StdError, User, Config, RefC, SafeSup, LoggerSup] ++ Timer}} end; init(safe) -> SupFlags = #{strategy => one_for_one, @@ -213,6 +188,39 @@ init(safe) -> {ok, {SupFlags, Boot ++ DiskLog ++ Pg2}}. +start_distribution() -> + Rpc = #{id => rex, + start => {rpc, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [rpc]}, + + Global = #{id => global_name_server, + start => {global, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global]}, + + DistAC = start_dist_ac(), + + NetSup = #{id => net_sup, + start => {erl_distribution, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [erl_distribution]}, + + GlGroup = #{id => global_group, + start => {global_group,start_link,[]}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [global_group]}, + + [Rpc, Global | DistAC] ++ [NetSup, GlGroup]. + start_dist_ac() -> Spec = [#{id => dist_ac, start => {dist_ac,start_link,[]}, diff --git a/lib/kernel/src/net.erl b/lib/kernel/src/net.erl deleted file mode 100644 index 2d0ae2ed0c..0000000000 --- a/lib/kernel/src/net.erl +++ /dev/null @@ -1,40 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% --module(net). - -%% Various network functions, kept here for compatibility - --export([call/4, - cast/4, - broadcast/3, - ping/1, - relay/1, - sleep/1]). - --deprecated(module). - -call(N,M,F,A) -> rpc:call(N,M,F,A). -cast(N,M,F,A) -> rpc:cast(N,M,F,A). -broadcast(M,F,A) -> rpc:eval_everywhere(M,F,A). -ping(Node) -> net_adm:ping(Node). -sleep(T) -> receive after T -> ok end. -relay(X) -> slave:relay(X). - - diff --git a/lib/kernel/src/net_kernel.erl b/lib/kernel/src/net_kernel.erl index a9dc77837e..83d3b4b5e1 100644 --- a/lib/kernel/src/net_kernel.erl +++ b/lib/kernel/src/net_kernel.erl @@ -1126,14 +1126,22 @@ do_disconnect(Node, State) -> {false, State} end. - disconnect_pid(Pid, State) -> exit(Pid, disconnect), + + %% This code used to only use exit + recv 'EXIT' to sync, + %% but since OTP-22 links are no longer broken atomically + %% so the exit message below can arrive before any remaining + %% exit messages have killed the distribution port + Ref = erlang:monitor(process, Pid), %% Sync wait for connection to die!!! receive - {'EXIT',Pid,Reason} -> - {_,State1} = handle_exit(Pid, Reason, State), - {true, State1} + {'DOWN',Ref,_,_,_} -> + receive + {'EXIT',Pid,Reason} -> + {_,State1} = handle_exit(Pid, Reason, State), + {true, State1} + end end. %% @@ -1433,7 +1441,7 @@ validate_hostname([$@|HostPart] = Host) -> end. valid_name_head(Head) -> - {ok, MP} = re:compile("^[0-9A-Za-z_\\-]*$", [unicode]), + {ok, MP} = re:compile("^[0-9A-Za-z_\\-]+$", [unicode]), case re:run(Head, MP) of {match, _} -> true; diff --git a/lib/kernel/src/user_drv.erl b/lib/kernel/src/user_drv.erl index 08286dd476..69ff8e7971 100644 --- a/lib/kernel/src/user_drv.erl +++ b/lib/kernel/src/user_drv.erl @@ -120,7 +120,7 @@ server1(Iport, Oport, Shell) -> {Curr,Shell1} = case init:get_argument(remsh) of {ok,[[Node]]} -> - ANode = list_to_atom(Node), + ANode = list_to_atom(append_hostname(Node)), RShell = {ANode,shell,start,[]}, RGr = group:start(self(), RShell, rem_sh_opts(ANode)), {RGr,RShell}; @@ -139,6 +139,12 @@ server1(Iport, Oport, Shell) -> %% Enter the server loop. server_loop(Iport, Oport, Curr, User, Gr, {false, queue:new()}). +append_hostname(Node) -> + case string:find(Node, "@") of + nomatch -> Node ++ string:find(atom_to_list(node()), "@"); + _ -> Node + end. + rem_sh_opts(Node) -> [{expand_fun,fun(B)-> rpc:call(Node,edlin_expand,expand,[B]) end}]. diff --git a/lib/kernel/test/Makefile b/lib/kernel/test/Makefile index 8a6ffe7e72..6763a04d9f 100644 --- a/lib/kernel/test/Makefile +++ b/lib/kernel/test/Makefile @@ -57,6 +57,7 @@ MODULES= \ prim_file_SUITE \ ram_file_SUITE \ gen_tcp_api_SUITE \ + gen_tcp_dist \ gen_tcp_echo_SUITE \ gen_tcp_misc_SUITE \ gen_udp_SUITE \ @@ -130,11 +131,17 @@ ERL_COMPILE_FLAGS += EBIN = . +TARGETS = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) + + # ---------------------------------------------------- # Targets # ---------------------------------------------------- -make_emakefile: +gen_tcp_dist.erl: ../examples/gen_tcp_dist/src/gen_tcp_dist.erl + cp $< $@ + +make_emakefile: $(ERL_FILES) $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) '*_SUITE_make' \ > $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) \ @@ -150,6 +157,9 @@ clean: docs: +targets: $(TARGETS) + + # ---------------------------------------------------- # Release Target # ---------------------------------------------------- diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl index 94d7c17712..1ab554db7c 100644 --- a/lib/kernel/test/application_SUITE.erl +++ b/lib/kernel/test/application_SUITE.erl @@ -2020,18 +2020,11 @@ set_env_errors(Conf) when is_list(Conf) -> "application: kernel; erroneous parameter: distributed" = badarg_msg(fun() -> application:set_env([{kernel, [{distributed, config}]}]) end), - %% This will raise in the future - ct:capture_start(), - _ = application:set_env([{foo, []}, {foo, []}]), - timer:sleep(100), - ct:capture_stop(), - [_ | _] = string:find(ct:capture_get(), "duplicate application config: foo"), - - ct:capture_start(), - _ = application:set_env([{foo, [{bar, baz}, {bar, bat}]}]), - timer:sleep(100), - ct:capture_stop(), - [_ | _] = string:find(ct:capture_get(), "application: foo; duplicate parameter: bar"), + "duplicate application config: foo" = + badarg_msg(fun() -> application:set_env([{foo, []}, {foo, []}]) end), + + "application: foo; duplicate parameter: bar" = + badarg_msg(fun() -> application:set_env([{foo, [{bar, baz}, {bar, bat}]}]) end), ok. diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 1314316c13..4f0847084f 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -25,8 +25,8 @@ -export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2]). -export([set_path/1, get_path/1, add_path/1, add_paths/1, del_path/1, replace_path/1, load_file/1, load_abs/1, ensure_loaded/1, - delete/1, purge/1, purge_many_exits/1, soft_purge/1, is_loaded/1, - all_loaded/1, + delete/1, purge/1, purge_many_exits/0, purge_many_exits/1, + soft_purge/1, is_loaded/1, all_loaded/1, load_binary/1, dir_req/1, object_code/1, set_path_file/1, upgrade/1, sticky_dir/1, pa_pz_option/1, add_del_path/1, @@ -55,7 +55,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{minutes,5}}]. + {timetrap,{seconds,30}}]. all() -> [set_path, get_path, add_path, add_paths, del_path, @@ -140,6 +140,11 @@ end_per_testcase(on_load_embedded, Config) -> LinkName = proplists:get_value(link_name, Config), _ = del_link(LinkName), end_per_testcase(Config); +end_per_testcase(upgrade, Config) -> + %% Make sure tracing is turned off even if the test times out. + erlang:trace_pattern({error_handler,undefined_function,3}, false, [global]), + erlang:trace(self(), false, [call]), + end_per_testcase(Config); end_per_testcase(_Func, Config) -> end_per_testcase(Config). @@ -391,6 +396,9 @@ purge(Config) when is_list(Config) -> process_flag(trap_exit, OldFlag), ok. +purge_many_exits() -> + [{timetrap, {minutes, 2}}]. + purge_many_exits(Config) when is_list(Config) -> OldFlag = process_flag(trap_exit, true), @@ -525,7 +533,7 @@ upgrade(Config) -> T = [beam, hipe], [upgrade_do(DataDir, Client, T) || Client <- T], - case hipe:llvm_support_available() of + case hipe:erllvm_is_supported() of false -> ok; true -> T2 = [beam, hipe_llvm], @@ -1021,6 +1029,13 @@ mult_lib_remove_prefix([H|T1], [H|T2]) -> mult_lib_remove_prefix([$/|T], []) -> T. bad_erl_libs(Config) when is_list(Config) -> + %% Preserve ERL_LIBS if set. + BadLibs0 = "/no/such/dir", + BadLibs = + case os:getenv("ERL_LIBS") of + false -> BadLibs0; + Libs -> BadLibs0 ++ ":" ++ Libs + end, {ok,Node} = test_server:start_node(bad_erl_libs, slave, []), Code = rpc:call(Node,code,get_path,[]), @@ -1028,10 +1043,9 @@ bad_erl_libs(Config) when is_list(Config) -> {ok,Node2} = test_server:start_node(bad_erl_libs, slave, - [{args,"-env ERL_LIBS /no/such/dir"}]), + [{args,"-env ERL_LIBS " ++ BadLibs}]), Code2 = rpc:call(Node,code,get_path,[]), test_server:stop_node(Node2), - %% Test that code path is not affected by the faulty ERL_LIBS Code = Code2, @@ -1550,6 +1564,11 @@ on_load_update_code_1(3, Mod) -> %% Test -on_load while trace feature 'on_load' is enabled (OTP-14612) on_load_trace_on_load(Config) -> + %% 'on_load' enables tracing for all newly loaded modules, so we make a dry + %% run to ensure that ancillary modules like 'merl' won't be loaded during + %% the actual test. + on_load_update(Config), + Papa = self(), Tracer = spawn_link(fun F() -> receive M -> Papa ! M end, F() end), {tracer,[]} = erlang:trace_info(self(),tracer), diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index 5a8bbd56c4..c3a022df0a 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -40,7 +40,8 @@ monitor_nodes_errors/1, monitor_nodes_combinations/1, monitor_nodes_cleanup/1, - monitor_nodes_many/1]). + monitor_nodes_many/1, + dist_ctrl_proc_smoke/1]). %% Performs the test at another node. -export([get_socket_priorities/0, @@ -52,7 +53,7 @@ -export([init_per_testcase/2, end_per_testcase/2]). --export([start_node/2]). +-export([dist_cntrlr_output_test/2]). -export([pinger/1]). @@ -67,10 +68,11 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{minutes,4}}]. + {timetrap,{minutes,12}}]. all() -> - [tick, tick_change, nodenames, hostnames, illegal_nodenames, + [dist_ctrl_proc_smoke, + tick, tick_change, nodenames, hostnames, illegal_nodenames, connect_node, hidden_node, setopts, table_waste, net_setuptime, inet_dist_options_options, @@ -116,10 +118,12 @@ connect_node(Config) when is_list(Config) -> ok. tick(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(erl_distribution_SUITE)), + run_dist_configs(fun tick/2, Config). +tick(DCfg, _Config) -> %% First check that the normal case is OK! - {ok, Node} = start_node(dist_test, "-pa " ++ PaDir), + [Name1, Name2] = get_nodenames(2, dist_test), + {ok, Node} = start_node(DCfg, Name1), rpc:call(Node, erl_distribution_SUITE, tick_cli_test, [node()]), erlang:monitor_node(Node, true), @@ -143,14 +147,12 @@ tick(Config) when is_list(Config) -> %% Set the ticktime on the server node to 100 secs so the server %% node doesn't tick the client node within the interval ... - {ok, ServNode} = start_node(dist_test_server, - "-kernel net_ticktime 100 " - "-pa " ++ PaDir), + {ok, ServNode} = start_node(DCfg, Name2, + "-kernel net_ticktime 100"), rpc:call(ServNode, erl_distribution_SUITE, tick_serv_test, [Node, node()]), - {ok, _} = start_node(dist_test, - "-kernel net_ticktime 12 " - "-pa " ++ PaDir), + {ok, Node} = start_node(DCfg, Name1, + "-kernel net_ticktime 12"), rpc:call(Node, erl_distribution_SUITE, tick_cli_test, [ServNode]), spawn_link(erl_distribution_SUITE, keep_conn, [Node]), @@ -180,6 +182,9 @@ tick(Config) when is_list(Config) -> %% Checks that pinging nonexistyent nodes does not waste space in distribution table. table_waste(Config) when is_list(Config) -> + run_dist_configs(fun table_waste/2, Config). + +table_waste(DCfg, _Config) -> {ok, HName} = inet:gethostname(), F = fun(0,_F) -> []; (N,F) -> @@ -189,7 +194,7 @@ table_waste(Config) when is_list(Config) -> F(N-1,F) end, F(256,F), - {ok, N} = start_node(erl_distribution_300,""), + {ok, N} = start_node(DCfg, erl_distribution_300), stop_node(N), ok. @@ -200,6 +205,9 @@ nodenames(Config) when is_list(Config) -> legal("a-1@b"), legal("a_1@b"), + %% Test that giving two -sname works as it should + test_node("a_1@b", false, long_or_short() ++ "a_0@b"), + illegal("cdé@a"), illegal("te欢st@a"). @@ -253,8 +261,11 @@ illegal(Name) -> test_node(Name) -> test_node(Name, false). test_node(Name, Illigal) -> + test_node(Name, Illigal, ""). +test_node(Name, Illigal, ExtraArgs) -> ProgName = ct:get_progname(), - Command = ProgName ++ " -noinput " ++ long_or_short() ++ Name ++ + Command = ProgName ++ " -noinput " ++ ExtraArgs ++ + long_or_short() ++ Name ++ " -eval \"net_adm:ping('" ++ atom_to_list(node()) ++ "')\"" ++ case Illigal of true -> @@ -295,13 +306,16 @@ gethostname() -> %% Test that pinging an illegal nodename does not kill the node. illegal_nodenames(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(erl_distribution_SUITE)), - {ok, Node}=start_node(illegal_nodenames, "-pa " ++ PaDir), + run_dist_configs(fun illegal_nodenames/2, Config). + +illegal_nodenames(DCfg, _Config) -> + {ok, Node}=start_node(DCfg, illegal_nodenames), monitor_node(Node, true), RPid=rpc:call(Node, erlang, spawn, [?MODULE, pinger, [self()]]), receive {RPid, pinged} -> + monitor_node(Node, false), ok; {nodedown, Node} -> ct:fail("Remote node died.") @@ -318,22 +332,25 @@ pinger(Starter) -> %% Test that you can set the net_setuptime properly. net_setuptime(Config) when is_list(Config) -> + run_dist_configs(fun net_setuptime/2, Config). + +net_setuptime(DCfg, _Config) -> + %% In this test case, we reluctantly accept shorter times than the given %% setup time, because the connection attempt can end in a %% "Host unreachable" error before the timeout fires. - Res0 = do_test_setuptime("2"), + Res0 = do_test_setuptime(DCfg, "2"), io:format("Res0 = ~p", [Res0]), true = (Res0 =< 4000), - Res1 = do_test_setuptime("0.3"), + Res1 = do_test_setuptime(DCfg, "0.3"), io:format("Res1 = ~p", [Res1]), true = (Res1 =< 500), ok. -do_test_setuptime(Setuptime) when is_list(Setuptime) -> - PaDir = filename:dirname(code:which(?MODULE)), - {ok, Node} = start_node(dist_setuptime_test, "-pa " ++ PaDir ++ - " -kernel net_setuptime " ++ Setuptime), +do_test_setuptime(DCfg, Setuptime) when is_list(Setuptime) -> + {ok, Node} = start_node(DCfg, dist_setuptime_test, + "-kernel net_setuptime " ++ Setuptime), Res = rpc:call(Node,?MODULE,time_ping,[?DUMMY_NODE]), stop_node(Node), Res. @@ -399,32 +416,36 @@ tick_cli_test1(Node) -> end. setopts(Config) when is_list(Config) -> + run_dist_configs(fun setopts/2, Config). + +setopts(DCfg, _Config) -> register(setopts_regname, self()), [N1,N2,N3,N4] = get_nodenames(4, setopts), - {_N1F,Port1} = start_node_unconnected(N1, ?MODULE, run_remote_test, + {_N1F,Port1} = start_node_unconnected(DCfg, N1, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "1", "ping"]), 0 = wait_for_port_exit(Port1), - {_N2F,Port2} = start_node_unconnected(N2, ?MODULE, run_remote_test, + {_N2F,Port2} = start_node_unconnected(DCfg, N2, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "2", "ping"]), 0 = wait_for_port_exit(Port2), {ok, LSock} = gen_tcp:listen(0, [{packet,2}, {active,false}]), {ok, LTcpPort} = inet:port(LSock), - {N3F,Port3} = start_node_unconnected(N3, ?MODULE, run_remote_test, + {N3F,Port3} = start_node_unconnected(DCfg, N3, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "1", integer_to_list(LTcpPort)]), wait_and_connect(LSock, N3F, Port3), 0 = wait_for_port_exit(Port3), - {N4F,Port4} = start_node_unconnected(N4, ?MODULE, run_remote_test, + {N4F,Port4} = start_node_unconnected(DCfg, N4, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "2", integer_to_list(LTcpPort)]), wait_and_connect(LSock, N4F, Port4), 0 = wait_for_port_exit(Port4), + unregister(setopts_regname), ok. wait_and_connect(LSock, NodeName, NodePort) -> @@ -518,9 +539,9 @@ opt_from_nr("2") -> {nodelay, false}. change_val(true) -> false; change_val(false) -> true. -start_node_unconnected(Name, Mod, Func, Args) -> +start_node_unconnected(DCfg, Name, Mod, Func, Args) -> FullName = full_node_name(Name), - CmdLine = mk_node_cmdline(Name,Mod,Func,Args), + CmdLine = mk_node_cmdline(DCfg, Name,Mod,Func,Args), io:format("Starting node ~p: ~s~n", [FullName, CmdLine]), case open_port({spawn, CmdLine}, [exit_status]) of Port when is_port(Port) -> @@ -534,7 +555,7 @@ full_node_name(PreName) -> atom_to_list(node())), list_to_atom(atom_to_list(PreName) ++ HostSuffix). -mk_node_cmdline(Name,Mod,Func,Args) -> +mk_node_cmdline(DCfg, Name,Mod,Func,Args) -> Static = "-noinput", Pa = filename:dirname(code:which(?MODULE)), Prog = case catch init:get_argument(progname) of @@ -551,6 +572,7 @@ mk_node_cmdline(Name,Mod,Func,Args) -> Prog ++ " " ++ Static ++ " " ++ NameSw ++ " " ++ NameStr + ++ " " ++ DCfg ++ " -pa " ++ Pa ++ " -env ERL_CRASH_DUMP " ++ Pwd ++ "/erl_crash_dump." ++ NameStr ++ " -setcookie " ++ atom_to_list(erlang:get_cookie()) @@ -560,7 +582,9 @@ mk_node_cmdline(Name,Mod,Func,Args) -> %% OTP-4255. tick_change(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(?MODULE)), + run_dist_configs(fun tick_change/2, Config). + +tick_change(DCfg, _Config) -> [BN, CN] = get_nodenames(2, tick_change), DefaultTT = net_kernel:get_net_ticktime(), unchanged = net_kernel:set_net_ticktime(DefaultTT, 60), @@ -577,14 +601,13 @@ tick_change(Config) when is_list(Config) -> end, wait_until(fun () -> 10 == net_kernel:get_net_ticktime() end), - {ok, B} = start_node(BN, "-kernel net_ticktime 10 -pa " ++ PaDir), - {ok, C} = start_node(CN, "-kernel net_ticktime 10 -hidden -pa " - ++ PaDir), + {ok, B} = start_node(DCfg, BN, "-kernel net_ticktime 10"), + {ok, C} = start_node(DCfg, CN, "-kernel net_ticktime 10 -hidden"), OTE = process_flag(trap_exit, true), case catch begin - run_tick_change_test(B, C, 10, 1, PaDir), - run_tick_change_test(B, C, 1, 10, PaDir) + run_tick_change_test(DCfg, B, C, 10, 1), + run_tick_change_test(DCfg, B, C, 1, 10) end of {'EXIT', Reason} -> stop_node(B), @@ -626,7 +649,7 @@ wait_for_nodedowns(Tester, Ref) -> end, wait_for_nodedowns(Tester, Ref). -run_tick_change_test(B, C, PrevTT, TT, PaDir) -> +run_tick_change_test(DCfg, B, C, PrevTT, TT) -> [DN, EN] = get_nodenames(2, tick_change), Tester = self(), @@ -640,8 +663,8 @@ run_tick_change_test(B, C, PrevTT, TT, PaDir) -> wait_for_nodedowns(Tester, Ref) end, - {ok, D} = start_node(DN, "-kernel net_ticktime " - ++ integer_to_list(PrevTT) ++ " -pa " ++ PaDir), + {ok, D} = start_node(DCfg, DN, "-kernel net_ticktime " + ++ integer_to_list(PrevTT)), NMA = spawn_link(fun () -> MonitorNodes([B, C, D]) end), NMB = spawn_link(B, fun () -> MonitorNodes([node(), C, D]) end), @@ -674,8 +697,8 @@ run_tick_change_test(B, C, PrevTT, TT, PaDir) -> sleep(7), change_initiated = rpc:call(C,net_kernel,set_net_ticktime,[TT,10]), - {ok, E} = start_node(EN, "-kernel net_ticktime " - ++ integer_to_list(TT) ++ " -pa " ++ PaDir), + {ok, E} = start_node(DCfg, EN, "-kernel net_ticktime " + ++ integer_to_list(TT)), NME = spawn_link(E, fun () -> MonitorNodes([node(), B, C, D]) end), NMA2 = spawn_link(fun () -> MonitorNodes([E]) end), NMB2 = spawn_link(B, fun () -> MonitorNodes([E]) end), @@ -735,12 +758,13 @@ run_tick_change_test(B, C, PrevTT, TT, PaDir) -> %% %% Basic test of hidden node. hidden_node(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(?MODULE)), - VArgs = "-pa " ++ PaDir, - HArgs = "-hidden -pa " ++ PaDir, - {ok, V} = start_node(visible_node, VArgs), + run_dist_configs(fun hidden_node/2, Config). + +hidden_node(DCfg, _Config) -> + HArgs = "-hidden", + {ok, V} = start_node(DCfg, visible_node), VMN = start_monitor_nodes_proc(V), - {ok, H} = start_node(hidden_node, HArgs), + {ok, H} = start_node(DCfg, hidden_node, HArgs), %% Connect visible_node -> hidden_node connect_nodes(V, H), test_nodes(V, H), @@ -748,9 +772,9 @@ hidden_node(Config) when is_list(Config) -> sleep(5), check_monitor_nodes_res(VMN, H), stop_node(V), - {ok, H} = start_node(hidden_node, HArgs), + {ok, H} = start_node(DCfg, hidden_node, HArgs), HMN = start_monitor_nodes_proc(H), - {ok, V} = start_node(visible_node, VArgs), + {ok, V} = start_node(DCfg, visible_node), %% Connect hidden_node -> visible_node connect_nodes(H, V), test_nodes(V, H), @@ -850,9 +874,9 @@ do_inet_dist_options_options(Prio) -> "-kernel inet_dist_connect_options "++PriorityString++" " "-kernel inet_dist_listen_options "++PriorityString, {ok,Node1} = - start_node(inet_dist_options_1, InetDistOptions), + start_node("", inet_dist_options_1, InetDistOptions), {ok,Node2} = - start_node(inet_dist_options_2, InetDistOptions), + start_node("", inet_dist_options_2, InetDistOptions), %% pong = rpc:call(Node1, net_adm, ping, [Node2]), @@ -885,6 +909,9 @@ get_socket_priorities() -> %% monitor_nodes_nodedown_reason(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_nodedown_reason/2, Config). + +monitor_nodes_nodedown_reason(DCfg, _Config) -> MonNodeState = monitor_node_state(), ok = net_kernel:monitor_nodes(true), ok = net_kernel:monitor_nodes(true, [nodedown_reason]), @@ -892,10 +919,10 @@ monitor_nodes_nodedown_reason(Config) when is_list(Config) -> Names = get_numbered_nodenames(5, node), [NN1, NN2, NN3, NN4, NN5] = Names, - {ok, N1} = start_node(NN1), - {ok, N2} = start_node(NN2), - {ok, N3} = start_node(NN3), - {ok, N4} = start_node(NN4, "-hidden"), + {ok, N1} = start_node(DCfg, NN1), + {ok, N2} = start_node(DCfg, NN2), + {ok, N3} = start_node(DCfg, NN3), + {ok, N4} = start_node(DCfg, NN4, "-hidden"), receive {nodeup, N1} -> ok end, receive {nodeup, N2} -> ok end, @@ -925,7 +952,7 @@ monitor_nodes_nodedown_reason(Config) when is_list(Config) -> ok = net_kernel:monitor_nodes(false, [nodedown_reason]), - {ok, N5} = start_node(NN5), + {ok, N5} = start_node(DCfg, NN5), stop_node(N5), receive {nodeup, N5} -> ok end, @@ -938,11 +965,14 @@ monitor_nodes_nodedown_reason(Config) when is_list(Config) -> monitor_nodes_complex_nodedown_reason(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_complex_nodedown_reason/2, Config). + +monitor_nodes_complex_nodedown_reason(DCfg, _Config) -> MonNodeState = monitor_node_state(), Me = self(), ok = net_kernel:monitor_nodes(true, [nodedown_reason]), [Name] = get_nodenames(1, monitor_nodes_complex_nodedown_reason), - {ok, Node} = start_node(Name, ""), + {ok, Node} = start_node(DCfg, Name, ""), Pid = spawn(Node, fun() -> Me ! {stuff, @@ -981,16 +1011,19 @@ monitor_nodes_complex_nodedown_reason(Config) when is_list(Config) -> %% monitor_nodes_node_type(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_node_type/2, Config). + +monitor_nodes_node_type(DCfg, _Config) -> MonNodeState = monitor_node_state(), ok = net_kernel:monitor_nodes(true), ok = net_kernel:monitor_nodes(true, [{node_type, all}]), Names = get_numbered_nodenames(9, node), [NN1, NN2, NN3, NN4, NN5, NN6, NN7, NN8, NN9] = Names, - {ok, N1} = start_node(NN1), - {ok, N2} = start_node(NN2), - {ok, N3} = start_node(NN3, "-hidden"), - {ok, N4} = start_node(NN4, "-hidden"), + {ok, N1} = start_node(DCfg, NN1), + {ok, N2} = start_node(DCfg, NN2), + {ok, N3} = start_node(DCfg, NN3, "-hidden"), + {ok, N4} = start_node(DCfg, NN4, "-hidden"), receive {nodeup, N1} -> ok end, receive {nodeup, N2} -> ok end, @@ -1014,15 +1047,15 @@ monitor_nodes_node_type(Config) when is_list(Config) -> receive {nodedown, N4, [{node_type, hidden}]} -> ok end, ok = net_kernel:monitor_nodes(false, [{node_type, all}]), - {ok, N5} = start_node(NN5), + {ok, N5} = start_node(DCfg, NN5), receive {nodeup, N5} -> ok end, stop_node(N5), receive {nodedown, N5} -> ok end, ok = net_kernel:monitor_nodes(true, [{node_type, hidden}]), - {ok, N6} = start_node(NN6), - {ok, N7} = start_node(NN7, "-hidden"), + {ok, N6} = start_node(DCfg, NN6), + {ok, N7} = start_node(DCfg, NN7, "-hidden"), receive {nodeup, N6} -> ok end, @@ -1037,8 +1070,8 @@ monitor_nodes_node_type(Config) when is_list(Config) -> ok = net_kernel:monitor_nodes(false, [{node_type, hidden}]), ok = net_kernel:monitor_nodes(false), - {ok, N8} = start_node(NN8), - {ok, N9} = start_node(NN9, "-hidden"), + {ok, N8} = start_node(DCfg, NN8), + {ok, N9} = start_node(DCfg, NN9, "-hidden"), receive {nodeup, N8, [{node_type, visible}]} -> ok end, stop_node(N8), @@ -1058,6 +1091,9 @@ monitor_nodes_node_type(Config) when is_list(Config) -> %% monitor_nodes_misc(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_misc/2, Config). + +monitor_nodes_misc(DCfg, _Config) -> MonNodeState = monitor_node_state(), ok = net_kernel:monitor_nodes(true), ok = net_kernel:monitor_nodes(true, [{node_type, all}, nodedown_reason]), @@ -1065,8 +1101,8 @@ monitor_nodes_misc(Config) when is_list(Config) -> Names = get_numbered_nodenames(3, node), [NN1, NN2, NN3] = Names, - {ok, N1} = start_node(NN1), - {ok, N2} = start_node(NN2, "-hidden"), + {ok, N1} = start_node(DCfg, NN1), + {ok, N2} = start_node(DCfg, NN2, "-hidden"), receive {nodeup, N1} -> ok end, @@ -1092,7 +1128,7 @@ monitor_nodes_misc(Config) when is_list(Config) -> ok = net_kernel:monitor_nodes(false, [{node_type, all}, nodedown_reason]), - {ok, N3} = start_node(NN3), + {ok, N3} = start_node(DCfg, NN3), receive {nodeup, N3} -> ok end, stop_node(N3), receive {nodedown, N3} -> ok end, @@ -1107,15 +1143,18 @@ monitor_nodes_misc(Config) when is_list(Config) -> %% messages from Node and that {nodedown, Node} messages are %% received after messages from Node. monitor_nodes_otp_6481(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_otp_6481/2, Config). + +monitor_nodes_otp_6481(DCfg, Config) -> io:format("Testing nodedown...~n"), - monitor_nodes_otp_6481_test(Config, nodedown), + monitor_nodes_otp_6481_test(DCfg, Config, nodedown), io:format("ok~n"), io:format("Testing nodeup...~n"), - monitor_nodes_otp_6481_test(Config, nodeup), + monitor_nodes_otp_6481_test(DCfg, Config, nodeup), io:format("ok~n"), ok. -monitor_nodes_otp_6481_test(Config, TestType) when is_list(Config) -> +monitor_nodes_otp_6481_test(DCfg, Config, TestType) when is_list(Config) -> MonNodeState = monitor_node_state(), NodeMsg = make_ref(), Me = self(), @@ -1164,7 +1203,7 @@ monitor_nodes_otp_6481_test(Config, TestType) when is_list(Config) -> end ++ MonNodeState, - {ok, Node} = start_node(Name, "", this), + {ok, Node} = start_node(DCfg, Name, "", this), receive {nodeup, Node} -> ok end, RemotePid = spawn(Node, @@ -1249,17 +1288,20 @@ monitor_nodes_errors(Config) when is_list(Config) -> ok. monitor_nodes_combinations(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_combinations/2, Config). + +monitor_nodes_combinations(DCfg, _Config) -> MonNodeState = monitor_node_state(), monitor_nodes_all_comb(true), [VisibleName, HiddenName] = get_nodenames(2, monitor_nodes_combinations), - {ok, Visible} = start_node(VisibleName, ""), + {ok, Visible} = start_node(DCfg, VisibleName, ""), receive_all_comb_nodeup_msgs(visible, Visible), no_msgs(), stop_node(Visible), receive_all_comb_nodedown_msgs(visible, Visible, connection_closed), no_msgs(), - {ok, Hidden} = start_node(HiddenName, "-hidden"), + {ok, Hidden} = start_node(DCfg, HiddenName, "-hidden"), receive_all_comb_nodeup_msgs(hidden, Hidden), no_msgs(), stop_node(Hidden), @@ -1395,6 +1437,9 @@ monitor_nodes_cleanup(Config) when is_list(Config) -> ok. monitor_nodes_many(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_many/2, Config). + +monitor_nodes_many(DCfg, _Config) -> MonNodeState = monitor_node_state(), [Name] = get_nodenames(1, monitor_nodes_many), %% We want to perform more than 2^16 net_kernel:monitor_nodes @@ -1402,7 +1447,7 @@ monitor_nodes_many(Config) when is_list(Config) -> No = (1 bsl 16) + 17, repeat(fun () -> ok = net_kernel:monitor_nodes(true) end, No), No = length(monitor_node_state()) - length(MonNodeState), - {ok, Node} = start_node(Name), + {ok, Node} = start_node(DCfg, Name), repeat(fun () -> receive {nodeup, Node} -> ok end end, No), stop_node(Node), repeat(fun () -> receive {nodedown, Node} -> ok end end, No), @@ -1411,8 +1456,118 @@ monitor_nodes_many(Config) when is_list(Config) -> MonNodeState = monitor_node_state(), ok. +dist_ctrl_proc_smoke(Config) when is_list(Config) -> + ThisNode = node(), + [Name1, Name2] = get_nodenames(2, dist_ctrl_proc_example_smoke), + GetSizeArg = " -gen_tcp_dist_output_loop " + ++ atom_to_list(?MODULE) ++ " " + ++ "dist_cntrlr_output_test", + {ok, Node1} = start_node("", Name1, "-proto_dist gen_tcp"), + {ok, Node2} = start_node("", Name2, "-proto_dist gen_tcp" ++ GetSizeArg), + pong = rpc:call(Node1, net_adm, ping, [Node2]), + NL1 = lists:sort([ThisNode, Node2]), + NL2 = lists:sort([ThisNode, Node1]), + NL1 = lists:sort(rpc:call(Node1, erlang, nodes, [])), + NL2 = lists:sort(rpc:call(Node2, erlang, nodes, [])), + + %% Verify that we actually are executing the distribution + %% module we expect and also massage message passing over + %% it a bit... + Ps1 = rpc:call(Node1, erlang, processes, []), + try + lists:foreach( + fun (P) -> + case rpc:call(Node1, erlang, process_info, [P, current_stacktrace]) of + undefined -> + ok; + {current_stacktrace, StkTrace} -> + lists:foreach(fun ({gen_tcp_dist, + dist_cntrlr_output_loop, + 2, _}) -> + io:format("~p ~p~n", [P, StkTrace]), + throw(found_it); + (_) -> + ok + end, StkTrace) + end + end, Ps1), + exit({missing, dist_cntrlr_output_loop}) + catch + throw:found_it -> ok + end, + + Ps2 = rpc:call(Node2, erlang, processes, []), + try + lists:foreach( + fun (P) -> + case rpc:call(Node2, erlang, process_info, [P, current_stacktrace]) of + undefined -> + ok; + {current_stacktrace, StkTrace} -> + lists:foreach(fun ({erl_distribution_SUITE, + dist_cntrlr_output_loop, + 2, _}) -> + io:format("~p ~p~n", [P, StkTrace]), + throw(found_it); + (_) -> + ok + end, StkTrace) + end + end, Ps2), + exit({missing, dist_cntrlr_output_loop}) + catch + throw:found_it -> ok + end, + + stop_node(Node1), + stop_node(Node2), + ok. + %% Misc. functions +run_dist_configs(Func, Config) -> + GetSizeArg = " -gen_tcp_dist_output_loop " + ++ atom_to_list(?MODULE) ++ " " + ++ "dist_cntrlr_output_test", + lists:map(fun ({DCfgName, DCfg}) -> + io:format("~n~n=== Running ~s configuration ===~n~n", + [DCfgName]), + Func(DCfg, Config) + end, + [{"default", ""}, + {"gen_tcp_dist", "-proto_dist gen_tcp"}, + {"gen_tcp_dist (get_size)", "-proto_dist gen_tcp" ++ GetSizeArg}]). + +dist_cntrlr_output_test(DHandle, Socket) -> + false = erlang:dist_ctrl_get_opt(DHandle, get_size), + false = erlang:dist_ctrl_set_opt(DHandle, get_size, true), + true = erlang:dist_ctrl_get_opt(DHandle, get_size), + true = erlang:dist_ctrl_set_opt(DHandle, get_size, false), + false = erlang:dist_ctrl_get_opt(DHandle, get_size), + false = erlang:dist_ctrl_set_opt(DHandle, get_size, true), + true = erlang:dist_ctrl_get_opt(DHandle, get_size), + dist_cntrlr_output_loop(DHandle, Socket). + +dist_cntrlr_send_data(DHandle, Socket) -> + case erlang:dist_ctrl_get_data(DHandle) of + none -> + erlang:dist_ctrl_get_data_notification(DHandle); + {Size, Data} -> + Size = erlang:iolist_size(Data), + ok = gen_tcp:send(Socket, Data), + dist_cntrlr_send_data(DHandle, Socket) + end. + +dist_cntrlr_output_loop(DHandle, Socket) -> + receive + dist_data -> + %% Outgoing data from this node... + dist_cntrlr_send_data(DHandle, Socket); + _ -> + ok %% Drop garbage message... + end, + dist_cntrlr_output_loop(DHandle, Socket). + monitor_node_state() -> erts_debug:set_internal_state(available_internal_state, true), MonitoringNodes = erts_debug:get_internal_state(monitoring_nodes), @@ -1438,25 +1593,25 @@ print_my_messages() -> sleep(T) -> receive after T * 1000 -> ok end. -start_node(Name, Param, this) -> +start_node(DCfg, Name, Param, this) -> NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), test_server:start_node(Name, peer, [{args, NewParam}, {erl, [this]}]); -start_node(Name, Param, "this") -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param, "this") -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, peer, [{args, NewParam}, {erl, [this]}]); -start_node(Name, Param, Rel) when is_atom(Rel) -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param, Rel) when is_atom(Rel) -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, peer, [{args, NewParam}, {erl, [{release, atom_to_list(Rel)}]}]); -start_node(Name, Param, Rel) when is_list(Rel) -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param, Rel) when is_list(Rel) -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, peer, [{args, NewParam}, {erl, [{release, Rel}]}]). -start_node(Name, Param) -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param) -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, slave, [{args, NewParam}]). -start_node(Name) -> - start_node(Name, ""). +start_node(DCfg, Name) -> + start_node(DCfg, Name, ""). stop_node(Node) -> test_server:stop_node(Node). diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index a51025cba6..3bc8e6e828 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -2191,6 +2191,9 @@ unc_paths(Config) when is_list(Config) -> {ok, _} = file:read_file_info("C:\\Windows\\explorer.exe"), {ok, _} = file:read_file_info("\\\\localhost\\c$\\Windows\\explorer.exe"), + {ok, Files} = file:list_dir("C:\\Windows\\"), + {ok, Files} = file:list_dir("\\\\localhost\\c$\\Windows\\"), + {ok, Cwd} = file:get_cwd(), try @@ -3741,19 +3744,33 @@ otp_10852(Config) when is_list(Config) -> ok = rpc_call(Node, read_file, [B]), ok = rpc_call(Node, make_link, [B,B]), case rpc_call(Node, make_symlink, [B,B]) of - ok -> ok; - {error, E} when (E =:= enotsup) or (E =:= eperm) -> - {win32,_} = os:type() + {error, eilseq} -> + %% Some versions of OS X refuse to create files with illegal names. + {unix,darwin} = os:type(); + {error, eperm} -> + %% The test user might not have permission to create symlinks. + {win32,_} = os:type(); + ok -> + ok end, ok = rpc_call(Node, delete, [B]), - ok = rpc_call(Node, make_dir, [B]), + case rpc_call(Node, make_dir, [B]) of + {error, eilseq} -> + {unix,darwin} = os:type(); + ok -> + ok + end, ok = rpc_call(Node, del_dir, [B]), - ok = rpc_call(Node, write_file, [B,B]), - {ok, Fd} = rpc_call(Node, open, [B,[read]]), - ok = rpc_call(Node, close, [Fd]), - {ok,0} = rpc_call(Node, copy, [B,B]), - {ok, Fd2, B} = rpc_call(Node, path_open, [["."], B, [read]]), - ok = rpc_call(Node, close, [Fd2]), + case rpc_call(Node, write_file, [B,B]) of + {error, eilseq} -> + {unix,darwin} = os:type(); + ok -> + {ok, Fd} = rpc_call(Node, open, [B,[read]]), + ok = rpc_call(Node, close, [Fd]), + {ok,0} = rpc_call(Node, copy, [B,B]), + {ok, Fd2, B} = rpc_call(Node, path_open, [["."], B, [read]]), + ok = rpc_call(Node, close, [Fd2]) + end, true = test_server:stop_node(Node), ok. @@ -4497,15 +4514,18 @@ run_large_file_test(Config, Run, Name) -> {{unix,sunos},OsVersion} when OsVersion < {5,5,1} -> {skip,"Only supported on Win32, Unix or SunOS >= 5.5.1"}; {{unix,_},_} -> - N = disc_free(proplists:get_value(priv_dir, Config)), - io:format("Free disk: ~w KByte~n", [N]), - if N < 5 * (1 bsl 20) -> - %% Less than 5 GByte free - {skip,"Less than 5 GByte free"}; - true -> - do_run_large_file_test(Config, Run, Name) - end; - _ -> + case disc_free(proplists:get_value(priv_dir, Config)) of + error -> + {skip, "Failed to query disk space for priv_dir. " + "Is it on a remote file system?~n"}; + N when N >= 5 * (1 bsl 20) -> + ct:pal("Free disk: ~w KByte~n", [N]), + do_run_large_file_test(Config, Run, Name); + N when N < 5 * (1 bsl 20) -> + ct:pal("Free disk: ~w KByte~n", [N]), + {skip,"Less than 5 GByte free"} + end; + _ -> {skip,"Only supported on Win32, Unix or SunOS >= 5.5.1"} end. @@ -4539,12 +4559,18 @@ do_run_large_file_test(Config, Run, Name0) -> disc_free(Path) -> Data = disksup:get_disk_data(), - {_,Tot,Perc} = hd(lists:filter( - fun({P,_Size,_Full}) -> - lists:prefix(filename:nativename(P), - filename:nativename(Path)) - end, lists:reverse(lists:sort(Data)))), - round(Tot * (1-(Perc/100))). + + %% What partitions could Data be mounted on? + Partitions = + [D || {P, _Tot, _Perc}=D <- Data, + lists:prefix(filename:nativename(P), filename:nativename(Path))], + + %% Sorting in descending order places the partition with the most specific + %% path first. + case lists:sort(fun erlang:'>='/2, Partitions) of + [{_,Tot, Perc} | _] -> round(Tot * (1-(Perc/100))); + [] -> error + end. memsize() -> {Tot,_Used,_} = memsup:get_memory_data(), diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index 3afc647081..26cfd187c7 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -632,10 +632,13 @@ make_icky_dir(Mod, IckyDirName) -> hopeless_darwin() -> case {os:type(),os:version()} of - {{unix,darwin},{Major,_,_}} when Major < 9 -> - true; - _ -> - false + {{unix,darwin},{Major,_,_}} -> + %% icky file names worked between 10 and 17, but started returning + %% EILSEQ in 18. The check against 18 is exact in case newer + %% versions of Darwin support them again. + Major < 9 orelse Major =:= 18; + _ -> + false end. make_very_icky_dir(Mod, DirName) -> diff --git a/lib/kernel/test/gen_sctp_SUITE.erl b/lib/kernel/test/gen_sctp_SUITE.erl index a0ae792ba9..e4c489bd10 100644 --- a/lib/kernel/test/gen_sctp_SUITE.erl +++ b/lib/kernel/test/gen_sctp_SUITE.erl @@ -1459,11 +1459,11 @@ do_open_and_connect(ServerAddresses, AddressToConnectTo) -> do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun). %% do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun) -> - ServerFamily = get_family_by_addrs(ServerAddresses), + {ServerFamily, ServerOpts} = get_family_by_addrs(ServerAddresses), io:format("Serving ~p addresses: ~p~n", [ServerFamily, ServerAddresses]), S1 = ok(gen_sctp:open(0, [{ip,Addr} || Addr <- ServerAddresses] ++ - [ServerFamily])), + [ServerFamily|ServerOpts])), ok = gen_sctp:listen(S1, true), P1 = ok(inet:port(S1)), ClientFamily = get_family_by_addr(AddressToConnectTo), @@ -1493,9 +1493,9 @@ do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun) -> %% If at least one of the addresses is an ipv6 address, return inet6, else inet. get_family_by_addrs(Addresses) -> case lists:usort([get_family_by_addr(Addr) || Addr <- Addresses]) of - [inet, inet6] -> inet6; - [inet] -> inet; - [inet6] -> inet6 + [inet, inet6] -> {inet6, [{ipv6_v6only, false}]}; + [inet] -> {inet, []}; + [inet6] -> {inet6, []} end. get_family_by_addr(Addr) when tuple_size(Addr) =:= 4 -> inet; diff --git a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c index b91dca61d4..96938f9071 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c +++ b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c @@ -30,6 +30,7 @@ #define sock_close(s) closesocket(s) #else #include <sys/socket.h> +#include <unistd.h> #define sock_close(s) close(s) #endif diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index 52edfaee29..edf30448c4 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -2086,8 +2086,39 @@ test_pktoptions(Family, Spec, CheckConnect, OSType, OSVer) -> %%% {ok,<<"hi">>} = gen_tcp:recv(S1, 2, Timeout), %% %% Verify returned remote options - {ok,[{pktoptions,OptsVals1}]} = inet:getopts(S1, [pktoptions]), - {ok,[{pktoptions,OptsVals2}]} = inet:getopts(S2, [pktoptions]), + VerifyRemOpts = + fun(S, Role) -> + case inet:getopts(S, [pktoptions]) of + {ok, [{pktoptions, PktOpts1}]} -> + PktOpts1; + {ok, UnexpOK1} -> + io:format("Unexpected OK (~w): " + "~n ~p" + "~n", [Role, UnexpOK1]), + exit({unexpected_getopts_ok, + Role, + Spec, + TrueRecvOpts, + OptsVals, + OptsValsDefault, + UnexpOK1}); + {error, UnexpERR1} -> + io:format("Unexpected ERROR (~w): " + "~n ~p" + "~n", [Role, UnexpERR1]), + exit({unexpected_getopts_failure, + Role, + Spec, + TrueRecvOpts, + OptsVals, + OptsValsDefault, + UnexpERR1}) + end + end, + OptsVals1 = VerifyRemOpts(S1, dest), + OptsVals2 = VerifyRemOpts(S2, orig), + %% {ok,[{pktoptions,OptsVals1}]} = inet:getopts(S1, [pktoptions]), + %% {ok,[{pktoptions,OptsVals2}]} = inet:getopts(S2, [pktoptions]), (Result1 = sets_eq(OptsVals1, OptsVals)) orelse io:format( "Accept differs: ~p neq ~p~n", [OptsVals1,OptsVals]), @@ -3430,7 +3461,7 @@ wait(Mref) -> %% OTP-15536 %% Test that send error works correctly for delay_send -delay_send_error(Config) -> +delay_send_error(_Config) -> {ok, LS} = gen_tcp:listen(0, [{reuseaddr, true}, {packet, 1}, {active, false}]), {ok,{{0,0,0,0},PortNum}}=inet:sockname(LS), P = spawn_link( diff --git a/lib/kernel/test/inet_SUITE.erl b/lib/kernel/test/inet_SUITE.erl index 5a2d809aa4..44ec7e7076 100644 --- a/lib/kernel/test/inet_SUITE.erl +++ b/lib/kernel/test/inet_SUITE.erl @@ -98,7 +98,7 @@ end_per_group(_GroupName, Config) -> init_per_testcase(lookup_bad_search_option, Config) -> Db = inet_db, Key = res_lookup, - %% The bad option can not enter through inet_db:set_lookup/1, + %% The bad option cannot enter through inet_db:set_lookup/1, %% but through e.g .inetrc. Prev = ets:lookup(Db, Key), ets:delete(Db, Key), diff --git a/lib/kernel/test/inet_res_SUITE.erl b/lib/kernel/test/inet_res_SUITE.erl index df6e48abae..cbec8d430c 100644 --- a/lib/kernel/test/inet_res_SUITE.erl +++ b/lib/kernel/test/inet_res_SUITE.erl @@ -531,7 +531,7 @@ edns0(Config) when is_list(Config) -> case os:version() of {M,V,_} when M < 5; M == 5, V =< 8 -> %% In our test park only known platform - %% with an DNS resolver that can not do + %% with an DNS resolver that cannot do %% EDNS0. {comment,"No EDNS0"} end diff --git a/lib/kernel/test/init_SUITE.erl b/lib/kernel/test/init_SUITE.erl index 4f90260f98..a0154b2694 100644 --- a/lib/kernel/test/init_SUITE.erl +++ b/lib/kernel/test/init_SUITE.erl @@ -459,7 +459,7 @@ find_system_procs([], SysProcs) -> SysProcs#sys_procs.prim_file]; find_system_procs([P|Ps], SysProcs) -> case process_info(P, [initial_call, priority]) of - [{initial_call,{otp_ring0,start,2}},_] -> + [{initial_call,{erl_init,start,2}},_] -> undefined = SysProcs#sys_procs.init, find_system_procs(Ps, SysProcs#sys_procs{init = P}); [{initial_call,{erts_code_purger,start,0}},_] -> diff --git a/lib/kernel/test/kernel_config_SUITE.erl b/lib/kernel/test/kernel_config_SUITE.erl index 9207025a2c..57c44c2498 100644 --- a/lib/kernel/test/kernel_config_SUITE.erl +++ b/lib/kernel/test/kernel_config_SUITE.erl @@ -21,7 +21,8 @@ -include_lib("common_test/include/ct.hrl"). --export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, sync/1]). +-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, + start_distribution_false/1, sync/1]). -export([init_per_suite/1, end_per_suite/1]). @@ -30,7 +31,7 @@ suite() -> {timetrap,{minutes,2}}]. all() -> - [sync]. + [sync, start_distribution_false]. groups() -> []. @@ -59,12 +60,9 @@ from(H, [H | T]) -> T; from(H, [_ | T]) -> from(H, T); from(_, []) -> []. -%%----------------------------------------------------------------- -%% Test suite for sync_nodes. This is quite tricky. -%% +%% Test sync_nodes. This is quite tricky. %% Should be started in a CC view with: %% erl -sname XXX where XX not in [cp1, cp2] -%%----------------------------------------------------------------- sync(Conf) when is_list(Conf) -> %% Write a config file Dir = proplists:get_value(priv_dir,Conf), @@ -106,9 +104,18 @@ wait_for_node(Node) -> _Other -> wait_for_node(Node) end. - stop_node(Node) -> M = list_to_atom(lists:concat([Node, [$@], from($@,atom_to_list(node()))])), rpc:cast(M, erlang, halt, []). + +start_distribution_false(Config) when is_list(Config) -> + %% When distribution is disabled, -sname/-name has no effect + Str = os:cmd(ct:get_progname() + ++ " -kernel start_distribution false" + ++ " -sname no_distribution" + ++ " -eval \"erlang:display(node())\"" + ++ " -noshell -s erlang halt"), + "'nonode@nohost'" ++ _ = Str, + ok. diff --git a/lib/kernel/test/logger_disk_log_h_SUITE.erl b/lib/kernel/test/logger_disk_log_h_SUITE.erl index 9bbec42de8..36d98eaa25 100644 --- a/lib/kernel/test/logger_disk_log_h_SUITE.erl +++ b/lib/kernel/test/logger_disk_log_h_SUITE.erl @@ -293,7 +293,7 @@ logging(Config) -> ok = start_and_add(Name, #{filter_default=>log, formatter=>{?MODULE,self()}}, #{file => LogFile}), - MsgFormatter = fun(Term) -> {io_lib:format("Term:~p",[Term]),[]} end, + MsgFormatter = fun(Term) -> {"Term:~p",[Term]} end, logger:notice([{x,y}], #{report_cb => MsgFormatter}), logger:notice([{x,y}], #{}), ct:pal("Checking contents of ~p", [?log_no(LogFile,1)]), @@ -1243,12 +1243,12 @@ restart_after(Config) -> {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), NewHConfig1 = HConfig#{config=>DLHConfig#{overload_kill_enable=>true, - overload_kill_qlen=>10, + overload_kill_qlen=>4, overload_kill_restart_after=>infinity}}, ok = logger:update_handler_config(?MODULE, NewHConfig1), MRef1 = erlang:monitor(process, whereis(h_proc_name())), %% kill handler - send_burst({n,100}, {spawn,4,0}, {chars,79}, notice), + send_burst({n,100}, {spawn,5,0}, {chars,79}, notice), receive {'DOWN', MRef1, _, _, _Reason1} -> file_delete(Log), @@ -1265,13 +1265,13 @@ restart_after(Config) -> RestartAfter = ?OVERLOAD_KILL_RESTART_AFTER, NewHConfig2 = HConfig#{config=>DLHConfig#{overload_kill_enable=>true, - overload_kill_qlen=>10, + overload_kill_qlen=>4, overload_kill_restart_after=>RestartAfter}}, ok = logger:update_handler_config(?MODULE, NewHConfig2), Pid0 = whereis(h_proc_name()), MRef2 = erlang:monitor(process, Pid0), %% kill handler - send_burst({n,100}, {spawn,4,0}, {chars,79}, notice), + send_burst({n,100}, {spawn,5,0}, {chars,79}, notice), receive {'DOWN', MRef2, _, _, _Reason2} -> file_delete(Log), diff --git a/lib/kernel/test/logger_std_h_SUITE.erl b/lib/kernel/test/logger_std_h_SUITE.erl index 0c5516f82b..16ab0e97fc 100644 --- a/lib/kernel/test/logger_std_h_SUITE.erl +++ b/lib/kernel/test/logger_std_h_SUITE.erl @@ -71,6 +71,14 @@ init_per_group(_Group, Config) -> end_per_group(_Group, _Config) -> ok. +init_per_testcase(reopen_changed_log=TC, Config) -> + case os:type() of + {win32,_} -> + {skip,"This test can only work with inodes, i.e. not on Windows"}; + _ -> + ct:print("********** ~w **********", [TC]), + Config + end; init_per_testcase(TestHooksCase, Config) when TestHooksCase == write_failure; TestHooksCase == sync_failure -> diff --git a/lib/kernel/test/seq_trace_SUITE.erl b/lib/kernel/test/seq_trace_SUITE.erl index ee8f4e94f8..663f910751 100644 --- a/lib/kernel/test/seq_trace_SUITE.erl +++ b/lib/kernel/test/seq_trace_SUITE.erl @@ -19,6 +19,9 @@ %% -module(seq_trace_SUITE). +%% label_capability_mismatch needs to run a part of the test on an OTP 20 node. +-compile(r20). + -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2]). @@ -358,7 +361,7 @@ do_incompatible_labels(Rel) -> Mdir = filename:dirname(Dir), true = rpc:call(Node,code,add_patha,[Mdir]), seq_trace:reset_trace(), - rpc:call(Node,?MODULE,start_tracer,[]), + true = is_pid(rpc:call(Node,?MODULE,start_tracer,[])), Receiver = spawn(Node,?MODULE,one_time_receiver,[]), %% This node does not support arbitrary labels, so it must fail with a @@ -385,7 +388,7 @@ do_compatible_labels(Rel) -> Mdir = filename:dirname(Dir), true = rpc:call(Node,code,add_patha,[Mdir]), seq_trace:reset_trace(), - rpc:call(Node,?MODULE,start_tracer,[]), + true = is_pid(rpc:call(Node,?MODULE,start_tracer,[])), Receiver = spawn(Node,?MODULE,one_time_receiver,[]), %% This node does not support arbitrary labels, but small integers should diff --git a/lib/megaco/configure.in b/lib/megaco/configure.in index eaa875d0a3..bae6144abe 100644 --- a/lib/megaco/configure.in +++ b/lib/megaco/configure.in @@ -22,10 +22,6 @@ dnl dnl define([AC_CACHE_LOAD], )dnl dnl define([AC_CACHE_SAVE], )dnl -if test "x$no_recursion" != "xyes" -a "x$OVERRIDE_CONFIG_CACHE" = "x"; then - # We do not want to use a common cache! - cache_file=/dev/null -fi AC_INIT(vsn.mk) @@ -42,11 +38,14 @@ else host_os=win32 fi - dnl ---------------------------------------------------------------------- dnl Checks for programs. dnl ---------------------------------------------------------------------- +AC_PROG_CC + +LM_WINDOWS_ENVIRONMENT + AC_DEFUN(ERL_REENTRANT_FLEX, [flex_compile='$LEX -R -Pconftest -oconftest.c conftest.flex 1>&AC_FD_CC' changequote(253, 273)dnl @@ -188,111 +187,7 @@ CFLAGS="$CFLAGS $sanitizers" LDFLAGS="$LDFLAGS $sanitizers" ]) -dnl -dnl If ${ERL_TOP}/make/otp_ded.mk.in exists and contains DED_MK_VSN > 0, -dnl every thing releted to compiling Dynamic Erlang Drivers can be found -dnl in $(ERL_TOP)/make/$(TARGET)/ded.mk at compile time. If not, try to -dnl figure these things out. -dnl - -AC_MSG_CHECKING([for usable Dynamic Erlang Driver configuration]) -[ - ded_mk_in="${ERL_TOP}/make/otp_ded.mk.in" - ded_mk_vsn= - test -r "$ded_mk_in" && - ded_mk_vsn=`sed -n "s/^DED_MK_VSN[ ]*=[ ]*\(.*\)/\1/p" < "$ded_mk_in"` - test "$ded_mk_vsn" != "" || ded_mk_vsn=0 -] - -if test $ded_mk_vsn -gt 0; then - -HAVE_USABLE_OTP_DED_MK=yes -AC_MSG_RESULT([yes]) - -CC=false -AC_SUBST(CC) -DED_LD=false -AC_SUBST(DED_LD) - -else dnl --- begin no usable otp_ded.mk.in --- - -HAVE_USABLE_OTP_DED_MK=no -AC_MSG_RESULT([no]) - -dnl -dnl C compiler (related) defs -dnl - -AC_PROG_CC - -dnl -dnl Flags to the C compiler -dnl - -if test "X$host" = "Xwin32"; then - DED_CFLAGS="$CFLAGS" -else - case $host_os in - darwin*) - CFLAGS="$CFLAGS -fno-common" - ;; - esac - - if test "x$GCC" = xyes; then - DED_CFLAGS="$CFLAGS -fPIC $DED_CFLAGS" - else - DED_CFLAGS="$CFLAGS $DED_CFLAGS" - fi -fi - -dnl emulator includes needed -DED_INCLUDES="-I${ERL_TOP}/erts/emulator/beam -I${ERL_TOP}/erts/include -I${ERL_TOP}/erts/include/$host -I${ERL_TOP}/erts/include/internal -I${ERL_TOP}/erts/include/internal/$host -I${ERL_TOP}/erts/emulator/sys/$ERLANG_OSTYPE" - -DED_THR_DEFS="-D_THREAD_SAFE -D_REENTRANT" - -case $host_os in - win32) - DED_LDFLAGS="-dll" - ;; - solaris2*|sysv4*) - DED_LDFLAGS="-G" - ;; - aix4*) - DED_LDFLAGS="-G -bnoentry -bexpall" - ;; - freebsd2*) - # Non-ELF GNU linker - DED_LDFLAGS="-Bshareable" - ;; - darwin*) - # Mach-O linker, a shared lib and a loadable - # object file is not the same thing. - DED_LDFLAGS="-bundle -flat_namespace -undefined suppress" - DED_LD="$CC" - ;; - *) - # assume GNU linker and ELF - DED_LDFLAGS="-shared" - ;; -esac - -AC_CHECK_PROGS(DED_LD, [$LD ld.sh]) -AC_CHECK_TOOL(DED_LD, ld, no_ld) -if test "$DED_LD" = no_ld; then - AC_MSG_ERROR([ld is required to build the flex scanner!]) -fi - -AC_MSG_CHECKING(for linker flags for loadable drivers) -DED_LDFLAGS="$LDFLAGS $DED_LDFLAGS" -AC_MSG_RESULT([$DED_LDFLAGS]) - -fi dnl --- end no usable otp_ded.mk.in --- - -AC_SUBST(HAVE_USABLE_OTP_DED_MK) -AC_SUBST(DED_CFLAGS) -AC_SUBST(DED_INCLUDES) -AC_SUBST(DED_THR_DEFS) -AC_SUBST(DED_LDFLAGS) +ERL_DED AC_CHECK_PROG(PERL, perl, perl, no_perl) if test "$PERL" = no_perl; then diff --git a/lib/megaco/src/flex/Makefile.in b/lib/megaco/src/flex/Makefile.in index c37ad4d702..26d2ddd44c 100644 --- a/lib/megaco/src/flex/Makefile.in +++ b/lib/megaco/src/flex/Makefile.in @@ -31,25 +31,6 @@ include ../../vsn.mk VSN=$(MEGACO_VSN) # ---------------------------------------------------- -# Dynamic Erlang Driver -# ---------------------------------------------------- -HAVE_USABLE_OTP_DED_MK = @HAVE_USABLE_OTP_DED_MK@ - -ifeq ($(HAVE_USABLE_OTP_DED_MK),yes) -# otp_ded.mk will be used on R13B04 and later -include $(ERL_TOP)/make/$(TARGET)/otp_ded.mk -else -# megacos configure provide the info instead -DED_CC = @CC@ -DED__NOWARN_NOTHR_CFLAGS = @DED_CFLAGS@ -DED_THR_DEFS = @DED_THR_DEFS@ -DED_LD = @DED_LD@ -DED_LDFLAGS = @DED_LDFLAGS@ -DED_INCLUDES = @DED_INCLUDES@ -DED_EXT = so -endif - -# ---------------------------------------------------- # The following variables differ on different systems, we set # reasonable defaults, if something different is needed it should # be set for that system only. @@ -57,20 +38,19 @@ endif FLEX_VSN = $(shell flex --version) -TMP_CFLAGS = $(DED__NOWARN_NOTHR_CFLAGS) @OTP_EXTRA_FLAGS@ +TMP_CFLAGS = @DED_BASIC_CFLAGS@ @OTP_EXTRA_FLAGS@ ifeq ($(TYPE),valgrind) CFLAGS = $(subst -O2, , $(TMP_CFLAGS)) -DVALGRIND else CFLAGS = $(TMP_CFLAGS) endif -CC = $(DED_CC) -CFLAGS_MT = $(CFLAGS) $(DED_THR_DEFS) -LD = $(DED_LD) -LDFLAGS = $(DED_LDFLAGS) +CC = @DED_CC@ +CFLAGS_MT = $(CFLAGS) @DED_THR_DEFS@ +LD = @DED_LD@ +LDFLAGS = @DED_LDFLAGS@ LEX = @LEX@ LEXLIB = @LEXLIB@ PERL = @PERL@ -ERLANG_OSTYPE = @ERLANG_OSTYPE@ # Shall we build the flex scanner or not. # We assume that it does not exist on windows... @@ -143,8 +123,8 @@ ifeq ($(findstring win32,$(TARGET)), win32) FLEX_SCANNER_SO = SOLIBS = $(FLEX_SCANNER_SO) else -FLEX_SCANNER_SO = $(LIBDIR)/$(STD_DRV).$(DED_EXT) -FLEX_SCANNER_MT_SO = $(LIBDIR)/$(MT_DRV).$(DED_EXT) +FLEX_SCANNER_SO = $(LIBDIR)/$(STD_DRV).@DED_EXT@ +FLEX_SCANNER_MT_SO = $(LIBDIR)/$(MT_DRV).@DED_EXT@ SOLIBS = $(FLEX_SCANNER_SO) $(FLEX_SCANNER_MT_SO) endif @@ -179,7 +159,7 @@ else CFLAGS += -DMFS_FLEX_DEBUG=0 endif -CFLAGS += $(DED_INCLUDES) -I$(ERL_TOP)/erts/$(TARGET) $(DRV_FLAGS) -funroll-loops -Wall +CFLAGS += @DED_INCLUDE@ -I$(ERL_TOP)/erts/$(TARGET) $(DRV_FLAGS) -funroll-loops -Wall #ifneq ($(FLEX_VSN),) #CFLAGS += -DFLEX_VERSION="$(FLEX_VSN)" @@ -398,10 +378,10 @@ $(OBJDIR)/$(MT_DRV).o: $(MT_DRV).c # No need to link with -lfl as we have also defined %option noyywrap - # and having -lfl doesn't work under Darwin for some reason. - Sean -$(LIBDIR)/$(STD_DRV).$(DED_EXT): $(OBJDIR)/$(STD_DRV).o +$(LIBDIR)/$(STD_DRV).@DED_EXT@: $(OBJDIR)/$(STD_DRV).o $(V_colon)@echo "linking std driver:" $(V_LD) $(LDFLAGS) -o $@ $< -$(LIBDIR)/$(MT_DRV).$(DED_EXT): $(OBJDIR)/$(MT_DRV).o +$(LIBDIR)/$(MT_DRV).@DED_EXT@: $(OBJDIR)/$(MT_DRV).o $(V_colon)@echo "linking multi-threaded driver:" $(V_LD) $(LDFLAGS) -o $@ $< diff --git a/lib/mnesia/doc/src/notes.xml b/lib/mnesia/doc/src/notes.xml index 22fb5e8c48..01d1666b8d 100644 --- a/lib/mnesia/doc/src/notes.xml +++ b/lib/mnesia/doc/src/notes.xml @@ -752,7 +752,7 @@ <p> Returns the same value for mnesia_loader:disc_load_table/2 as - mnesia_loader:net_load_table/4 if a table copy can not be + mnesia_loader:net_load_table/4 if a table copy cannot be found. (Thanks to Uwe Dauernheim)</p> <p> Own Id: OTP-10015</p> diff --git a/lib/mnesia/examples/bench/README b/lib/mnesia/examples/bench/README index 5d31b5ba25..b8209b19b8 100644 --- a/lib/mnesia/examples/bench/README +++ b/lib/mnesia/examples/bench/README @@ -46,7 +46,7 @@ you need to: - put the $ERL_TOP/bin directory in your path on all nodes - bind IP adresses to hostnames (e.g via DNS or /etc/hosts) - - enable usage of rsh so it does not prompt for password + - enable usage of ssh so it does not prompt for password If you cannot achieve this, it is possible to run the benchmark anyway, but it requires more manual work to be done for each @@ -141,7 +141,7 @@ statistics_detail following atoms: normal, debug and debug2. debug enables a finer grain of statistics to be reported, but since it requires more counters, to be updated by the generator processes it may - cause slightly worse benchmark performace figures than the brief + cause slightly worse benchmark performance figures than the brief default case, that is normal. debug2 prints out the debug info and formats it according to LMC's benchmark program. @@ -160,7 +160,7 @@ n_fragments Defines how many fragments each table should be divided in. Default is 100. The fragments are evenly distributed over - all table nodes. The group table not devided in fragments. + all table nodes. The group table not divided in fragments. n_replicas diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index 223dba3f90..02bc884e36 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -160,7 +160,7 @@ {'sync_transaction', Retries::non_neg_integer()}. -type table() :: atom(). -type storage_type() :: 'ram_copies' | 'disc_copies' | 'disc_only_copies'. --type index_attr() :: atom() | non_neg_integer(). +-type index_attr() :: atom() | non_neg_integer() | {atom()}. -type write_locks() :: 'write' | 'sticky_write'. -type read_locks() :: 'read'. -type lock_kind() :: write_locks() | read_locks(). @@ -838,18 +838,20 @@ read(Tid, Ts, Tab, Key, LockKind) tid -> Store = Ts#tidstore.store, Oid = {Tab, Key}, - Objs = - case LockKind of - read -> - mnesia_locker:rlock(Tid, Store, Oid); - write -> - mnesia_locker:rwlock(Tid, Store, Oid); - sticky_write -> - mnesia_locker:sticky_rwlock(Tid, Store, Oid); - _ -> - abort({bad_type, Tab, LockKind}) - end, - add_written(?ets_lookup(Store, Oid), Tab, Objs); + ObjsFun = + fun() -> + case LockKind of + read -> + mnesia_locker:rlock(Tid, Store, Oid); + write -> + mnesia_locker:rwlock(Tid, Store, Oid); + sticky_write -> + mnesia_locker:sticky_rwlock(Tid, Store, Oid); + _ -> + abort({bad_type, Tab, LockKind}) + end + end, + add_written(?ets_lookup(Store, Oid), Tab, ObjsFun, LockKind); _Protocol -> dirty_read(Tab, Key) end; @@ -1202,14 +1204,20 @@ add_previous(_Tid, Ts, _Type, Tab) -> %% This routine fixes up the return value from read/1 so that %% it is correct with respect to what this particular transaction %% has already written, deleted .... etc +%% The actual read from the table is not done if not needed due to local +%% transaction context, and if so, no extra read lock is needed either. -add_written([], _Tab, Objs) -> - Objs; % standard normal fast case -add_written(Written, Tab, Objs) -> +add_written([], _Tab, ObjsFun, _LockKind) -> + ObjsFun(); % standard normal fast case +add_written(Written, Tab, ObjsFun, LockKind) -> case val({Tab, setorbag}) of bag -> - add_written_to_bag(Written, Objs, []); + add_written_to_bag(Written, ObjsFun(), []); + _ when LockKind == read; + LockKind == write -> + add_written_to_set(Written); _ -> + _ = ObjsFun(), % Fall back to request new lock and read from source add_written_to_set(Written) end. @@ -1269,6 +1277,14 @@ match_object(Tid, Ts, Tab, Pat, LockKind) match_object(_Tid, _Ts, Tab, Pat, _LockKind) -> abort({bad_type, Tab, Pat}). +add_written_index(Store, Pos, Tab, Key, Objs) when is_integer(Pos) -> + Pat = setelement(Pos, val({Tab, wild_pattern}), Key), + add_written_match(Store, Pat, Tab, Objs); +add_written_index(Store, Pos, Tab, Key, Objs) when is_tuple(Pos) -> + IxF = mnesia_index:index_vals_f(val({Tab, storage_type}), Tab, Pos), + Ops = find_ops(Store, Tab, '_'), + add_ix_match(Ops, Objs, IxF, Key, val({Tab, setorbag})). + add_written_match(S, Pat, Tab, Objs) -> Ops = find_ops(S, Tab, Pat), FixedRes = add_match(Ops, Objs, val({Tab, setorbag})), @@ -1295,6 +1311,46 @@ add_match([{_Oid, Val, write}|R], Objs, bag) -> add_match([{Oid, Val, write}|R], Objs, set) -> add_match(R, [Val | deloid(Oid,Objs)],set). +add_ix_match([], Objs, _IxF, _Key, _Type) -> + Objs; +add_ix_match(Written, Objs, IxF, Key, ordered_set) -> + %% Must use keysort which is stable + add_ordered_match(lists:keysort(1, ix_filter_ops(IxF, Key, Written)), Objs, []); +add_ix_match([{Oid, _, delete}|R], Objs, IxF, Key, Type) -> + add_ix_match(R, deloid(Oid, Objs), IxF, Key, Type); +add_ix_match([{_Oid, Val, delete_object}|R], Objs, IxF, Key, Type) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, lists:delete(Val, Objs), IxF, Key, Type); + false -> + add_ix_match(R, Objs, IxF, Key, Type) + end; +add_ix_match([{_Oid, Val, write}|R], Objs, IxF, Key, bag) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, [Val | lists:delete(Val, Objs)], IxF, Key, bag); + false -> + add_ix_match(R, Objs, IxF, Key, bag) + end; +add_ix_match([{Oid, Val, write}|R], Objs, IxF, Key, set) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, [Val | deloid(Oid,Objs)],IxF,Key,set); + false -> + add_ix_match(R, Objs, IxF, Key, set) + end. + +ix_match(Val, IxF, Key) -> + lists:member(Key, IxF(Val)). + +ix_filter_ops(IxF, Key, Ops) -> + lists:filter( + fun({_Oid, Obj, write}) -> + ix_match(Obj, IxF, Key); + (_) -> + true + end, Ops). + %% For ordered_set only !! add_ordered_match(Written = [{{_, Key}, _, _}|_], [Obj|Objs], Acc) when Key > element(2, Obj) -> @@ -1633,6 +1689,16 @@ index_match_object(Tid, Ts, Tab, Pat, Attr, LockKind) dirty_index_match_object(Tab, Pat, Attr); % Should be optimized? tid -> case mnesia_schema:attr_tab_to_pos(Tab, Attr) of + {_} -> + case LockKind of + read -> + Store = Ts#tidstore.store, + mnesia_locker:rlock_table(Tid, Store, Tab), + Objs = dirty_match_object(Tab, Pat), + add_written_match(Store, Pat, Tab, Objs); + _ -> + abort({bad_type, Tab, LockKind}) + end; Pos when Pos =< tuple_size(Pat) -> case LockKind of read -> @@ -1680,8 +1746,8 @@ index_read(Tid, Ts, Tab, Key, Attr, LockKind) false -> Store = Ts#tidstore.store, Objs = mnesia_index:read(Tid, Store, Tab, Key, Pos), - Pat = setelement(Pos, val({Tab, wild_pattern}), Key), - add_written_match(Store, Pat, Tab, Objs); + add_written_index( + Ts#tidstore.store, Pos, Tab, Key, Objs); true -> abort({bad_type, Tab, Attr, Key}) end; @@ -1817,7 +1883,7 @@ remote_dirty_match_object(Tab, Pat) -> false -> mnesia_lib:db_match_object(Tab, Pat); true -> - PosList = val({Tab, index}), + PosList = regular_indexes(Tab), remote_dirty_match_object(Tab, Pat, PosList) end. @@ -1849,7 +1915,7 @@ remote_dirty_select(Tab, Spec) -> false -> mnesia_lib:db_select(Tab, Spec); true -> - PosList = val({Tab, index}), + PosList = regular_indexes(Tab), remote_dirty_select(Tab, Spec, PosList) end; _ -> @@ -1916,6 +1982,8 @@ dirty_index_match_object(Pat, _Attr) -> dirty_index_match_object(Tab, Pat, Attr) when is_atom(Tab), Tab /= schema, is_tuple(Pat), tuple_size(Pat) > 2 -> case mnesia_schema:attr_tab_to_pos(Tab, Attr) of + {_} -> + dirty_match_object(Tab, Pat); Pos when Pos =< tuple_size(Pat) -> case has_var(element(2, Pat)) of false -> @@ -3246,3 +3314,7 @@ put_activity_id(Activity) -> mnesia_tm:put_activity_id(Activity). put_activity_id(Activity,Fun) -> mnesia_tm:put_activity_id(Activity,Fun). + +regular_indexes(Tab) -> + PosList = val({Tab, index}), + [P || P <- PosList, is_integer(P)]. diff --git a/lib/mnesia/src/mnesia_dumper.erl b/lib/mnesia/src/mnesia_dumper.erl index cb2ee504f9..8ab11be2d3 100644 --- a/lib/mnesia/src/mnesia_dumper.erl +++ b/lib/mnesia/src/mnesia_dumper.erl @@ -272,17 +272,12 @@ do_insert_rec(Tid, Rec, InPlace, InitBy, LogV) -> end end, D = Rec#commit.disc_copies, - ExtOps = commit_ext(Rec), insert_ops(Tid, disc_copies, D, InPlace, InitBy, LogV), - [insert_ops(Tid, Ext, Ops, InPlace, InitBy, LogV) || - {Ext, Ops} <- ExtOps, - storage_semantics(Ext) == disc_copies], + insert_ext_ops(Tid, commit_ext(Rec), InPlace, InitBy), case InitBy of startup -> DO = Rec#commit.disc_only_copies, - insert_ops(Tid, disc_only_copies, DO, InPlace, InitBy, LogV), - [insert_ops(Tid, Ext, Ops, InPlace, InitBy, LogV) || - {Ext, Ops} <- ExtOps, storage_semantics(Ext) == disc_only_copies]; + insert_ops(Tid, disc_only_copies, DO, InPlace, InitBy, LogV); _ -> ignore end. @@ -290,11 +285,8 @@ do_insert_rec(Tid, Rec, InPlace, InitBy, LogV) -> commit_ext(#commit{ext = []}) -> []; commit_ext(#commit{ext = Ext}) -> case lists:keyfind(ext_copies, 1, Ext) of - {_, C} -> - lists:foldl(fun({Ext0, Op}, D) -> - orddict:append(Ext0, Op, D) - end, orddict:new(), C); - false -> [] + {_, C} -> C; + false -> [] end. update(_Tid, [], _DumperMode) -> @@ -330,6 +322,21 @@ perform_update(Tid, SchemaOps, _DumperMode, _UseDir) -> fatal("Schema update error ~tp ~tp", [{Reason,ST}, SchemaOps]) end. +insert_ext_ops(Tid, ExtOps, InPlace, InitBy) -> + %% Note: ext ops cannot be part of pre-4.3 logs, so there's no need + %% to support the old operation order, as in `insert_ops' + lists:foreach( + fun ({Ext, Op}) -> + case storage_semantics(Ext) of + Semantics when Semantics == disc_copies; + Semantics == disc_only_copies, InitBy == startup -> + insert_op(Tid, Ext, Op, InPlace, InitBy); + _Other -> + ok + end + end, + ExtOps). + insert_ops(_Tid, _Storage, [], _InPlace, _InitBy, _) -> ok; insert_ops(Tid, Storage, [Op], InPlace, InitBy, Ver) when Ver >= "4.3"-> insert_op(Tid, Storage, Op, InPlace, InitBy), diff --git a/lib/mnesia/src/mnesia_index.erl b/lib/mnesia/src/mnesia_index.erl index 098265d5fc..6f1c21e3b9 100644 --- a/lib/mnesia/src/mnesia_index.erl +++ b/lib/mnesia/src/mnesia_index.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% +%% %% Copyright Ericsson AB 1996-2018. All Rights Reserved. -%% +%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at @@ -14,7 +14,7 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%% +%% %% %CopyrightEnd% %% @@ -37,7 +37,7 @@ db_match_erase/2, get_index_table/2, get_index_table/3, - + tab2filename/2, init_index/2, init_indecies/3, @@ -45,6 +45,7 @@ del_transient/3, del_index_table/3, + index_vals_f/3, index_info/2, ext_index_instances/1]). @@ -60,9 +61,14 @@ read(Tid, Store, Tab, IxKey, Pos) -> ResList = mnesia_locker:ixrlock(Tid, Store, Tab, IxKey, Pos), %% Remove all tuples which don't include Ixkey, happens when Tab is a bag case val({Tab, setorbag}) of - bag -> + bag when is_integer(Pos) -> mnesia_lib:key_search_all(IxKey, Pos, ResList); - _ -> + bag when is_tuple(Pos) -> + TabStorage = val({Tab, storage_type}), + ValsF = index_vals_f(TabStorage, Tab, Pos), + [Obj || Obj <- ResList, + lists:member(IxKey, ValsF(Obj))]; + _ -> ResList end. @@ -136,7 +142,7 @@ del_object_index2([], _, _Storage, _Tab, _K, _Obj) -> ok; del_object_index2([{{Pos, Type}, Ixt} | Tail], SoB, Storage, Tab, K, Obj) -> ValsF = index_vals_f(Storage, Tab, Pos), case SoB of - bag -> + bag -> del_object_bag(Type, ValsF, Tab, K, Obj, Ixt); _ -> %% If set remove the tuple in index table del_ixes(Type, Ixt, ValsF, Obj, K) @@ -197,7 +203,7 @@ merge([], _, _, Ack) -> realkeys(Tab, Pos, IxKey) -> Index = get_index_table(Tab, Pos), db_get(Index, IxKey). % a list on the form [{IxKey, RealKey1} , .... - + dirty_select(Tab, Spec, Pos) when is_integer(Pos) -> %% Assume that we are on the node where the replica is %% Returns the records without applying the match spec @@ -233,7 +239,7 @@ dirty_read2(Tab, IxKey, Pos) -> end, Acc, mnesia_lib:db_get(Storage, Tab, K)) end, [], Keys)). -pick_index([{{{Pfx,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> +pick_index([{{{Pfx,_,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> {IxType, Ixt}; pick_index([{{Pos,IxType}, Ixt}|_], _Tab, Pos) -> {IxType, Ixt}; @@ -242,7 +248,7 @@ pick_index([_|T], Tab, Pos) -> pick_index([], Tab, Pos) -> mnesia:abort({no_exist, Tab, {index, Pos}}). - + %%%%%%% Creation, Init and deletion routines for index tables %% We can have several indexes on the same table @@ -387,12 +393,12 @@ init_ext_index(Tab, Storage, Alias, Mod, [{Pos,Type} | Tail]) -> create_fun(Cont, Tab, Pos) -> IxF = index_vals_f(disc_only_copies, Tab, Pos), fun(read) -> - Data = + Data = case Cont of {start, KeysPerChunk} -> mnesia_lib:db_init_chunk( disc_only_copies, Tab, KeysPerChunk); - '$end_of_table' -> + '$end_of_table' -> '$end_of_table'; _Else -> mnesia_lib:db_chunk(disc_only_copies, Cont) @@ -462,7 +468,7 @@ add_index_info(Tab, SetOrBag, IxElem) -> %% Check later if mnesia_tm is sensitive about the order mnesia_lib:set({Tab, index_info}, IndexInfo), mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit([IndexInfo | Commit])); {value, Old} -> %% We could check for consistency here @@ -470,7 +476,7 @@ add_index_info(Tab, SetOrBag, IxElem) -> mnesia_lib:set({Tab, index_info}, Index), mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end. @@ -488,19 +494,19 @@ del_index_info(Tab, Pos) -> element(1,P)=/=Pos end, Old#index.pos_list) of - [] -> + [] -> IndexInfo = index_info(Old#index.setorbag,[]), mnesia_lib:set({Tab, index_info}, IndexInfo), mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), NewC = lists:keydelete(index, 1, Commit), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)); New -> Index = Old#index{pos_list = New}, mnesia_lib:set({Tab, index_info}, Index), mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end end. @@ -537,7 +543,7 @@ db_match_erase({{ext,_,_} = Ext, Ixt}, Pat) -> mnesia_lib:db_match_erase(Ext, Ixt, Pat); db_match_erase({dets, Ixt}, Pat) -> ok = dets:match_delete(Ixt, Pat). - + db_select({ram, Ixt}, Pat) -> ets:select(Ixt, Pat); db_select({{ext,_,_} = Ext, Ixt}, Pat) -> @@ -545,7 +551,7 @@ db_select({{ext,_,_} = Ext, Ixt}, Pat) -> db_select({dets, Ixt}, Pat) -> dets:select(Ixt, Pat). - + get_index_table(Tab, Pos) -> get_index_table(Tab, val({Tab, storage_type}), Pos). diff --git a/lib/mnesia/src/mnesia_lib.erl b/lib/mnesia/src/mnesia_lib.erl index a884b8e086..6abc05fade 100644 --- a/lib/mnesia/src/mnesia_lib.erl +++ b/lib/mnesia/src/mnesia_lib.erl @@ -929,7 +929,7 @@ error_desc(no_transaction) -> "Operation not allowed outside transactions"; error_desc(combine_error) -> "Table options were ilegally combined"; error_desc(bad_index) -> "Index already exists or was out of bounds"; error_desc(already_exists) -> "Some schema option we try to set is already on"; -error_desc(index_exists)-> "Some ops can not be performed on tabs with index"; +error_desc(index_exists)-> "Some ops cannot be performed on tabs with index"; error_desc(no_exists)-> "Tried to perform op on non-existing (non alive) item"; error_desc(system_limit) -> "Some system_limit was exhausted"; error_desc(mnesia_down) -> "A transaction involving objects at some remote " diff --git a/lib/mnesia/src/mnesia_tm.erl b/lib/mnesia/src/mnesia_tm.erl index 4b3fffc735..cbf7db28f0 100644 --- a/lib/mnesia/src/mnesia_tm.erl +++ b/lib/mnesia/src/mnesia_tm.erl @@ -1661,7 +1661,7 @@ commit_participant(Coord, Tid, Bin, C0, DiscNs, _RamNs) -> ?eval_debug_fun({?MODULE, commit_participant, pre}, [{tid, Tid}]), try mnesia_schema:prepare_commit(Tid, C0, {part, Coord}) of {Modified, C = #commit{}, DumperMode} -> - %% If we can not find any local unclear decision + %% If we cannot find any local unclear decision %% we should presume abort at startup recovery case lists:member(node(), DiscNs) of false -> diff --git a/lib/mnesia/test/Makefile b/lib/mnesia/test/Makefile index 5b61b1af65..b43bc82801 100644 --- a/lib/mnesia/test/Makefile +++ b/lib/mnesia/test/Makefile @@ -53,7 +53,8 @@ MODULES= \ mnesia_measure_test \ mnesia_cost \ mnesia_dbn_meters \ - ext_test + ext_test \ + mnesia_index_plugin_test DocExamplesDir := ../doc/src/ diff --git a/lib/mnesia/test/README b/lib/mnesia/test/README index e0ced7399d..30a0d2fd64 100644 --- a/lib/mnesia/test/README +++ b/lib/mnesia/test/README @@ -51,7 +51,7 @@ stated as test suite configuration parameters, but by default the extra node names are generated. In this example the names will be: a, a1 and a2. It is enough to start the first node manually, the extra nodes will automatically be started if -neccessary. +necessary. The attached UNIX shell script mt, does not work on all platforms, but it may be used as a source for inspiration. It @@ -63,7 +63,7 @@ test cases (i.e. test cases that encountered an error). During development we want to be able to run the test cases in the debugger. This demands a little bit of preparations: - - Start the neccessary number of nodes (normally 3). + - Start the necessary number of nodes (normally 3). This may either be done by running the mt script or by starting the main node and then invoke mt:start_nodes() to start the extra nodes with slave. @@ -73,7 +73,7 @@ in the debugger. This demands a little bit of preparations: - Load all files that needs to be interpreted. This is typically all Mnesia files plus the test case. By invoking mnesia:ni() - and mnesia:ni([TestModule]) the neccessary modules will be + and mnesia:ni([TestModule]) the necessary modules will be loaded on all CONNECTED nodes. The test case execution is supervised in order to ensure that no test diff --git a/lib/mnesia/test/mnesia_SUITE.erl b/lib/mnesia/test/mnesia_SUITE.erl index 24c1def6da..b41bf22efa 100644 --- a/lib/mnesia/test/mnesia_SUITE.erl +++ b/lib/mnesia/test/mnesia_SUITE.erl @@ -69,12 +69,13 @@ groups() -> %% covered. [{light, [], [{group, install}, {group, nice}, {group, evil}, - {group, mnesia_frag_test, light}, {group, qlc}, + {group, mnesia_frag_test, light}, {group, qlc}, {group, index_plugins}, {group, registry}, {group, config}, {group, examples}]}, {install, [], [{mnesia_install_test, all}]}, {nice, [], [{mnesia_nice_coverage_test, all}]}, {evil, [], [{mnesia_evil_coverage_test, all}]}, {qlc, [], [{mnesia_qlc_test, all}]}, + {index_plugins, [], [{mnesia_index_plugin_test, all}]}, {registry, [], [{mnesia_registry_test, all}]}, {config, [], [{mnesia_config_test, all}]}, {examples, [], [{mnesia_examples_test, all}]}, diff --git a/lib/mnesia/test/mnesia_index_plugin_test.erl b/lib/mnesia/test/mnesia_index_plugin_test.erl new file mode 100644 index 0000000000..44fe047c50 --- /dev/null +++ b/lib/mnesia/test/mnesia_index_plugin_test.erl @@ -0,0 +1,261 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +-module(mnesia_index_plugin_test). +-author('[email protected]'). + +-export([init_per_testcase/2, end_per_testcase/2, + init_per_group/2, end_per_group/2, + init_per_suite/1, end_per_suite/1, + all/0, groups/0]). + +-export([ + add_rm_plugin/1, + tab_with_plugin_index/1, + tab_with_multiple_plugin_indexes/1, + ix_match_w_plugin/1, + ix_match_w_plugin_ordered/1, + ix_match_w_plugin_bag/1 + ]). + +-export([ix_prefixes/3, % test plugin + ix_prefixes2/3]). % test plugin 2 + +-include("mnesia_test_lib.hrl"). + +init_per_suite(Conf) -> + Conf. + +end_per_suite(Conf) -> + Conf. + +init_per_testcase(Func, Conf) -> + mnesia_test_lib:init_per_testcase(Func, Conf). + +end_per_testcase(Func, Conf) -> + mnesia_test_lib:end_per_testcase(Func, Conf). + +all() -> + [add_rm_plugin, + tab_with_plugin_index, + tab_with_multiple_plugin_indexes, + ix_match_w_plugin, + ix_match_w_plugin_ordered, + ix_match_w_plugin_bag]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + +add_rm_plugin(suite) -> []; +add_rm_plugin(Config) when is_list(Config) -> + [N1, N2] = Nodes = ?acquire_nodes(2, Config), + ok = add_plugin(), + ok = rpc_check_plugin(N1), + ok = rpc_check_plugin(N2), + ok = add_plugin2(), + ok = del_plugin(), + ok = del_plugin2(), + ok = add_plugin(), + ok = add_plugin2(), + ok = del_plugin(), + ok = del_plugin2(), + ?verify_mnesia(Nodes, []). + +-define(PLUGIN1, {{pfx},?MODULE,ix_prefixes}). +-define(PLUGIN2, {{pfx2},?MODULE,ix_prefixes2}). + +add_plugin() -> + {atomic, ok} = mnesia_schema:add_index_plugin({pfx}, ?MODULE, ix_prefixes), + [?PLUGIN1] = mnesia_schema:index_plugins(), + ok. + +add_plugin2() -> + {atomic, ok} = mnesia_schema:add_index_plugin({pfx2}, ?MODULE, ix_prefixes2), + [?PLUGIN1, ?PLUGIN2] = lists:sort(mnesia_schema:index_plugins()), + ok. + +del_plugin() -> + {atomic, ok} = mnesia_schema:delete_index_plugin({pfx}), + [?PLUGIN2] = mnesia_schema:index_plugins(), + ok. + +del_plugin2() -> + {atomic, ok} = mnesia_schema:delete_index_plugin({pfx2}), + [] = mnesia_schema:index_plugins(), + ok. + +rpc_check_plugin(N) -> + [?PLUGIN1] = + rpc:call(N, mnesia_schema, index_plugins, []), + ok. + +tab_with_plugin_index(suite) -> []; +tab_with_plugin_index(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(t, [{attributes, [k,v1,v2]}, + {index, [{{pfx}, ordered}, + {v1, ordered}, + v2]}]), + [ok,ok,ok,ok] = + [mnesia:dirty_write({t, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"6789"}, + {4,d,nil}]], + [{t,1,a,"123"},{t,2,b,"12345"}] = + mnesia:dirty_index_read(t,<<"123">>,{pfx}), + [{t,3,c,"6789"}] = + mnesia:dirty_index_read(t,"6789",v2), + [{t,1,a,"123"}] = + mnesia:dirty_match_object({t,'_',a,"123"}), + [{t,1,a,"123"}] = + mnesia:dirty_select(t, [{ {t,'_',a,"123"}, [], ['$_']}]), + mnesia:dirty_delete(t,2), + [{t,1,a,"123"}] = + mnesia:dirty_index_read(t,<<"123">>,{pfx}), + ?verify_mnesia(Nodes, []). + +tab_with_multiple_plugin_indexes(suite) -> []; +tab_with_multiple_plugin_indexes(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + ok = add_plugin2(), + {atomic, ok} = + mnesia:create_table(u, [{attributes, [k,v1,v2]}, + {index, [{{pfx}, ordered}, + {{pfx2}, ordered}]}]), + [ok,ok,ok,ok] = + [mnesia:dirty_write({u, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"6789"}, + {4,d,nil}]], + [{u,1,a,"123"},{u,2,b,"12345"}] = + mnesia:dirty_index_read(u,<<"123">>,{pfx}), + [{u,1,a,"123"},{u,2,b,"12345"}] = + mnesia:dirty_index_read(u,<<"321">>,{pfx2}), + ?verify_mnesia(Nodes, []). + +ix_match_w_plugin(suite) -> []; +ix_match_w_plugin(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im1, [{attributes, [k, v1, v2]}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im1, set), + ?verify_mnesia(Nodes, []). + + +ix_match_w_plugin_ordered(suite) -> []; +ix_match_w_plugin_ordered(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im2, [{attributes, [k, v1, v2]}, + {type, ordered_set}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im2, ordered_set), + ?verify_mnesia(Nodes, []). + +ix_match_w_plugin_bag(suite) -> []; +ix_match_w_plugin_bag(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im3, [{attributes, [k, v1, v2]}, + {type, bag}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im3, bag), + ?verify_mnesia(Nodes, []). + +fill_and_test_index_match(Tab, Type) -> + [ok,ok,ok,ok,ok,ok,ok,ok,ok] = + [mnesia:dirty_write({Tab, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"123"}, + {4,d,nil}, + {5,e,nil}, + {6,f,nil}, + {7,g,nil}, %% overwritten if not bag + {7,g,"234"}, + {8,h,"123"}]], + mnesia:activity( + transaction, + fun() -> + ok = mnesia:write({Tab, 1, aa, "1234"}), %% replaces if not bag + ok = mnesia:delete({Tab, 2}), + ok = mnesia:delete({Tab, 4}), + ok = mnesia:write({Tab, 6, ff, nil}), + ok = mnesia:write({Tab, 7, gg, "123"}), + ok = mnesia:write({Tab, 100, x, nil}), + ok = mnesia:delete_object({Tab,3,c,"123"}), + ok = mnesia:delete_object({Tab,5,e,nil}), + Res = mnesia:index_read(Tab, <<"123">>, {pfx}), + SetRes = [{Tab,1,aa,"1234"}, {Tab,7,gg,"123"}, {Tab,8,h,"123"}], + case Type of + set -> + SetRes = lists:sort(Res); + ordered_set -> + SetRes = Res; + bag -> + [{Tab,1,a,"123"}, {Tab,1,aa,"1234"}, + {Tab,7,gg,"123"}, {Tab,8,h,"123"}] = lists:sort(Res) + end + end). + +%% ============================================================ +%% +ix_prefixes(_Tab, _Pos, Obj) -> + lists:foldl( + fun(V, Acc) when is_list(V) -> + try Pfxs = prefixes(list_to_binary(V)), + Pfxs ++ Acc + catch + error:_ -> + Acc + end; + (V, Acc) when is_binary(V) -> + Pfxs = prefixes(V), + Pfxs ++ Acc; + (_, Acc) -> + Acc + end, [], tl(tuple_to_list(Obj))). + +ix_prefixes2(Tab, Pos, Obj) -> + [rev(P) || P <- ix_prefixes(Tab, Pos, Obj)]. + +rev(B) when is_binary(B) -> + list_to_binary(lists:reverse(binary_to_list(B))). + +prefixes(<<P:3/binary, _/binary>>) -> + [P]; +prefixes(_) -> + []. diff --git a/lib/mnesia/test/mt.erl b/lib/mnesia/test/mt.erl index 5a981bf539..037d6adb38 100644 --- a/lib/mnesia/test/mt.erl +++ b/lib/mnesia/test/mt.erl @@ -67,6 +67,7 @@ alias(recovery) -> mnesia_recovery_test; alias(registry) -> mnesia_registry_test; alias(suite) -> mnesia_SUITE; alias(trans) -> mnesia_trans_access_test; +alias(ixp) -> mnesia_index_plugin_test; alias(Other) -> Other. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/observer/src/cdv_bin_cb.erl b/lib/observer/src/cdv_bin_cb.erl index a4a542297c..91d33474c8 100644 --- a/lib/observer/src/cdv_bin_cb.erl +++ b/lib/observer/src/cdv_bin_cb.erl @@ -49,7 +49,7 @@ format_bin_fun(Format,Bin) -> try io_lib:format(Format,[Bin]) of Str -> plain_html(lists:flatten(Str)) catch error:badarg -> - Warning = "This binary can not be formatted with " ++ Format, + Warning = "This binary cannot be formatted with " ++ Format, observer_html_lib:warning(Warning) end end. @@ -59,7 +59,7 @@ binary_to_term_fun(Bin) -> try binary_to_term(Bin) of Term -> plain_html(io_lib:format("~tp",[Term])) catch error:badarg -> - Warning = "This binary can not be converted to an Erlang term", + Warning = "This binary cannot be converted to an Erlang term", observer_html_lib:warning(Warning) end end. diff --git a/lib/observer/src/cdv_term_cb.erl b/lib/observer/src/cdv_term_cb.erl index 91de6449c4..85da1d227a 100644 --- a/lib/observer/src/cdv_term_cb.erl +++ b/lib/observer/src/cdv_term_cb.erl @@ -51,7 +51,7 @@ format_term_fun(Format,Term,Tab) -> try io_lib:format(Format,[Term]) of Str -> {expand, plain_html(Str), Tab} catch error:badarg -> - Warning = "This term can not be formatted with " ++ Format, + Warning = "This term cannot be formatted with " ++ Format, observer_html_lib:warning(Warning) after observer_lib:report_progress({ok,stop_pulse}) diff --git a/lib/observer/src/cdv_wx.erl b/lib/observer/src/cdv_wx.erl index 811c767e66..7100cc8790 100644 --- a/lib/observer/src/cdv_wx.erl +++ b/lib/observer/src/cdv_wx.erl @@ -459,10 +459,7 @@ maybe_warn_filename(FileName) -> true -> continue; false -> - DumpName = case os:getenv("ERL_CRASH_DUMP") of - false -> filename:absname("erl_crash.dump"); - Name -> filename:absname(Name) - end, + DumpName = filename:absname(os:getenv("ERL_CRASH_DUMP", "erl_crash.dump")), case filename:absname(FileName) of DumpName -> Warning = diff --git a/lib/observer/src/observer_tv_wx.erl b/lib/observer/src/observer_tv_wx.erl index 247b3e869f..9743a6ed42 100644 --- a/lib/observer/src/observer_tv_wx.erl +++ b/lib/observer/src/observer_tv_wx.erl @@ -148,7 +148,7 @@ handle_event(#wx{event=#wxList{type=command_list_item_activated, State=#state{holder=Holder, node=Node, opts=#opts{type=Type}, grid=Grid}) -> case get_table(Holder, Index) of #tab{protection=private} -> - self() ! {error, "Table has 'private' protection and can not be read"}; + self() ! {error, "Table has 'private' protection and cannot be read"}; #tab{}=Table -> observer_tv_table:start_link(Grid, [{node,Node}, {type,Type}, {table,Table}]); _ -> ignore @@ -188,7 +188,7 @@ handle_event(#wx{id=?ID_SHOW_TABLE}, R when is_integer(R) -> case get_table(Holder, R) of #tab{protection=private} -> - self() ! {error, "Table has 'private' protection and can not be read"}; + self() ! {error, "Table has 'private' protection and cannot be read"}; #tab{}=Table -> observer_tv_table:start_link(Grid, [{node,Node}, {type,Type}, {table,Table}]); _ -> ignore diff --git a/lib/observer/src/observer_wx.erl b/lib/observer/src/observer_wx.erl index 71db586845..de7d821030 100644 --- a/lib/observer/src/observer_wx.erl +++ b/lib/observer/src/observer_wx.erl @@ -820,7 +820,7 @@ is_rb_compatible(Node) -> is_rb_server_running(Node, LogState) -> %% If already started, somebody else may use it. - %% We can not use it too, as far log file would be overriden. Not fair. + %% We cannot use it too, as far log file would be overriden. Not fair. case rpc:block_call(Node, erlang, whereis, [rb_server]) of Pid when is_pid(Pid), (LogState == false) -> throw("Error: rb_server is already started and maybe used by someone."); diff --git a/lib/observer/test/crashdump_helper.erl b/lib/observer/test/crashdump_helper.erl index d6b5eff9b5..84ed99afa5 100644 --- a/lib/observer/test/crashdump_helper.erl +++ b/lib/observer/test/crashdump_helper.erl @@ -24,7 +24,7 @@ create_binaries/0,create_sub_binaries/1, dump_persistent_terms/0, create_persistent_terms/0]). --compile(r18). +-compile(r20). -include_lib("common_test/include/ct.hrl"). n1_proc(N2,Creator) -> diff --git a/lib/observer/test/crashdump_viewer_SUITE_data/old_format.dump b/lib/observer/test/crashdump_viewer_SUITE_data/old_format.dump index 2c8944fa9d..0b30fdf2f1 100644 --- a/lib/observer/test/crashdump_viewer_SUITE_data/old_format.dump +++ b/lib/observer/test/crashdump_viewer_SUITE_data/old_format.dump @@ -10,7 +10,7 @@ Compiled on Wed Dec 4 11:11:21 2002 Process Information -------------------------------------------------- <0.0.0> Waiting. Registered as: init -Spawned as: otp_ring0:start/2 +Spawned as: erl_init:start/2 Message buffer data: 4 words Link list: [<0.5.0>,<0.4.0>,<0.2.0>] Reductions 3125 stack+heap 610 old_heap_sz=233 @@ -68,7 +68,7 @@ y(3) [gen_event,<0.1.0>,<0.1.0>,{local,error_logger},[],[],[]] Spawned as: proc_lib:init_p/5 Message buffer data: 4 words Link list: [<0.7.0>,<0.0.0>] -Dictionary: [{'$initial_call',{gen,init_it,[gen_server,<0.1.0>,<0.1.0>,{local,application_controller},application_controller,{application,kernel,[{description,"ERTS CXC 138 10"},{vsn,"2.6.3.15"},{id,[]},{modules,[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,otp_ring0]},{registered,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2]},{applications,[]},{included_applications,[]},{env,[{error_logger,tty}]},{start_phases,undefined},{maxT,infinity},{maxP,infinity},{mod,{kernel,[]}}]},[]]}},{'$ancestors',[<0.1.0>]}] +Dictionary: [{'$initial_call',{gen,init_it,[gen_server,<0.1.0>,<0.1.0>,{local,application_controller},application_controller,{application,kernel,[{description,"ERTS CXC 138 10"},{vsn,"2.6.3.15"},{id,[]},{modules,[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,erl_init]},{registered,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2]},{applications,[]},{included_applications,[]},{env,[{error_logger,tty}]},{start_phases,undefined},{maxT,infinity},{maxP,infinity},{mod,{kernel,[]}}]},[]]}},{'$ancestors',[<0.1.0>]}] Reductions 527 stack+heap 1597 old_heap_sz=0 Heap unused=833 OldHeap unused=0 Stack dump: @@ -88,13 +88,13 @@ y(5) <0.1.0> y(0) Catch 0x33480C (proc_lib:init_p/5 + 164) y(1) gen y(2) init_it -y(3) [gen_server,<0.1.0>,<0.1.0>,{local,application_controller},application_controller,{application,kernel,[{description,"ERTS CXC 138 10"},{vsn,"2.6.3.15"},{id,[]},{modules,[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,otp_ring0]},{registered,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2]},{applications,[]},{included_applications,[]},{env,[{error_logger,tty}]},{start_phases,undefined},{maxT,infinity},{maxP,infinity},{mod,{kernel,[]}}]},[]] +y(3) [gen_server,<0.1.0>,<0.1.0>,{local,application_controller},application_controller,{application,kernel,[{description,"ERTS CXC 138 10"},{vsn,"2.6.3.15"},{id,[]},{modules,[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,erl_init]},{registered,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2]},{applications,[]},{included_applications,[]},{env,[{error_logger,tty}]},{start_phases,undefined},{maxT,infinity},{maxP,infinity},{mod,{kernel,[]}}]},[]] -------------------------------------------------- <0.7.0> Waiting. Spawned as: proc_lib:init_p/5 Message buffer data: 0 words Link list: [<0.8.0>,<0.5.0>] -Dictionary: [{'$initial_call',{application_master,init,[<0.5.0>,<0.6.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,otp_ring0],[],infinity,infinity},normal]}},{'$ancestors',[<0.6.0>]}] +Dictionary: [{'$initial_call',{application_master,init,[<0.5.0>,<0.6.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,erl_init],[],infinity,infinity},normal]}},{'$ancestors',[<0.6.0>]}] Reductions 45 stack+heap 377 old_heap_sz=377 Heap unused=306 OldHeap unused=377 Stack dump: @@ -103,14 +103,14 @@ cp = 0x33480c (proc_lib:init_p/5 + 164) arity = 0 347174 Return addr 0x33480C (proc_lib:init_p/5 + 164) -y(0) {state,<0.8.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,otp_ring0],[],infinity,infinity},[],0,<0.0.0>} +y(0) {state,<0.8.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,erl_init],[],infinity,infinity},[],0,<0.0.0>} y(1) <0.5.0> 347180 Return addr 0xFCEC8 (<terminate process normally>) y(0) Catch 0x33480C (proc_lib:init_p/5 + 164) y(1) application_master y(2) init -y(3) [<0.5.0>,<0.6.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,otp_ring0],[],infinity,infinity},normal] +y(3) [<0.5.0>,<0.6.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_server,code_server,disk_log_server,disk_log_sup,erl_prim_loader,error_logger,file_server,fixtable_server,global_group,global_name_server,heart,init,kernel_config,kernel_sup,net_kernel,net_sup,rex,user,os_server,ddll_server,erl_epmd,inet_db,pg2],undefined,{kernel,[]},[application,application_controller,application_master,application_starter,auth,code,code_aux,code_server,code_server_int,dist_util,erl_boot_server,erl_distribution,erl_open_port,erl_prim_loader,erl_reply,erlang,error_handler,error_logger,file,global,global_group,global_search,group,heart,inet6_tcp,inet6_tcp_dist,inet6_udp,inet_config,inet_hosts,inet_gethost_native,inet_tcp_dist,otp_pre_init,init,kernel,kernel_config,net,net_adm,net_kernel,os,ram_file,rpc,user,user_drv,user_sup,disk_log,disk_log_1,disk_log_server,disk_log_sup,dist_ac,erl_atom_cache,erl_ddll,erl_epmd,erl_external,erts_debug,fixtable_server,gen_tcp,gen_udp,prim_inet,inet,inet_db,inet_dns,inet_parse,inet_res,inet_tcp,inet_udp,pg2,seq_trace,socks5,socks5_auth,socks5_tcp,socks5_udp,wrap_log_reader,erl_init],[],infinity,infinity},normal] -------------------------------------------------- <0.8.0> Waiting. Spawned as: application_master:start_it/4 @@ -652,7 +652,7 @@ Not alive Loaded Modules Information -------------------------------------------------- -otp_ring0 448 +erl_init 448 init 28000 prim_inet 34800 erl_prim_loader 14187 @@ -1202,7 +1202,7 @@ udp_error inet_async inet_reply empty_out_q -otp_ring0 +erl_init boot init run @@ -1285,7 +1285,7 @@ sleep start_prim_loader erl_prim_loader set_path -'can not start loader' +'cannot start loader' add_to_kernel prim_load_flags '-loader' @@ -1304,7 +1304,7 @@ path_flags '-pz' get_boot not_found -'can not get bootfile' +'cannot get bootfile' 'bootfile format error' get_file script @@ -1349,7 +1349,7 @@ extension '-boot/1-fun-0-' '-bs2ss/1-fun-0-' '-bs2as/1-fun-0-' -'can not load' +'cannot load' 'unexpected command in bootfile' prim_inet open @@ -3071,7 +3071,7 @@ change_group change_time master start_slave -'can not get remote filer ' +'cannot get remote filer ' start_relay relay 'Port controlling ~w terminated in file_server' diff --git a/lib/observer/test/observer_SUITE.erl b/lib/observer/test/observer_SUITE.erl index 40f5d44847..7d54bb9b3b 100644 --- a/lib/observer/test/observer_SUITE.erl +++ b/lib/observer/test/observer_SUITE.erl @@ -41,7 +41,8 @@ %% Default timetrap timeout (set in init_per_testcase) -define(default_timeout, ?t:minutes(2)). -suite() -> [{ct_hooks,[ts_install_cth]}]. +suite() -> [{timetrap, {minutes, 5}}, + {ct_hooks,[ts_install_cth]}]. all() -> [app_file, appup_file, {group, gui}]. @@ -301,7 +302,7 @@ table_win(Config) when is_list(Config) -> Notebook = setup_whitebox_testing(), Parent = get_top_level_parent(Notebook), TObj = observer_tv_table:start_link(Parent, [{node,node()}, {type,ets}, {table,#tab{name=foo, id=Table}}]), - %% Modal can not test edit.. + %% Modal cannot test edit.. %% TPid = wx_object:get_pid(TObj), %% TPid ! #wx{event=#wxList{type=command_list_item_activated, itemIndex=12}}, timer:sleep(3000), diff --git a/lib/odbc/c_src/Makefile.in b/lib/odbc/c_src/Makefile.in index 784e73c47e..294d832797 100644 --- a/lib/odbc/c_src/Makefile.in +++ b/lib/odbc/c_src/Makefile.in @@ -52,7 +52,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/odbc-$(VSN) # Target Specs # ---------------------------------------------------- EI_ROOT = $(ERL_TOP)/lib/erl_interface -EI_INCLUDE = -I$(EI_ROOT)/include +EI_INCLUDE = -I$(EI_ROOT)/include -I$(EI_ROOT)/include/$(TARGET) ifeq ($(findstring win32,$(TARGET)),win32) EI_LIB = -lerl_interface_md -lei_md ENTRY_OBJ=$(ERL_TOP)/erts/obj/$(TARGET)/port_entry.o diff --git a/lib/odbc/configure.in b/lib/odbc/configure.in index 2dec6e5abf..c5cf2786ca 100644 --- a/lib/odbc/configure.in +++ b/lib/odbc/configure.in @@ -21,12 +21,6 @@ dnl dnl define([AC_CACHE_LOAD], )dnl dnl define([AC_CACHE_SAVE], )dnl -if test "x$no_recursion" != "xyes" -a "x$OVERRIDE_CONFIG_CACHE" = "x"; then - # We do not want to use a common cache! - cache_file=/dev/null -fi - - dnl Process this file with autoconf to produce a configure script. AC_INIT(c_src/odbcserver.c) diff --git a/lib/odbc/doc/src/databases.xml b/lib/odbc/doc/src/databases.xml index a060f87e46..614c327a46 100644 --- a/lib/odbc/doc/src/databases.xml +++ b/lib/odbc/doc/src/databases.xml @@ -65,7 +65,7 @@ <p>Another obstacle is that some drivers do not support scrollable cursors which has the effect that the only way to traverse the result set is sequentially, with next, from the first row to the - last, and once you pass a row you can not go back. This means + last, and once you pass a row you cannot go back. This means that some functions in the interface will not work together with certain drivers. A similar problem is that not all drivers support "row count" for select queries, hence resulting in that diff --git a/lib/odbc/doc/src/error_handling.xml b/lib/odbc/doc/src/error_handling.xml index 4c80cff12f..8d794d2c99 100644 --- a/lib/odbc/doc/src/error_handling.xml +++ b/lib/odbc/doc/src/error_handling.xml @@ -138,7 +138,7 @@ should be run through the erlang port was not compiled for your platform.</item> <item>Errors discovered by the ODBC driver - If calls to the - ODBC-driver fails due to circumstances that can not be + ODBC-driver fails due to circumstances that cannot be controlled by the Erlang ODBC application programmer, an error string will be dug up from the driver. This string will be the <c>Reason</c> in the <c>{error, Reason} </c> diff --git a/lib/odbc/doc/src/getting_started.xml b/lib/odbc/doc/src/getting_started.xml index f2ff8b8993..b27754ff22 100644 --- a/lib/odbc/doc/src/getting_started.xml +++ b/lib/odbc/doc/src/getting_started.xml @@ -50,7 +50,7 @@ and paths to appropriate values. This may differ a lot between different os's, databases and ODBC drivers. This is a configuration problem related to the third party product - and hence we can not give you a standard solution in this guide.</item> + and hence we cannot give you a standard solution in this guide.</item> <item>The Erlang ODBC application consists of both <c>Erlang</c> and <c>C</c> code. The <c>C</c> code is delivered as a precompiled executable for windows, solaris and linux (SLES10) in the commercial diff --git a/lib/odbc/doc/src/notes_history.xml b/lib/odbc/doc/src/notes_history.xml index 22a92f67cd..d0c0a472e7 100644 --- a/lib/odbc/doc/src/notes_history.xml +++ b/lib/odbc/doc/src/notes_history.xml @@ -63,7 +63,7 @@ <list type="bulleted"> <item> <p>The erlang odbc process will now die normally if a - connection can not be established. No connection no + connection cannot be established. No connection no process it is expected. And as the client has already received the error message that would be the reason with which the erlang process would be stopped, the supervisor @@ -193,7 +193,7 @@ </item> <item> <p>The erlang odbc process will now die normally if a - connection can not be established. No connection no + connection cannot be established. No connection no process it is expected. And as the client has already received the error message that would be the reason with which the erlang process would be stopped, the supervisor diff --git a/lib/odbc/doc/src/odbc.xml b/lib/odbc/doc/src/odbc.xml index 4d941b3b36..1a3063e6ce 100644 --- a/lib/odbc/doc/src/odbc.xml +++ b/lib/odbc/doc/src/odbc.xml @@ -246,7 +246,7 @@ <p>Closes a connection to a database. This will also terminate all processes that may have been spawned when the connection was opened. This call will always succeed. - If the connection can not be disconnected gracefully it will + If the connection cannot be disconnected gracefully it will be brutally killed. However you may receive an error message as result if you try to disconnect a connection started by another process. diff --git a/lib/odbc/test/odbc_connect_SUITE.erl b/lib/odbc/test/odbc_connect_SUITE.erl index 261dfc6f20..94ca62b3fb 100644 --- a/lib/odbc/test/odbc_connect_SUITE.erl +++ b/lib/odbc/test/odbc_connect_SUITE.erl @@ -259,7 +259,7 @@ not_exist_db(_Config) -> %%------------------------------------------------------------------------- no_c_executable() -> - [{doc,"Test what happens if the port-program can not be found"}]. + [{doc,"Test what happens if the port-program cannot be found"}]. no_c_executable(_Config) -> process_flag(trap_exit, true), Dir = filename:nativename(filename:join(code:priv_dir(odbc), diff --git a/lib/os_mon/Makefile b/lib/os_mon/Makefile index 1eff8a785a..40ce94e0c7 100644 --- a/lib/os_mon/Makefile +++ b/lib/os_mon/Makefile @@ -23,11 +23,7 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk # # Macros # -ifeq ($(findstring win32,$(TARGET)),win32) -SUB_DIRECTORIES = src c_src mibs doc/src -else -SUB_DIRECTORIES = src c_src mibs doc/src -endif +SUB_DIRECTORIES = src c_src doc/src include vsn.mk VSN = $(OS_MON_VSN) diff --git a/lib/os_mon/doc/src/Makefile b/lib/os_mon/doc/src/Makefile index 354f8ed26b..8e9a4c333c 100644 --- a/lib/os_mon/doc/src/Makefile +++ b/lib/os_mon/doc/src/Makefile @@ -39,7 +39,6 @@ XML_APPLICATION_FILES = ref_man.xml XML_REF3_FILES = cpu_sup.xml \ disksup.xml \ memsup.xml \ - os_mon_mib.xml \ os_sup.xml \ nteventlog.xml diff --git a/lib/os_mon/doc/src/os_mon_app.xml b/lib/os_mon/doc/src/os_mon_app.xml index 99492a2021..c77a9d0411 100644 --- a/lib/os_mon/doc/src/os_mon_app.xml +++ b/lib/os_mon/doc/src/os_mon_app.xml @@ -88,33 +88,6 @@ </section> <section> - <title>SNMP MIBs</title> - <p>The following MIBs are defined in OS_Mon:</p> - <taglist> - <tag>OTP-OS-MON-MIB</tag> - <item> - <p>This MIB contains objects for instrumentation of disk, - memory and CPU usage of the nodes in the system.</p> - </item> - </taglist> - <p>The MIB is stored in the <c>mibs</c> directory. It is defined - in SNMPv2 SMI syntax. An SNMPv1 version of the MIB is delivered - in the <c>mibs/v1</c> directory.</p> - <p>The compiled MIB is located under <c>priv/mibs</c>, and - the generated <c>.hrl</c> file under the <c>include</c> directory. - To compile a MIB that IMPORTS the <c>OTP-OS-MON-MIB</c>, give - the option <c>{il, ["os_mon/priv/mibs"]}</c> to the MIB compiler.</p> - <p>If the MIB should be used in a system, it should be loaded into - an agent with a call to <c>os_mon_mib:load(Agent)</c>, where - <c>Agent</c> is the pid or registered name of an SNMP agent. Use - <c>os_mon_mib:unload(Agent)</c> to unload the MIB. - The implementation of this MIB uses Mnesia to store a cache with - data needed, which implicates that Mnesia must be up and running. - The MIB also use functions defined for the <c>OTP-MIB</c>, thus - that MIB must be loaded as well.</p> - </section> - - <section> <title>See Also</title> <p><seealso marker="cpu_sup">cpu_sup(3)</seealso>, <seealso marker="disksup">disksup(3)</seealso>, diff --git a/lib/os_mon/doc/src/os_mon_mib.xml b/lib/os_mon/doc/src/os_mon_mib.xml deleted file mode 100644 index f6d0b20094..0000000000 --- a/lib/os_mon/doc/src/os_mon_mib.xml +++ /dev/null @@ -1,74 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> - -<erlref> - <header> - <copyright> - <year>2004</year><year>2018</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>os_mon_mib</title> - <prepared>Ingela Andin</prepared> - <responsible></responsible> - <docno></docno> - <date></date> - <rev></rev> - </header> - <module since="">os_mon_mib</module> - <modulesummary>Loading and Unloading of OTP-OS-MON-MIB</modulesummary> - <description> - <p>Functions for loading and unloading the OTP-OS-MON-MIB into/from - an SNMP agent. The instrumentation of the OTP-OS-MON-MIB uses - Mnesia, hence Mnesia must be started prior to loading - the OTP-OS-MON-MIB.</p> - <warning> - <p>This module has been deprecated and will be removed in a furture release.</p> - </warning> - </description> - <funcs> - <func> - <name since="">load(Agent) -> ok | {error, Reason}</name> - <fsummary>Load the OTP-OS-MON-MIB</fsummary> - <type> - <v>Agent = pid() | atom()</v> - <v>Reason = term()</v> - </type> - <desc> - <p>Loads the OTP-OS-MON-MIB.</p> - </desc> - </func> - <func> - <name since="">unload(Agent) -> ok | {error, Reason}</name> - <fsummary>Unload the OTP-OS-MON-MIB</fsummary> - <type> - <v>Agent = pid() | atom() </v> - <v>Reason = term()</v> - </type> - <desc> - <p>Unloads the OTP-OS-MON-MIB.</p> - </desc> - </func> - </funcs> - - <section> - <title>See Also</title> - <p><seealso marker="os_mon_app">os_mon(6)</seealso>, - <seealso marker="snmp:snmp">snmp(3)</seealso></p> - </section> -</erlref> - diff --git a/lib/os_mon/doc/src/ref_man.xml b/lib/os_mon/doc/src/ref_man.xml index a8f847a8ba..57dd5c5f0b 100644 --- a/lib/os_mon/doc/src/ref_man.xml +++ b/lib/os_mon/doc/src/ref_man.xml @@ -36,7 +36,6 @@ <xi:include href="cpu_sup.xml"/> <xi:include href="disksup.xml"/> <xi:include href="memsup.xml"/> - <xi:include href="os_mon_mib.xml"/> <xi:include href="os_sup.xml"/> <xi:include href="nteventlog.xml"/> </application> diff --git a/lib/os_mon/mibs/Makefile b/lib/os_mon/mibs/Makefile deleted file mode 100644 index dbc105ee3d..0000000000 --- a/lib/os_mon/mibs/Makefile +++ /dev/null @@ -1,101 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 1997-2016. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# %CopyrightEnd% -# -include $(ERL_TOP)/make/target.mk -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -# ---------------------------------------------------- -# Application version -# ---------------------------------------------------- -include ../vsn.mk -VSN=$(OS_MON_VSN) - -# ---------------------------------------------------- -# Release directory specification -# ---------------------------------------------------- -RELSYSDIR = $(RELEASE_PATH)/lib/os_mon-$(VSN) - -# ---------------------------------------------------- -# Target Specs -# ---------------------------------------------------- - -MIB_FILES= OTP-OS-MON-MIB.mib -FUNCS_FILES = OTP-OS-MON-MIB.funcs - -BIN_TARGETS= $(MIB_FILES:%.mib=$(SNMP_BIN_TARGET_DIR)/%.bin) -HRL_TARGETS= $(MIB_FILES:%.mib=$(SNMP_HRL_TARGET_DIR)/%.hrl) -V1_MIB_FILES= $(MIB_FILES:%.mib=v1/%.mib.v1) - -TARGET_FILES= $(SNMP_BIN_TARGET_DIR)/OTP-REG.bin \ - $(SNMP_BIN_TARGET_DIR)/OTP-TC.bin \ - $(SNMP_BIN_TARGET_DIR)/OTP-MIB.bin \ - $(BIN_TARGETS) $(HRL_TARGETS) $(V1_MIB_FILES) - -# ---------------------------------------------------- -# FLAGS -# ---------------------------------------------------- -SNMP_FLAGS = -I $(SNMP_BIN_TARGET_DIR) - -# ---------------------------------------------------- -# Targets -# ---------------------------------------------------- - -debug opt: $(TARGET_FILES) - -clean: - rm -f $(TARGET_FILES) - rm -f core - -docs: - -OTP_MIBDIR = $(shell if test -d ../../otp_mibs; then echo otp_mibs; \ - else echo sasl; fi) - -$(SNMP_BIN_TARGET_DIR)/OTP-REG.bin: $(ERL_TOP)/lib/$(OTP_MIBDIR)/mibs/OTP-REG.mib - $(snmp_verbose)$(ERLC) -pa $(SNMP_TOOLKIT)/ebin -I $(SNMP_TOOLKIT)/priv/mibs $(SNMP_FLAGS) -o $(SNMP_BIN_TARGET_DIR) $< - -$(SNMP_BIN_TARGET_DIR)/OTP-TC.bin: $(ERL_TOP)/lib/$(OTP_MIBDIR)/mibs/OTP-TC.mib - $(snmp_verbose)$(ERLC) -pa $(SNMP_TOOLKIT)/ebin -I $(SNMP_TOOLKIT)/priv/mibs $(SNMP_FLAGS) -o $(SNMP_BIN_TARGET_DIR) $< - -$(SNMP_BIN_TARGET_DIR)/OTP-MIB.bin: $(ERL_TOP)/lib/$(OTP_MIBDIR)/mibs/OTP-MIB.mib - $(snmp_verbose)$(ERLC) -pa $(SNMP_TOOLKIT)/ebin -I $(SNMP_TOOLKIT)/priv/mibs $(SNMP_FLAGS) -o $(SNMP_BIN_TARGET_DIR) $< - -v1/%.mib.v1: %.mib - $(gen_verbose)$(ERL_TOP)/lib/snmp/bin/snmp-v2tov1 -o $@ $< - -$(SNMP_BIN_TARGET_DIR)/OTP-OS-MON-MIB.bin: \ - $(SNMP_BIN_TARGET_DIR)/OTP-REG.bin \ - $(SNMP_BIN_TARGET_DIR)/OTP-MIB.bin \ - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_spec: opt - $(INSTALL_DIR) "$(RELSYSDIR)/mibs" - $(INSTALL_DIR) "$(RELSYSDIR)/mibs/v1" - $(INSTALL_DATA) $(MIB_FILES) $(FUNCS_FILES) "$(RELSYSDIR)/mibs" - $(INSTALL_DATA) $(V1_MIB_FILES) "$(RELSYSDIR)/mibs/v1" - $(INSTALL_DIR) "$(RELSYSDIR)/include" - $(INSTALL_DATA) $(HRL_TARGETS) "$(RELSYSDIR)/include" - $(INSTALL_DIR) "$(RELSYSDIR)/priv/mibs" - $(INSTALL_DATA) $(BIN_TARGETS) "$(RELSYSDIR)/priv/mibs" - -release_docs_spec: diff --git a/lib/os_mon/mibs/OTP-OS-MON-MIB.funcs b/lib/os_mon/mibs/OTP-OS-MON-MIB.funcs deleted file mode 100644 index 7ed76517b9..0000000000 --- a/lib/os_mon/mibs/OTP-OS-MON-MIB.funcs +++ /dev/null @@ -1,5 +0,0 @@ -{loadMemorySystemWatermark, {os_mon_mib, mem_sys_mark, []}}. -{loadMemoryErlProcWatermark, {os_mon_mib, mem_proc_mark, []}}. -{loadTable, {os_mon_mib, load_table, []}}. -{diskAlmostFullThreshold, {os_mon_mib, disk_threshold, []}}. -{diskTable, {os_mon_mib, disk_table, []}}. diff --git a/lib/os_mon/mibs/OTP-OS-MON-MIB.mib b/lib/os_mon/mibs/OTP-OS-MON-MIB.mib deleted file mode 100644 index e027e96154..0000000000 --- a/lib/os_mon/mibs/OTP-OS-MON-MIB.mib +++ /dev/null @@ -1,423 +0,0 @@ --- --- %CopyrightBegin% --- --- Copyright Ericsson AB 1997-2016. All Rights Reserved. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- %CopyrightEnd% --- - -OTP-OS-MON-MIB DEFINITIONS ::= BEGIN - -IMPORTS - MODULE-IDENTITY, NOTIFICATION-TYPE, OBJECT-TYPE, - Counter32, Gauge32, Integer32, Unsigned32, Counter64 - FROM SNMPv2-SMI - TEXTUAL-CONVENTION, DisplayString - FROM SNMPv2-TC - MODULE-COMPLIANCE, NOTIFICATION-GROUP, OBJECT-GROUP - FROM SNMPv2-CONF - otpModules, otpApplications - FROM OTP-REG - erlNodeId - FROM OTP-MIB - ; - - -otpOsMonModule MODULE-IDENTITY - LAST-UPDATED "0305090900Z" - ORGANIZATION "Ericsson" - CONTACT-INFO - "Contact: Erlang Support see license agreement for Erlang/OTP." - - DESCRIPTION - "This MIB is part of the OTP MIB. It defines MIB objects - for the os_mon application in OTP." - - REVISION "0508260900Z" - DESCRIPTION - "Removed dependeny on EVA." - REVISION "0305090900Z" - DESCRIPTION - "Changed CONTACT-INFO as it was outdated, made it more generic - to avoid such changes in the future." - - REVISION "9807080900Z" - DESCRIPTION - "Changed MAX-ACCESS for diskDescr from not-accessible to - read-only." - - REVISION "9801270900Z" - DESCRIPTION - "Changed erroneous name of this module to otpOsMonModule." - - REVISION "9712010900Z" - DESCRIPTION - "Converted to v2 SMI and placed in the OTP tree." - - REVISION "9608191700Z" - DESCRIPTION - "The initial revision of MIB module OTP-OS-MON-MIB." - ::= { otpModules 4 } - -OTPCounterBasedGauge64 ::= TEXTUAL-CONVENTION - STATUS current - DESCRIPTION - "The CounterBasedGauge64 type represents a non-negative - integer, which may increase or decrease, but shall never - exceed a maximum value, nor fall below a minimum value. The - maximum value can not be greater than 2^64-1 - (18446744073709551615 decimal), and the minimum value can - - not be smaller than 0. The value of a CounterBasedGauge64 - has its maximum value whenever the information being modeled - is greater than or equal to its maximum value, and has its - minimum value whenever the information being modeled is - smaller than or equal to its minimum value. If the - information being modeled subsequently decreases below - (increases above) the maximum (minimum) value, the - CounterBasedGauge64 also decreases (increases). - - Note that this TC is not strictly supported in SMIv2, - because the 'always increasing' and 'counter wrap' semantics - associated with the Counter64 base type are not preserved. - It is possible that management applications which rely - solely upon the (Counter64) ASN.1 tag to determine object - semantics will mistakenly operate upon objects of this type - as they would for Counter64 objects. - - This textual convention represents a limited and short-term - solution, and may be deprecated as a long term solution is - defined and deployed to replace it." - SYNTAX Counter64 - -otpOsMonMIB OBJECT IDENTIFIER ::= { otpApplications 2 } -otpOsMonMIBConformance - OBJECT IDENTIFIER ::= { otpOsMonMIB 1 } -otpOsMonMIBObjects - OBJECT IDENTIFIER ::= { otpOsMonMIB 2 } -otpOsMonMIBAlarms - OBJECT IDENTIFIER ::= { otpOsMonMIB 4 } -otpOsMonMIBAlarmsV2 - OBJECT IDENTIFIER ::= { otpOsMonMIBAlarms 0 } - - --- Datatypes - --- Managed Objects - -load OBJECT IDENTIFIER ::= { otpOsMonMIBObjects 1 } -disk OBJECT IDENTIFIER ::= { otpOsMonMIBObjects 2 } - -loadMemorySystemWatermark OBJECT-TYPE - SYNTAX Integer32 (0..100) - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "Threshold in percent of the total available system - memory, which specifies how much memory can be allocated - by the system before an alarm is sent." - ::= { load 1 } - -loadMemoryErlProcWatermark OBJECT-TYPE - SYNTAX Integer32 (0..100) - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "Threshold in percent of the total available system - memory, which specifies how much memory can be allocated - by one Erlang process before an alarm is sent." - ::= { load 2 } - -loadTable OBJECT-TYPE - SYNTAX SEQUENCE OF LoadEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A table with load and memory information - for each node." - ::= { load 3 } - -loadEntry OBJECT-TYPE - SYNTAX LoadEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A conceptual row in the loadTable." - INDEX { loadErlNodeName } - ::= { loadTable 1 } - -LoadEntry ::= SEQUENCE { - loadErlNodeName DisplayString, - loadSystemTotalMemory Gauge32, - loadSystemUsedMemory Gauge32, - loadLargestErlProcess DisplayString, - loadLargestErlProcessUsedMemory Gauge32, - loadCpuLoad Integer32, - loadCpuLoad5 Integer32, - loadCpuLoad15 Integer32, - loadOsWordsize Unsigned32, - loadSystemTotalMemory64 OTPCounterBasedGauge64, - loadSystemUsedMemory64 OTPCounterBasedGauge64, - loadLargestErlProcessUsedMemory64 OTPCounterBasedGauge64 - } - -loadErlNodeName OBJECT-TYPE - SYNTAX DisplayString - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "The name of the erlang node, e.g. erlnode@host1." - ::= { loadEntry 1 } - -loadSystemTotalMemory OBJECT-TYPE - SYNTAX Gauge32 - UNITS "bytes" - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The amount of total memory in the system." - ::= { loadEntry 2 } - -loadSystemUsedMemory OBJECT-TYPE - SYNTAX Gauge32 - UNITS "bytes" - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The amount of used memory." - ::= { loadEntry 3 } - -loadLargestErlProcess OBJECT-TYPE - SYNTAX DisplayString - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The process identifier (Pid) of the largest Erlang - process." - ::= { loadEntry 4 } - -loadLargestErlProcessUsedMemory OBJECT-TYPE - SYNTAX Gauge32 - UNITS "bytes" - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The amount of memory used by the largest Erlang - process." - ::= { loadEntry 5 } - -loadCpuLoad OBJECT-TYPE - SYNTAX Integer32 (0..100) - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The average load the last minute in percent of the CPU - where the Erlang node runs." - ::= { loadEntry 6 } - -loadCpuLoad5 OBJECT-TYPE - SYNTAX Integer32 (0..100) - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The average load the last 5 minutes in percent of the CPU - where the Erlang node runs." - ::= { loadEntry 7} - -loadCpuLoad15 OBJECT-TYPE - SYNTAX Integer32 (0..100) - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The average load the last 15 minutes in percent of the CPU - where the Erlang node runs." - ::= { loadEntry 8} - -loadOsWordsize OBJECT-TYPE - SYNTAX Unsigned32 - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The wordsize of the operating operating system." - ::= { loadEntry 9 } - -loadSystemTotalMemory64 OBJECT-TYPE - SYNTAX OTPCounterBasedGauge64 - UNITS "bytes" - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The amount of total memory in the system for 64-bit operating system." - ::= { loadEntry 10 } - -loadSystemUsedMemory64 OBJECT-TYPE - SYNTAX OTPCounterBasedGauge64 - UNITS "bytes" - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The amount of used memory for 64-bit operating system." - ::= { loadEntry 11 } - -loadLargestErlProcessUsedMemory64 OBJECT-TYPE - SYNTAX OTPCounterBasedGauge64 - UNITS "bytes" - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The amount of memory used by the largest Erlang - process for 64-bit operating system.." - ::= { loadEntry 12 } - -diskAlmostFullThreshold OBJECT-TYPE - SYNTAX Integer32 (0..100) - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "Threshold in percent of the available disk space, - which specifies how much disk space can be used by - a disk or partition before an alarm is sent." - ::= { disk 1 } - -diskTable OBJECT-TYPE - SYNTAX SEQUENCE OF DiskEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A table with all local disks or partitions on each - node." - ::= { disk 2 } - -diskEntry OBJECT-TYPE - SYNTAX DiskEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A conceptual row in the diskTable." - INDEX { erlNodeId, diskId } - ::= { diskTable 1 } - -DiskEntry ::= SEQUENCE { - diskId Integer32, - diskDescr DisplayString, - diskKBytes Gauge32, - diskCapacity Integer32 - } - -diskId OBJECT-TYPE - SYNTAX Integer32 - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "An integer that uniquely identifies the disk - or partition." - ::= { diskEntry 1 } - -diskDescr OBJECT-TYPE - SYNTAX DisplayString - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "A string that identifies the disk or partition." - ::= { diskEntry 2 } - -diskKBytes OBJECT-TYPE - SYNTAX Gauge32 - UNITS "kbytes" - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The amount of total disk/partition space. " - ::= { diskEntry 3 } - -diskCapacity OBJECT-TYPE - SYNTAX Integer32 (0..100) - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "How much of the disk's/partition's total capacity has - been used, in percent." - ::= { diskEntry 4 } - - --- conformance information - -otpOsMonMIBCompliances - OBJECT IDENTIFIER ::= { otpOsMonMIBConformance 1 } -otpOsMonMIBGroups - OBJECT IDENTIFIER ::= { otpOsMonMIBConformance 2 } - - --- compliance statements - -otpOsMonBasicCompliance MODULE-COMPLIANCE - STATUS current - DESCRIPTION - "The compliance statement for SNMPv2 entities which - implement the OTP-OS-MON-MIB." - MODULE -- this module - GROUP loadGroup - DESCRIPTION - "This group is mandatory for systems implementing the - load supervison functionality." - GROUP loadAlarmsGroup - DESCRIPTION - "This group is optional for systems implementing the - load supervison functionality." - GROUP diskGroup - DESCRIPTION - "This group is mandatory for system implementing the - disk supervison functionality." - GROUP diskAlarmsGroup - DESCRIPTION - "This group is optional for systems implementing the - disk supervison functionality." - ::= { otpOsMonMIBCompliances 1 } - - --- units of conformance - -loadGroup OBJECT-GROUP - OBJECTS { loadMemorySystemWatermark, - loadMemoryErlProcWatermark, - loadSystemTotalMemory, - loadSystemUsedMemory, - loadLargestErlProcess, - loadLargestErlProcessUsedMemory, - loadCpuLoad, - loadCpuLoad5, - loadCpuLoad15, - loadOsWordsize, - loadSystemTotalMemory64, - loadSystemUsedMemory64, - loadLargestErlProcessUsedMemory64} - STATUS current - DESCRIPTION - "A collection of objects providing basic instrumentation - of the load of the OTP system." - ::= { otpOsMonMIBGroups 1 } - -diskGroup OBJECT-GROUP - OBJECTS { diskAlmostFullThreshold, - diskDescr, - diskKBytes, - diskCapacity } - STATUS current - DESCRIPTION - "A collection of objects providing basic instrumentation - of the disks in the OTP system." - ::= { otpOsMonMIBGroups 3 } - -END diff --git a/lib/os_mon/mibs/v1/.gitignore b/lib/os_mon/mibs/v1/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/os_mon/mibs/v1/.gitignore +++ /dev/null diff --git a/lib/os_mon/src/Makefile b/lib/os_mon/src/Makefile index fc2eb22393..923a31f290 100644 --- a/lib/os_mon/src/Makefile +++ b/lib/os_mon/src/Makefile @@ -34,8 +34,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/os_mon-$(VSN) # ---------------------------------------------------- # Target Specs # ---------------------------------------------------- -MODULES= disksup memsup cpu_sup os_mon os_mon_mib os_sup os_mon_sysinfo \ - nteventlog +MODULES= disksup memsup cpu_sup os_mon os_sup os_mon_sysinfo nteventlog INCLUDE=../include CSRC=../c_src diff --git a/lib/os_mon/src/cpu_sup.erl b/lib/os_mon/src/cpu_sup.erl index ba2d89313e..d28f229b3e 100644 --- a/lib/os_mon/src/cpu_sup.erl +++ b/lib/os_mon/src/cpu_sup.erl @@ -68,7 +68,7 @@ -type util_cpus() :: 'all' | integer() | [integer()]. -type util_state() :: 'user' | 'nice_user' | 'kernel' | 'wait' | 'idle'. --type util_value() :: [{util_state(), float()}] | float(). +-type util_value() :: [{util_state(), number()}] | number(). -type util_desc() :: {util_cpus(), util_value(), util_value(), []}. %%---------------------------------------------------------------------- @@ -122,7 +122,7 @@ util(Args) when is_list (Args) -> util(_) -> erlang:error(badarg). --spec util() -> float() | {'error', any()}. +-spec util() -> number() | {'error', any()}. util() -> case util([]) of diff --git a/lib/os_mon/src/memsup.erl b/lib/os_mon/src/memsup.erl index 9d6447430d..b69d657aa7 100644 --- a/lib/os_mon/src/memsup.erl +++ b/lib/os_mon/src/memsup.erl @@ -701,6 +701,7 @@ get_os_wordsize_with_uname() -> "sparc64" -> 64; "amd64" -> 64; "ppc64" -> 64; + "ppc64le" -> 64; "s390x" -> 64; _ -> 32 end. diff --git a/lib/os_mon/src/os_mon.app.src b/lib/os_mon/src/os_mon.app.src index 8be94f65d4..6c9b0d7576 100644 --- a/lib/os_mon/src/os_mon.app.src +++ b/lib/os_mon/src/os_mon.app.src @@ -21,7 +21,7 @@ {application, os_mon, [{description, "CPO CXC 138 46"}, {vsn, "%VSN%"}, - {modules, [os_mon, os_mon_mib, os_sup, + {modules, [os_mon, os_sup, disksup, memsup, cpu_sup, os_mon_sysinfo, nteventlog]}, {registered, [os_mon_sup, os_mon_sysinfo, disksup, memsup, cpu_sup, os_sup_server]}, @@ -31,6 +31,4 @@ {start_memsup, true}, {start_os_sup, false}]}, {mod, {os_mon, []}}, - {runtime_dependencies, ["stdlib-2.0","snmp-4.25.1","sasl-2.4", - "otp_mibs-1.0.9","mnesia-4.12","kernel-3.0", - "erts-6.0"]}]}. + {runtime_dependencies, ["stdlib-2.0","sasl-2.4","kernel-3.0","erts-6.0"]}]}. diff --git a/lib/os_mon/src/os_mon_mib.erl b/lib/os_mon/src/os_mon_mib.erl deleted file mode 100644 index 9b5d2fbba6..0000000000 --- a/lib/os_mon/src/os_mon_mib.erl +++ /dev/null @@ -1,251 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% --module(os_mon_mib). -%%%----------------------------------------------------------------- -%%% Description: This module implements the OS-MON-MIB. -%%% The tables are implemented as shadow tables with the module -%%% snmp_shadow_table. Here the update functions are implemented. -%%%----------------------------------------------------------------- - --include("../../otp_mibs/include/OTP-MIB.hrl"). - -%% API --export([load/1, unload/1]). - -%% Deprecated API --export([init/1, stop/1]). - --deprecated([{init,1,eventually}, - {stop,1,eventually}]). - -%% SNMP instrumentation --export([load_table/1, load_table/3, disk_table/1, disk_table/3, - mem_sys_mark/1, mem_proc_mark/1, disk_threshold/1]). - -%% SNMP shadow functions --export([update_load_table/0, update_disk_table/0]). - -%% Exported for internal use via rpc --export([get_load/1, get_disks/1]). - -%% Shadow tables --record(loadTable, { - loadErlNodeName, - loadSystemTotalMemory, - loadSystemUsedMemory, - loadLargestErlProcess, - loadLargestErlProcessUsedMemory, - loadCpuLoad, - loadCpuLoad5, - loadCpuLoad15, - loadOsWordsize, - loadSystemTotalMemory64, - loadSystemUsedMemory64, - loadLargestErlProcessUsedMemory64}). - --record(diskTable, - {key, diskDescr, diskKBytes, diskCapacity}). - -%% Shadow argument macros --define(loadShadowArgs, - {loadTable, string, record_info(fields, loadTable), 5000, - fun os_mon_mib:update_load_table/0}). - --define(diskShadowArgs, - {diskTable, {integer, integer}, record_info(fields, diskTable), 5000, - fun os_mon_mib:update_disk_table/0}). - -%% Misc --record(diskAlloc, {diskDescr, diskId}). - -%%%========================================================================= -%%% API -%%%========================================================================= - -%%------------------------------------------------------------------------- -%% load(Agent) -> ok | {error, Reason} -%% Agent - pid() | atom() -%% Reason - term() -%% Description: Loads the OTP-OS-MON-MIB -%%------------------------------------------------------------------------- -load(Agent) -> - MibDir = filename:join(code:priv_dir(os_mon), "mibs"), - snmpa:load_mibs(Agent, [filename:join(MibDir, "OTP-OS-MON-MIB")]). - -%%------------------------------------------------------------------------- -%% unload(Agent) -> ok | {error, Reason} -%% Agent - pid() | atom() -%% Reason - term() -%% Description: Unloads the OTP-OS-MON-MIB -%%------------------------------------------------------------------------- -unload(Agent) -> - snmpa:unload_mibs(Agent, ["OTP-OS-MON-MIB"]). - -%% To be backwards compatible -init(Agent) -> - load(Agent). -stop(Agent) -> - unload(Agent). - -%%%========================================================================= -%%% SNMP instrumentation -%%%========================================================================= -load_table(Op) -> - snmp_shadow_table:table_func(Op, ?loadShadowArgs). -load_table(Op, RowIndex, Cols) -> - snmp_shadow_table:table_func(Op, RowIndex, Cols, ?loadShadowArgs). - -disk_table(new) -> - Tab = diskAlloc, - Storage = ram_copies, - case lists:member(Tab, mnesia:system_info(tables)) of - true -> - case mnesia:table_info(Tab, storage_type) of - unknown -> - {atomic, ok}=mnesia:add_table_copy(Tab, node(), Storage); - Storage -> - catch delete_all(Tab) - end; - false -> - Nodes = [node()], - Props = [{type, set}, - {attributes, record_info(fields, diskAlloc)}, - {local_content, true}, - {Storage, Nodes}], - {atomic, ok} = mnesia:create_table(Tab, Props) - - end, - Rec = #diskAlloc{diskDescr = next_index, diskId = 1}, - ok = mnesia:dirty_write(Rec), - snmp_shadow_table:table_func(new, ?diskShadowArgs). - -disk_table(Op, RowIndex, Cols) -> - snmp_shadow_table:table_func(Op, RowIndex, Cols, ?diskShadowArgs). - -mem_sys_mark(get) -> - {value, memsup:get_sysmem_high_watermark()}; -mem_sys_mark(_) -> - ok. - -mem_proc_mark(get) -> - {value, memsup:get_procmem_high_watermark()}; -mem_proc_mark(_) -> - ok. - -disk_threshold(get) -> - {value, disksup:get_almost_full_threshold()}; -disk_threshold(_) -> - ok. - -%%%========================================================================= -%%% SNMP shadow functions -%%%========================================================================= -update_load_table() -> - delete_all(loadTable), - lists:foreach( - fun(Node) -> - case rpc:call(Node, os_mon_mib, get_load, [Node]) of - Load when is_record(Load,loadTable) -> - ok = mnesia:dirty_write(Load); - _Else -> - ok - end - end, [node() | nodes()]). - - -update_disk_table() -> - delete_all(diskTable), - node_update_disk_table( - otp_mib:erl_node_table(get_next, [], [?erlNodeName,?erlNodeOutBytes])). - -%%%======================================================================== -%%% Exported for internal use via rpc -%%%======================================================================== -get_load(Node) -> - {Total, Allocated, PidString, PidAllocated} = case memsup:get_memory_data() of - {MemTot, MemAlloc, undefined} -> {MemTot, MemAlloc, "undefined", 0}; - {MemTot, MemAlloc, {Pid, PidMem}} -> {MemTot, MemAlloc, pid_to_str(Pid), PidMem} - end, - OsWordsize = case memsup:get_os_wordsize() of - WS when is_integer(WS) -> WS; - _ -> 0 - end, - #loadTable{ - loadErlNodeName = atom_to_list(Node), - loadSystemTotalMemory = mask_int32(Total), - loadSystemUsedMemory = mask_int32(Allocated), - loadLargestErlProcess = PidString, - loadLargestErlProcessUsedMemory = mask_int32(PidAllocated), - loadCpuLoad = get_cpu_load(avg1), - loadCpuLoad5 = get_cpu_load(avg5), - loadCpuLoad15 = get_cpu_load(avg15), - loadOsWordsize = OsWordsize, - loadSystemTotalMemory64 = Total, - loadSystemUsedMemory64 = Allocated, - loadLargestErlProcessUsedMemory64 = PidAllocated - }. - -mask_int32(Value) -> Value band ((1 bsl 32) - 1). - -get_disks(NodeId) -> - element(1, - lists:mapfoldl( - fun({Descr, KByte, Capacity}, DiskId) -> - {#diskTable{key = {NodeId, DiskId}, - diskDescr = Descr, - diskKBytes = KByte, - diskCapacity = Capacity}, - DiskId + 1} - end, 1, disksup:get_disk_data())). - - -%%%======================================================================== -%%% Internal functions -%%%======================================================================== -node_update_disk_table([_, endOfTable]) -> - ok; - -node_update_disk_table([{[?erlNodeName | IndexList], NodeStr}, _]) -> - Disks = rpc:call(list_to_atom(NodeStr), os_mon_mib, get_disks, - IndexList), - lists:foreach(fun(Disk) -> - mnesia:dirty_write(Disk) - end, Disks), - node_update_disk_table(otp_mib:erl_node_table(get_next, - IndexList, - [?erlNodeName, - ?erlNodeOutBytes])). - -get_cpu_load(X) when X == avg1; X == avg5; X == avg15 -> - case erlang:round(apply(cpu_sup, X, [])/2.56) of - Large when Large > 100 -> - 100; - Load -> - Load - end. - -delete_all(Name) -> delete_all(mnesia:dirty_first(Name), Name). -delete_all('$end_of_table', _Name) -> done; -delete_all(Key, Name) -> - Next = mnesia:dirty_next(Name, Key), - ok = mnesia:dirty_delete({Name, Key}), - delete_all(Next, Name). - -pid_to_str(Pid) -> lists:flatten(io_lib:format("~w", [Pid])). diff --git a/lib/os_mon/test/Makefile b/lib/os_mon/test/Makefile index 6ac67e6bae..03c73b95ec 100644 --- a/lib/os_mon/test/Makefile +++ b/lib/os_mon/test/Makefile @@ -30,7 +30,6 @@ MODULES= \ disksup_SUITE \ memsup_SUITE \ cpu_sup_SUITE \ - os_mon_mib_SUITE \ os_sup_SUITE \ os_mon_conf @@ -87,7 +86,6 @@ release_tests_spec: make_emakefile $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) os_mon.spec os_mon.cover os_mon_smoke.spec \ $(EMAKEFILE) $(SOURCE) "$(RELSYSDIR)" - $(INSTALL_DATA) os_mon_mib_SUITE.cfg "$(RELSYSDIR)" ## tar chf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) diff --git a/lib/os_mon/test/os_mon.spec b/lib/os_mon/test/os_mon.spec index 4b4286b313..d292b258f3 100644 --- a/lib/os_mon/test/os_mon.spec +++ b/lib/os_mon/test/os_mon.spec @@ -1,2 +1 @@ {suites,"../os_mon_test",all}. -{config,"os_mon_mib_SUITE.cfg"}.
\ No newline at end of file diff --git a/lib/os_mon/test/os_mon_mib_SUITE.cfg b/lib/os_mon/test/os_mon_mib_SUITE.cfg deleted file mode 100644 index a33c23530b..0000000000 --- a/lib/os_mon/test/os_mon_mib_SUITE.cfg +++ /dev/null @@ -1,8 +0,0 @@ -%% -*- erlang -*- -{snmp, [{start_agent,true}, - {users,[{os_mon_mib_test,[snmpm_user_default,[]]}]}, - {managed_agents,[{os_mon_mib_test, - [os_mon_mib_test, {127,0,0,1}, 4000, []]}]}, - {agent_sysname,"Test os_mon_mibs"}, - {mgr_port,5001} - ]}. diff --git a/lib/os_mon/test/os_mon_mib_SUITE.erl b/lib/os_mon/test/os_mon_mib_SUITE.erl deleted file mode 100644 index f40d5f442c..0000000000 --- a/lib/os_mon/test/os_mon_mib_SUITE.erl +++ /dev/null @@ -1,578 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% --module(os_mon_mib_SUITE). - -%%----------------------------------------------------------------- -%% This suite can no longer be executed standalone, i.e. it must be -%% executed with common test. The reason is that ct_snmp is used -%% instead of the snmp application directly. The suite requires a -%% config file, os_mon_mib_SUITE.cfg, found in the same directory as -%% the suite. -%% -%% Execute with: -%% > ct_run -suite os_mon_mib_SUITE -config os_mon_mib_SUITE.cfg -%%----------------------------------------------------------------- - --include_lib("common_test/include/ct.hrl"). --include_lib("os_mon/include/OTP-OS-MON-MIB.hrl"). --include_lib("snmp/include/snmp_types.hrl"). - -% Test server specific exports --export([all/0, suite/0, groups/0, - init_per_suite/1, end_per_suite/1]). - - -% Test cases must be exported. --export([update_load_table/1]). - --export([get_mem_sys_mark/1, get_mem_proc_mark/1, get_disk_threshold/1, - get_load_table/1, get_disk_table/1, - real_snmp_request/1, load_unload/1]). - --export([sys_tot_mem/1, sys_used_mem/1, large_erl_process/1, - large_erl_process_mem/1, cpu_load/1, cpu_load5/1, cpu_load15/1, - os_wordsize/1, sys_tot_mem64/1, sys_used_mem64/1, - large_erl_process_mem64/1, disk_descr/1, disk_kbytes/1, - disk_capacity/1]). - --export([otp_6351/1, otp_7441/1]). - --define(TRAP_UDP, 5000). --define(AGENT_UDP, 4000). --define(CONF_FILE_VER, [v2]). --define(SYS_NAME, "Test os_mon_mibs"). --define(MAX_MSG_SIZE, 484). --define(ENGINE_ID, "mgrEngine"). --define(MGR_PORT, 5001). - -%%--------------------------------------------------------------------- - -suite() -> - [{ct_hooks,[ts_install_cth]}, - {timetrap,{minutes,6}}, - {require, snmp_mgr_agent, snmp}]. - -all() -> - [load_unload, get_mem_sys_mark, get_mem_proc_mark, - get_disk_threshold, get_load_table, - {group, get_next_load_table}, get_disk_table, - {group, get_next_disk_table}, real_snmp_request, - update_load_table, {group, tickets}]. - -groups() -> - [{tickets, [], [otp_6351, otp_7441]}, - {get_next_load_table, [], - [sys_tot_mem, sys_used_mem, large_erl_process, - large_erl_process_mem, cpu_load, cpu_load5, cpu_load15, - os_wordsize, sys_tot_mem64, sys_used_mem64, - large_erl_process_mem64]}, - {get_next_disk_table, [], - [disk_descr, disk_kbytes, disk_capacity]}]. - - -%%--------------------------------------------------------------------- -%%-------------------------------------------------------------------- -%% Function: init_per_suite(Config) -> Config -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Description: Initiation before the whole suite -%% -%% Note: This function is free to add any key/value pairs to the Config -%% variable, but should NOT alter/remove any existing entries. -%%-------------------------------------------------------------------- -init_per_suite(Config) -> - application:start(sasl), - application:start(mnesia), - application:start(os_mon), - - ok = ct_snmp:start(Config,snmp_mgr_agent), - - %% Load the mibs that should be tested - otp_mib:load(snmp_master_agent), - os_mon_mib:load(snmp_master_agent), - - Config. -%%-------------------------------------------------------------------- -%% Function: end_per_suite(Config) -> _ -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Description: Cleanup after the whole suite -%%-------------------------------------------------------------------- -end_per_suite(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - ConfDir = filename:join(PrivDir,"conf"), - DbDir = filename:join(PrivDir,"db"), - MgrDir = filename:join(PrivDir, "mgr"), - - %% Uload mibs - snmpa:unload_mibs(snmp_master_agent,["OTP-OS-MON-MIB"]), - otp_mib:unload(snmp_master_agent), - - %% Clean up - application:stop(snmp), - application:stop(mnesia), - application:stop(os_mon), - - del_dir(ConfDir), - del_dir(DbDir), - (catch del_dir(MgrDir)), - ok. - -%%--------------------------------------------------------------------- -%% Test cases -%%--------------------------------------------------------------------- - -%% Test to unload and the reload the OTP.mib -load_unload(Config) when is_list(Config) -> - os_mon_mib:unload(snmp_master_agent), - os_mon_mib:load(snmp_master_agent), - ok. -%%--------------------------------------------------------------------- - -%% check os_mon_mib:update_load_table error handling -update_load_table(Config) when is_list(Config) -> - Node = start_node(), - ok = rpc:call(Node,application,start,[sasl]), - ok = rpc:call(Node,application,start,[os_mon]), - ok = os_mon_mib:update_load_table(), - rpc:call(Node,application,stop,[os_mon]), - ok = os_mon_mib:update_load_table(), - stop_node(Node), - ok. - -%% like update_load_table, when memsup_system_only==true -otp_6351(Config) when is_list(Config) -> - Node = start_node(), - ok = rpc:call(Node,application,start,[sasl]), - ok = rpc:call(Node,application,load,[os_mon]), - ok = rpc:call(Node,application,set_env, - [os_mon,memsup_system_only,true]), - ok = rpc:call(Node,application,start,[os_mon]), - Res = rpc:call(Node,os_mon_mib,get_load,[Node]), - if - is_tuple(Res), element(1, Res)==loadTable -> - ok; - true -> - ct:fail(Res) - end, - rpc:call(Node,application,stop,[os_mon]), - stop_node(Node), - ok. - - -%%--------------------------------------------------------------------- -%% Simulates a get call to test the instrumentation function -%% for the loadMemorySystemWatermark variable. -get_mem_sys_mark(Config) when is_list(Config) -> - case os_mon_mib:mem_sys_mark(get) of - {value, SysMark} when is_integer(SysMark) -> - ok; - _ -> - ct:fail(sys_mark_value_not_integer) - end. -%%--------------------------------------------------------------------- -%% Simulates a get call to test the instrumentation function -%% for the loadMemoryErlProcWatermark variable. -get_mem_proc_mark(Config) when is_list(Config) -> - case os_mon_mib:mem_proc_mark(get) of - {value, ProcMark} when is_integer(ProcMark) -> - ok; - _ -> - ct:fail(proc_mark_value_not_integer) - end. -%%--------------------------------------------------------------------- -%% Simulates a get call to test the instrumentation function -%% for the diskAlmostFullThreshold variable. -get_disk_threshold(Config) when is_list(Config) -> - case os_mon_mib:disk_threshold(get) of - {value, ProcMark} when is_integer(ProcMark) -> - ok; - _ -> - ct:fail(disk_threshold_value_not_integer) - end. -%%--------------------------------------------------------------------- - -%%% Note that when we have a string key, as in loadTable, the -%%% instrumentation will deal with the [length(String), String]. We -%%% have to know about this, when short cutting SNMP and calling -%%% instrumentation functions directly as done in most test cases in -%%% this test suite - -%% Simulates get calls to test the instrumentation function -%% for the loadTable -get_load_table(Config) when is_list(Config) -> - - NodeStr = atom_to_list(node()), - NodeLen = length(NodeStr), - - {_, _, {Pid, _}} = memsup:get_memory_data(), - PidStr = lists:flatten(io_lib:format("~w", [Pid])), - [{value, NodeStr},{value, PidStr}] = - os_mon_mib:load_table(get, [NodeLen | NodeStr], - [?loadErlNodeName, ?loadLargestErlProcess]), - - Values = os_mon_mib:load_table(get, [NodeLen | NodeStr] , - [?loadSystemTotalMemory, - ?loadSystemUsedMemory, - ?loadLargestErlProcessUsedMemory, - ?loadCpuLoad, - ?loadCpuLoad5, - ?loadCpuLoad15, - ?loadOsWordsize, - ?loadSystemTotalMemory64, - ?loadSystemUsedMemory64, - ?loadLargestErlProcessUsedMemory64]), - - IsInt = fun({value, Val}) when is_integer(Val) -> - true; - (_) -> - false - end, - - NewValues = lists:filter(IsInt, Values), - - case length(NewValues) of - 10 -> - ok; - _ -> - ct:fail(value_not_integer) - end, - - [{noValue,noSuchInstance}, {noValue,noSuchInstance}, - {noValue,noSuchInstance}, {noValue,noSuchInstance}, - {noValue,noSuchInstance}, {noValue,noSuchInstance}, - {noValue,noSuchInstance}, {noValue,noSuchInstance}, - {noValue,noSuchInstance}, {noValue,noSuchInstance}, - {noValue,noSuchInstance}, {noValue,noSuchInstance}] = - os_mon_mib:load_table(get, [3, 102, 111, 111], - [?loadErlNodeName, - ?loadSystemTotalMemory, - ?loadSystemUsedMemory, - ?loadLargestErlProcess, - ?loadLargestErlProcessUsedMemory, - ?loadCpuLoad, - ?loadCpuLoad5, - ?loadCpuLoad15, - ?loadOsWordsize, - ?loadSystemTotalMemory64, - ?loadSystemUsedMemory64, - ?loadLargestErlProcessUsedMemory64]), - - ok. -%%--------------------------------------------------------------------- - -sys_tot_mem(Config) when is_list(Config) -> - [{[?loadSystemTotalMemory, Len | NodeStr], Mem}] = - os_mon_mib:load_table(get_next, [], [?loadSystemTotalMemory]), - Len = length(NodeStr), - true = lists:member(list_to_atom(NodeStr), [node() | nodes()]), - - case Mem of - Mem when is_integer(Mem) -> - ok; - _ -> - ct:fail(sys_tot_mem_value_not_integer) - end. - -sys_used_mem(Config) when is_list(Config) -> - [{[?loadSystemUsedMemory, Len | NodeStr], Mem}] = - os_mon_mib:load_table(get_next,[], [?loadSystemUsedMemory]), - Len = length(NodeStr), - true = lists:member(list_to_atom(NodeStr), [node() | nodes()]), - - case Mem of - Mem when is_integer(Mem) -> - ok; - _ -> - ct:fail(sys_used_mem_value_not_integer) - end. - -large_erl_process(Config) when is_list(Config) -> - {_, _, {Pid, _}} = memsup:get_memory_data(), - PidStr = lists:flatten(io_lib:format("~w", [Pid])), - [{[?loadLargestErlProcess, Len | NodeStr], PidStr}] = - os_mon_mib:load_table(get_next,[], [?loadLargestErlProcess]), - Len = length(NodeStr), - true = lists:member(list_to_atom(NodeStr), [node() | nodes()]), - ok. - -large_erl_process_mem(Config) when is_list(Config) -> - - [{[?loadLargestErlProcessUsedMemory, Len | NodeStr], Mem}] = - os_mon_mib:load_table(get_next,[], - [?loadLargestErlProcessUsedMemory]), - Len = length(NodeStr), - true = lists:member(list_to_atom(NodeStr), [node() | nodes()]), - - case Mem of - Mem when is_integer(Mem) -> - ok; - _ -> - ct:fail(erl_pid_mem_value_not_integer) - end. - -cpu_load(Config) when is_list(Config) -> - [{[?loadCpuLoad, Len | NodeStr], Load}] = - os_mon_mib:load_table(get_next,[], [?loadCpuLoad]), - Len = length(NodeStr), - true = lists:member(list_to_atom(NodeStr), [node() | nodes()]), - - case Load of - Load when is_integer(Load) -> - ok; - _ -> - ct:fail(cpu_load_value_not_integer) - end. - -cpu_load5(Config) when is_list(Config) -> - [{[?loadCpuLoad5, Len | NodeStr], Load}] = - os_mon_mib:load_table(get_next,[], [?loadCpuLoad5]), - Len = length(NodeStr), - true = lists:member(list_to_atom(NodeStr), [node() | nodes()]), - - case Load of - Load when is_integer(Load) -> - ok; - _ -> - ct:fail(cpu_load5_value_not_integer) - end. - -cpu_load15(Config) when is_list(Config) -> - [{[?loadCpuLoad15, Len | NodeStr], Load}] = - os_mon_mib:load_table(get_next,[], [?loadCpuLoad15]), - Len = length(NodeStr), - true = lists:member(list_to_atom(NodeStr), [node() | nodes()]), - - case Load of - Load when is_integer(Load) -> - ok; - _ -> - ct:fail(cpu_load15_value_not_integer) - end. - -os_wordsize(Config) when is_list(Config) -> - [{[?loadOsWordsize, Len | NodeStr], Wordsize}] = - os_mon_mib:load_table(get_next,[], [?loadOsWordsize]), - Len = length(NodeStr), - true = lists:member(list_to_atom(NodeStr), [node() | nodes()]), - - case Wordsize of - Wordsize when is_integer(Wordsize) -> - ok; - _ -> - ct:fail(os_wordsize_value_not_integer) - end. - -sys_tot_mem64(Config) when is_list(Config) -> - [{[?loadSystemTotalMemory64, Len | NodeStr], Mem}] = - os_mon_mib:load_table(get_next, [], [?loadSystemTotalMemory64]), - Len = length(NodeStr), - true = lists:member(list_to_atom(NodeStr), [node() | nodes()]), - - case Mem of - Mem when is_integer(Mem) -> - ok; - _ -> - ct:fail(sys_tot_mem_value_not_integer) - end. - -sys_used_mem64(Config) when is_list(Config) -> - [{[?loadSystemUsedMemory64, Len | NodeStr], Mem}] = - os_mon_mib:load_table(get_next,[], [?loadSystemUsedMemory64]), - Len = length(NodeStr), - true = lists:member(list_to_atom(NodeStr), [node() | nodes()]), - - case Mem of - Mem when is_integer(Mem) -> - ok; - _ -> - ct:fail(sys_used_mem_value_not_integer) - end. - -large_erl_process_mem64(Config) when is_list(Config) -> - - [{[?loadLargestErlProcessUsedMemory64, Len | NodeStr], Mem}] = - os_mon_mib:load_table(get_next,[], - [?loadLargestErlProcessUsedMemory64]), - Len = length(NodeStr), - true = lists:member(list_to_atom(NodeStr), [node() | nodes()]), - - case Mem of - Mem when is_integer(Mem) -> - ok; - _ -> - ct:fail(erl_pid_mem_value_not_integer) - end. -%%--------------------------------------------------------------------- -%% Simulates get calls to test the instrumentation function -%% for the diskTable. -get_disk_table(Config) when is_list(Config) -> - - DiskData = disksup:get_disk_data(), - DiskDataLen = length(DiskData), - - if - DiskDataLen > 0 -> - [{value, Value}] = - os_mon_mib:disk_table(get, [1,1], [?diskDescr]), - - case is_list(Value) of - true -> - ok; - false -> - ct:fail(value_not_a_string) - end, - - Values = os_mon_mib:disk_table(get, [1,1], - [?diskId, - ?diskKBytes, - ?diskCapacity]), - - IsInt = fun({value, Val}) when is_integer(Val) -> - true; - (_) -> - false - end, - - NewValues = lists:filter(IsInt, Values), - - case length(NewValues) of - 3 -> - ok; - _ -> - ct:fail(value_not_integer) - end - end, - - [{noValue,noSuchInstance}, {noValue,noSuchInstance}, - {noValue,noSuchInstance}, {noValue,noSuchInstance}] = - os_mon_mib:disk_table(get, [1, DiskDataLen + 1], [?diskId, - ?diskDescr, - ?diskKBytes, - ?diskCapacity]), - - ok. - -%%--------------------------------------------------------------------- - -disk_descr(Config) when is_list(Config) -> - [{[?diskDescr, 1,1], Descr}] = - os_mon_mib:disk_table(get_next, [], [?diskDescr]), - - case Descr of - Descr when is_list(Descr) -> - ok; - _ -> - ct:fail(disk_descr_value_not_a_string) - end. - -disk_kbytes(Config) when is_list(Config) -> - [{[?diskKBytes, 1,1], Kbytes}] = - os_mon_mib:disk_table(get_next,[], [?diskKBytes]), - - case Kbytes of - Kbytes when is_integer(Kbytes) -> - ok; - _ -> - ct:fail(disk_kbytes_value_not_integer) - end. - - -disk_capacity(Config) when is_list(Config) -> - [{[?diskCapacity, 1,1], Capacity}] = - os_mon_mib:disk_table(get_next,[], [?diskCapacity]), - - case Capacity of - Capacity when is_integer(Capacity) -> - ok; - _ -> - ct:fail(disk_capacity_value_not_integer) - end. - -%%--------------------------------------------------------------------- -%% Starts an snmp manager and sends a real snmp-request. i.e. -%% sends a udp message on the correct format. -real_snmp_request(Config) when is_list(Config) -> - NodStr = atom_to_list(node()), - Len = length(NodStr), - {_, _, {Pid, _}} = memsup:get_memory_data(), - PidStr = lists:flatten(io_lib:format("~w", [Pid])), - io:format("FOO: ~p~n", [PidStr]), - ok = snmp_get([?loadEntry ++ - [?loadLargestErlProcess, Len | NodStr]], - PidStr), - ok = snmp_get_next([?loadEntry ++ - [?loadSystemUsedMemory, Len | NodStr]], - ?loadEntry ++ [?loadSystemUsedMemory + 1, Len - | NodStr], PidStr), - ok = snmp_set([?loadEntry ++ [?loadLargestErlProcess, Len | NodStr]], - s, "<0.101.0>", Config), - ok. - -%% Starts an snmp manager and requests total memory. Was previously -%% integer32 which was errornous on 64 bit machines. -otp_7441(Config) when is_list(Config) -> - NodStr = atom_to_list(node()), - Len = length(NodStr), - Oids = [Oid|_] = [?loadEntry ++ [?loadSystemTotalMemory, Len | NodStr]], - {noError,0,[#varbind{oid = Oid, variabletype = 'Unsigned32'}]} = - ct_snmp:get_values(os_mon_mib_test, Oids, snmp_mgr_agent), - - ok. - -%%--------------------------------------------------------------------- -%% Internal functions -%%--------------------------------------------------------------------- -start_node() -> - Pa = filename:dirname(code:which(?MODULE)), - {ok,Node} = test_server:start_node(testnisse, slave, [{args, " -pa " ++ Pa}]), - Node. - -stop_node(Node) -> - test_server:stop_node(Node). - -del_dir(Dir) -> - io:format("Deleting: ~s~n",[Dir]), - {ok, Files} = file:list_dir(Dir), - FullPathFiles = lists:map(fun(File) -> filename:join(Dir, File) end, - Files), - lists:foreach(fun file:delete/1, FullPathFiles), - file:del_dir(Dir). - -%%--------------------------------------------------------------------- -snmp_get(Oids = [Oid |_], Result) -> - {noError,0,[#varbind{oid = Oid, - variabletype = 'OCTET STRING', - value = Result}]} = - ct_snmp:get_values(os_mon_mib_test, Oids, snmp_mgr_agent), - ok. - -snmp_get_next(Oids, NextOid, Result) -> - {noError,0,[#varbind{oid = NextOid, - variabletype = 'OCTET STRING', - value = Result}]} = - ct_snmp:get_next_values(os_mon_mib_test, Oids, snmp_mgr_agent), - ok. - -snmp_set(Oid, ValuType, Value, Config) -> - {notWritable, _, _} = - ct_snmp:set_values(os_mon_mib_test, [{Oid, ValuType, Value}], - snmp_mgr_agent, Config), - ok. diff --git a/lib/otp_mibs/AUTHORS b/lib/otp_mibs/AUTHORS deleted file mode 100644 index 3f570082f4..0000000000 --- a/lib/otp_mibs/AUTHORS +++ /dev/null @@ -1,8 +0,0 @@ -Original Authors and Contributors: - -Martin Bj�rklund -Lars Thorsen -Claes Wikstr�m -Kent Boortz -Bj�rn Gustavsson -Ingela Anderton - Created otp_mibs app. to eliminate SASL's SNMP dependence. diff --git a/lib/otp_mibs/Makefile b/lib/otp_mibs/Makefile deleted file mode 100644 index 64bd683c7a..0000000000 --- a/lib/otp_mibs/Makefile +++ /dev/null @@ -1,37 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 2003-2016. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# %CopyrightEnd% -# -include $(ERL_TOP)/make/target.mk -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -# ---------------------------------------------------- -# Macros -# ---------------------------------------------------- - -SUB_DIRECTORIES = src mibs doc/src - -include vsn.mk -VSN = $(OTP_MIBS_VSN) - -SPECIAL_TARGETS = - -# ---------------------------------------------------- -# Default Subdir Targets -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_subdir.mk diff --git a/lib/otp_mibs/doc/html/.gitignore b/lib/otp_mibs/doc/html/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/otp_mibs/doc/html/.gitignore +++ /dev/null diff --git a/lib/otp_mibs/doc/man3/.gitignore b/lib/otp_mibs/doc/man3/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/otp_mibs/doc/man3/.gitignore +++ /dev/null diff --git a/lib/otp_mibs/doc/pdf/.gitignore b/lib/otp_mibs/doc/pdf/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/otp_mibs/doc/pdf/.gitignore +++ /dev/null diff --git a/lib/otp_mibs/doc/src/Makefile b/lib/otp_mibs/doc/src/Makefile deleted file mode 100644 index 22c3c127ac..0000000000 --- a/lib/otp_mibs/doc/src/Makefile +++ /dev/null @@ -1,116 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 2003-2018. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# %CopyrightEnd% -# - -include $(ERL_TOP)/make/target.mk -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -# ---------------------------------------------------- -# Application version -# ---------------------------------------------------- -include ../../vsn.mk -VSN=$(OTP_MIBS_VSN) -APPLICATION=otp_mibs - -# ---------------------------------------------------- -# Release directory specification -# ---------------------------------------------------- -RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) -# ---------------------------------------------------- -# Target Specs -# ---------------------------------------------------- -XML_APPLICATION_FILES = ref_man.xml -XML_REF3_FILES = otp_mib.xml - -XML_PART_FILES = part.xml -XML_CHAPTER_FILES = \ - introduction.xml \ - mibs.xml \ - notes.xml - -BOOK_FILES = book.xml - -XML_FILES = \ - $(BOOK_FILES) $(XML_CHAPTER_FILES) \ - $(XML_PART_FILES) $(XML_REF3_FILES) $(XML_APPLICATION_FILES) - -GIF_FILES = - -# ---------------------------------------------------- - -HTML_FILES = $(XML_APPLICATION_FILES:%.xml=$(HTMLDIR)/%.html) \ - $(XML_PART_FILES:%.xml=$(HTMLDIR)/%.html) - -INFO_FILE = ../../info - -MAN3_FILES = $(XML_REF3_FILES:%.xml=$(MAN3DIR)/%.3) - -HTML_REF_MAN_FILE = $(HTMLDIR)/index.html - -TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf - -# ---------------------------------------------------- -# FLAGS -# ---------------------------------------------------- -XML_FLAGS += -DVIPS_FLAGS += - -# ---------------------------------------------------- -# Targets -# ---------------------------------------------------- -$(HTMLDIR)/%.gif: %.gif - $(INSTALL_DATA) $< $@ - -docs: pdf html man - -$(TOP_PDF_FILE): $(XML_FILES) - -pdf: $(TOP_PDF_FILE) - -html: gifs $(HTML_REF_MAN_FILE) - -man: $(MAN3_FILES) - -gifs: $(GIF_FILES:%=$(HTMLDIR)/%) - -debug opt: - -clean clean_docs: - rm -rf $(HTMLDIR)/* - rm -rf $(XMLDIR) - rm -f $(MAN3DIR)/* - rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) - rm -f errs core *~ - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_docs_spec: docs - $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html" - $(INSTALL_DATA) $(HTMLDIR)/* \ - "$(RELSYSDIR)/doc/html" - $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" - $(INSTALL_DIR) "$(RELEASE_PATH)/man/man3" - $(INSTALL_DATA) $(MAN3DIR)/* "$(RELEASE_PATH)/man/man3" - -release_spec: diff --git a/lib/otp_mibs/doc/src/book.xml b/lib/otp_mibs/doc/src/book.xml deleted file mode 100644 index 482da46876..0000000000 --- a/lib/otp_mibs/doc/src/book.xml +++ /dev/null @@ -1,49 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE book SYSTEM "book.dtd"> - -<book xmlns:xi="http://www.w3.org/2001/XInclude"> - <header titlestyle="normal"> - <copyright> - <year>2003</year><year>2016</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>OTP_Mibs</title> - <prepared>Ingela Anderton</prepared> - <docno></docno> - <date>2003-04-15</date> - <rev>A</rev> - </header> - <insidecover> - </insidecover> - <pagetext>OTP_Mibs application</pagetext> - <preamble> - <contents level="2"></contents> - </preamble> - <parts lift="no"> - <xi:include href="part.xml"/> - </parts> - <applications> - <xi:include href="ref_man.xml"/> - </applications> - <releasenotes> - <xi:include href="notes.xml"/> - </releasenotes> - <index></index> -</book> - - diff --git a/lib/otp_mibs/doc/src/introduction.xml b/lib/otp_mibs/doc/src/introduction.xml deleted file mode 100644 index 7046cbb8ae..0000000000 --- a/lib/otp_mibs/doc/src/introduction.xml +++ /dev/null @@ -1,47 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2003</year><year>2016</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Introduction</title> - <prepared>Ingela Anderton</prepared> - <docno></docno> - <date>2003-05-19</date> - <rev>A</rev> - <file>introduction.xml</file> - </header> - - <section> - <title>Purpose</title> - <p>The purpose of the OTP_Mibs application is to provide an SNMP - management information base for Erlang nodes.</p> - </section> - - <section> - <title>Pre-requisites</title> - <p>It is assumed that the reader is familiar with the Erlang - programming language, concepts of OTP and has a basic knowledge - of SNMP.</p> - </section> -</chapter> - - diff --git a/lib/otp_mibs/doc/src/mibs.xml b/lib/otp_mibs/doc/src/mibs.xml deleted file mode 100644 index a32d5ea5f5..0000000000 --- a/lib/otp_mibs/doc/src/mibs.xml +++ /dev/null @@ -1,71 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2003</year><year>2016</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>Mibs</title> - <prepared>Ingela Anderton</prepared> - <docno></docno> - <date>2003-05-19</date> - <rev>A</rev> - <file>mibs.xml</file> - </header> - - <section> - <title>Structure</title> - <p>The OTP mibs are stored in the - <c>$OTP_ROOT/lib/otp_mibs/mibs/</c> directory. They - are defined in SNMPv2 SMI syntax. An SNMPv1 version of the mib is - delivered in the <c>mibs/v1</c> directory. The compiled MIB is - located under <c>priv/mibs</c>, and the generated <c>.hrl</c> - file under the <c>include</c> directory. To compile a MIB that - IMPORTS a MIB in the OTP_Mibs application, give the option - <c>{il, ["otp_mibs/priv/mibs"]}</c> to the MIB compiler.</p> - </section> - - <section> - <title>OTP-MIB</title> - <p>The OTP-MIB mib represents information about Erlang nodes such as - node name, number of running processes, virtual machine version - etc. If the MIB should be used in a system, it should be - loaded into an SNMP agent by using the API function - <c>otp_mib:load/1</c>.</p> - </section> - - <section> - <title>OTP-REG</title> - <p>The OTP-REG mib defines the unique OTP subtree of object - identifiers under the Ericsson subtree. Under the OTP subtree - several object identifiers are defined. This module is typically - included by OTP applications defining their own mibs, or ASN.1 - modules in general, that require unique object identifiers under - the OTP subtree.</p> - </section> - - <section> - <title>OTP-TC</title> - <p>The OTP-TC mib provides the textual convention datatype - <c>OwnerString</c>.</p> - </section> -</chapter> - - diff --git a/lib/otp_mibs/doc/src/notes.xml b/lib/otp_mibs/doc/src/notes.xml deleted file mode 100644 index 443f08f1e1..0000000000 --- a/lib/otp_mibs/doc/src/notes.xml +++ /dev/null @@ -1,327 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2004</year><year>2018</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>OTP_Mibs Release Notes</title> - <prepared>otp_appnotes</prepared> - <docno>nil</docno> - <date>nil</date> - <rev>nil</rev> - <file>notes.xml</file> - </header> - <p>This document describes the changes made to the OTP_Mibs - application.</p> - -<section><title>Otp_Mibs 1.2.1</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Improved documentation.</p> - <p> - Own Id: OTP-15190</p> - </item> - </list> - </section> - -</section> - -<section><title>Otp_Mibs 1.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The otp_mibs application has been deprecated and will be - removed in a future release.</p> - <p> - Own Id: OTP-15141</p> - </item> - </list> - </section> - -</section> - -<section><title>Otp_Mibs 1.1.2</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> Removed all old unused files in the documentation. - </p> - <p> - Own Id: OTP-14475 Aux Id: ERL-409, PR-1493 </p> - </item> - </list> - </section> - -</section> - -<section><title>Otp_Mibs 1.1.1</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Internal changes</p> - <p> - Own Id: OTP-13551</p> - </item> - </list> - </section> - -</section> - -<section><title>Otp_Mibs 1.1</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Change license text from Erlang Public License to Apache - Public License v2</p> - <p> - Own Id: OTP-12845</p> - </item> - </list> - </section> - -</section> - -<section><title>Otp_Mibs 1.0.10</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Make sure the clean rule for ssh, ssl, eunit and otp_mibs - actually removes generated files.</p> - <p> - Own Id: OTP-12200</p> - </item> - </list> - </section> - -</section> - -<section><title>Otp_Mibs 1.0.9</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Add type based integer value truncation/reset.</p> - <p> - This fixes errors when querying e.g. the - erlNodeReductions, erlNodeInBytes and erlNodeOutBytes - objects in long-running Erlang/OTP systems.</p> - <p> - Update types of applicable MIB objects to 64bit based - types.</p> - <p> - Potential incompatibility: Type change of Counter32 to - Counter64 in OTP-MIB.mib</p> - <p> - (Thanks to Tobias Schlager)</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-11203</p> - </item> - <item> - <p> - Application upgrade (appup) files are corrected for the - following applications: </p> - <p> - <c>asn1, common_test, compiler, crypto, debugger, - dialyzer, edoc, eldap, erl_docgen, et, eunit, gs, hipe, - inets, observer, odbc, os_mon, otp_mibs, parsetools, - percept, public_key, reltool, runtime_tools, ssh, - syntax_tools, test_server, tools, typer, webtool, wx, - xmerl</c></p> - <p> - A new test utility for testing appup files is added to - test_server. This is now used by most applications in - OTP.</p> - <p> - (Thanks to Tobias Schlager)</p> - <p> - Own Id: OTP-11744</p> - </item> - </list> - </section> - -</section> - -<section><title>Otp_Mibs 1.0.8</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Misc build updates</p> - <p> - Own Id: OTP-10784</p> - </item> - </list> - </section> - -</section> - -<section><title>Otp_Mibs 1.0.7</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Tuple funs (a two-element tuple with a module name and a - function) are now officially deprecated and will be - removed in R16. Use '<c>fun M:F/A</c>' instead. To make - you aware that your system uses tuple funs, the very - first time a tuple fun is applied, a warning will be sent - to the error logger.</p> - <p> - Own Id: OTP-9649</p> - </item> - </list> - </section> - -</section> - -<section><title>Otp_Mibs 1.0.6</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The documentation is now built with open source tools - (xsltproc and fop) that exists on most platforms. One - visible change is that the frames are removed.</p> - <p> - Own Id: OTP-8201</p> - </item> - </list> - </section> - -</section> - -<section><title>Otp_Mibs 1.0.5</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p>The copyright notices have been updated.</p> - <p> - Own Id: OTP-7851</p> - </item> - </list> - </section> - -</section> - - <section> - <title>Otp_Mibs 1.0.4.1</title> - - <section> - <title>Improvements and New Features</title> - <list type="bulleted"> - <item> - <p>Minor Makefile changes.</p> - <p>Own Id: OTP-6689</p> - </item> - </list> - </section> - </section> - - <section> - <title>OTP_Mibs 1.0.4</title> - - <section> - <title>Improvements and New Features</title> - <list type="bulleted"> - <item> - <p>Replaced calls to deprecated functions in <c>snmp</c> - with calls to the equivalent functions in <c>snmpa</c>.</p> - <p>Own Id: OTP-6112</p> - </item> - </list> - </section> - </section> - - <section> - <title>OTP_Mibs 1.0.3</title> - - <section> - <title>Improvements and New Features</title> - <list type="bulleted"> - <item> - <p>The <c>otp_mib</c> module has been cleaned up to improve the - maintainability. It should have no effect on the - functionality of the OTP_Mibs application.</p> - <p>Own Id: OTP-4982</p> - </item> - </list> - </section> - </section> - - <section> - <title>OTP_Mibs 1.0.2</title> - - <section> - <title>Fixed Bugs and Malfunctions</title> - <list type="bulleted"> - <item> - <p>Incorrect <c>.app</c> file (missing mandatory - <c>registered</c>).</p> - <p>Own Id: OTP-4823 Aux Id: Seq8145, OTP-4801 </p> - </item> - </list> - </section> - </section> - - <section> - <title>OTP_Mibs 1.0.1</title> - - <section> - <title>Fixed Bugs and Malfunctions</title> - <list type="bulleted"> - <item> - <p>Missing <c>.app</c> and <c>appup</c> files in <c>ebin</c>.</p> - <p>Own Id: OTP-4801 Aux Id: Seq8145 </p> - </item> - </list> - </section> - </section> - - <section> - <title>OTP_Mibs 1.0</title> - <p>The OTP mibs that where included in the SASL application - have been moved to this new application OTP_Mibs. The OTP - mibs had no real connection to SASL and it is desirable that - the core of Erlang/OTP is not dependent on SNMP.</p> - <p>Own Id: OTP-4686</p> - </section> -</chapter> - - diff --git a/lib/otp_mibs/doc/src/otp_mib.xml b/lib/otp_mibs/doc/src/otp_mib.xml deleted file mode 100644 index e7d338c165..0000000000 --- a/lib/otp_mibs/doc/src/otp_mib.xml +++ /dev/null @@ -1,73 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> - -<erlref> - <header> - <copyright> - <year>2003</year><year>2018</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>otp_mib</title> - <prepared>Ingela Anderton</prepared> - <docno></docno> - <date></date> - <rev></rev> - </header> - <module since="">otp_mib</module> - <modulesummary>Handles the OTP-MIB</modulesummary> - <description> - <p>The SNMP application should be used to start an SNMP agent. Then - the API functions below can be used to load/unload the OTP-MIB - into/from the agent. The instrumentation of the OTP-MIB uses - Mnesia, hence Mnesia must be started prior to loading the OTP-MIB.</p> - <warning> - <p>This application has been deprecated and will be removed in a furture release.</p> - </warning> - </description> - <funcs> - <func> - <name since="">load(Agent) -> ok | {error, Reason}</name> - <fsummary>Load the OTP-MIB</fsummary> - <type> - <v>Agent = pid() | atom()</v> - <v>Reason = term()</v> - </type> - <desc> - <p>Loads the OTP-MIB.</p> - </desc> - </func> - <func> - <name since="">unload(Agent) -> ok | {error, Reason}</name> - <fsummary>Unload the OTP-MIB</fsummary> - <type> - <v>Agent = pid() | atom()</v> - <v>Reason = term()</v> - </type> - <desc> - <p>Unloads the OTP-MIB.</p> - </desc> - </func> - </funcs> - - <section> - <title>See Also</title> - <p>snmp(3)</p> - </section> -</erlref> - - diff --git a/lib/otp_mibs/doc/src/ref_man.xml b/lib/otp_mibs/doc/src/ref_man.xml deleted file mode 100644 index 06c5aadcd9..0000000000 --- a/lib/otp_mibs/doc/src/ref_man.xml +++ /dev/null @@ -1,38 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE application SYSTEM "application.dtd"> - -<application xmlns:xi="http://www.w3.org/2001/XInclude"> - <header> - <copyright> - <year>2003</year><year>2016</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>OTP_Mibs Reference Manual</title> - <prepared>Ingela Anderton</prepared> - <docno></docno> - <date>2002-09-13</date> - <rev>A</rev> - </header> - <description> - <p>The <em>OTP_Mibs</em> application provides an SNMP management - information base for Erlang nodes.</p> - </description> - <xi:include href="otp_mib.xml"/> -</application> - - diff --git a/lib/otp_mibs/ebin/.gitignore b/lib/otp_mibs/ebin/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/otp_mibs/ebin/.gitignore +++ /dev/null diff --git a/lib/otp_mibs/include/.gitignore b/lib/otp_mibs/include/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/otp_mibs/include/.gitignore +++ /dev/null diff --git a/lib/otp_mibs/info b/lib/otp_mibs/info deleted file mode 100644 index aedd1c883b..0000000000 --- a/lib/otp_mibs/info +++ /dev/null @@ -1,2 +0,0 @@ -group: oam Operation & Maintenance Applications -short: SNMP management information base for Erlang/OTP nodes. diff --git a/lib/otp_mibs/mibs/Makefile b/lib/otp_mibs/mibs/Makefile deleted file mode 100644 index 11d790d014..0000000000 --- a/lib/otp_mibs/mibs/Makefile +++ /dev/null @@ -1,89 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 1997-2016. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# %CopyrightEnd% -# - -include $(ERL_TOP)/make/target.mk -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -# ---------------------------------------------------- -# Application version -# Release directory specification -# ---------------------------------------------------- -include ../vsn.mk -ifdef SASL_VSN -VSN=$(SASL_VSN) -RELSYSDIR = $(RELEASE_PATH)/lib/sasl-$(VSN) -else -VSN=$(OTP_MIBS_VSN) -RELSYSDIR = $(RELEASE_PATH)/lib/otp_mibs-$(VSN) -endif - -# ---------------------------------------------------- -# Target Specs -# ---------------------------------------------------- - -MIB_FILES= OTP-REG.mib OTP-TC.mib OTP-MIB.mib -FUNCS_FILES = OTP-MIB.funcs - -BIN_TARGETS= $(MIB_FILES:%.mib=$(SNMP_BIN_TARGET_DIR)/%.bin) -HRL_TARGETS= $(MIB_FILES:%.mib=$(SNMP_HRL_TARGET_DIR)/%.hrl) -V1_MIB_FILES= $(MIB_FILES:%.mib=v1/%.mib.v1) - -TARGET_FILES= $(BIN_TARGETS) $(HRL_TARGETS) $(V1_MIB_FILES) - -# ---------------------------------------------------- -# FLAGS -# ---------------------------------------------------- -SNMP_FLAGS = -I ../priv/mibs - -# ---------------------------------------------------- -# Targets -# ---------------------------------------------------- - -debug opt: $(TARGET_FILES) - -clean: - rm -f $(TARGET_FILES) - rm -f core - -docs: - -# ---------------------------------------------------- -# Special Build Targets -# ---------------------------------------------------- - -v1/%.mib.v1: %.mib - $(gen_verbose)$(ERL_TOP)/lib/snmp/bin/snmp-v2tov1 -o $@ $< - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_spec: opt - $(INSTALL_DIR) "$(RELSYSDIR)/mibs" - $(INSTALL_DIR) "$(RELSYSDIR)/mibs/v1" - $(INSTALL_DATA) $(MIB_FILES) $(FUNCS_FILES) "$(RELSYSDIR)/mibs" - $(INSTALL_DATA) $(V1_MIB_FILES) "$(RELSYSDIR)/mibs/v1" - $(INSTALL_DIR) "$(RELSYSDIR)/include" - $(INSTALL_DATA) $(HRL_TARGETS) "$(RELSYSDIR)/include" - $(INSTALL_DIR) "$(RELSYSDIR)/priv/mibs" - $(INSTALL_DATA) $(BIN_TARGETS) "$(RELSYSDIR)/priv/mibs" - -release_docs_spec: diff --git a/lib/otp_mibs/mibs/OTP-EVA-MIB.mib b/lib/otp_mibs/mibs/OTP-EVA-MIB.mib deleted file mode 100644 index 4d0c53ed95..0000000000 --- a/lib/otp_mibs/mibs/OTP-EVA-MIB.mib +++ /dev/null @@ -1,569 +0,0 @@ --- --- %CopyrightBegin% --- --- Copyright Ericsson AB 2004-2016. All Rights Reserved. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- %CopyrightEnd% --- - -OTP-EVA-MIB DEFINITIONS ::= BEGIN - -IMPORTS - MODULE-IDENTITY, OBJECT-TYPE, NOTIFICATION-TYPE, - Counter32, Gauge32, Integer32 - FROM SNMPv2-SMI - TEXTUAL-CONVENTION, DisplayString, DateAndTime - FROM SNMPv2-TC - MODULE-COMPLIANCE, OBJECT-GROUP, NOTIFICATION-GROUP - FROM SNMPv2-CONF - otpModules, otpApplications - FROM OTP-REG - OwnerString - FROM OTP-TC - ; - -otpEvaModule MODULE-IDENTITY - LAST-UPDATED "200305090900Z" - ORGANIZATION "Ericsson" - CONTACT-INFO - "Contact: Erlang Support see license agreement for Erlang/OTP." - DESCRIPTION - "This MIB is part of the OTP MIB. It defines MIB objects - for the eva application in OTP." - - REVISION "200305090900Z" - DESCRIPTION - "Changed CONTACT-INFO as it was outdated, made it more generic - to avoid such changes in the future." - - REVISION "199801270900Z" - DESCRIPTION - "Changed erroneous defintion of alarmCleared notification. - Changed erroneous name of this module to otpEvaModule." - REVISION "199712010900Z" - DESCRIPTION - "Converted to v2 SMI and placed in the OTP tree." - REVISION "199705020900Z" - DESCRIPTION - "The initial version of this MIB module. It is very much - inspired by the ANS-ALM-MIB and Axd301Eva-OMS mibs." - ::= { otpModules 6 } - -otpEvaMIB OBJECT IDENTIFIER ::= { otpApplications 4 } -otpEvaMIBConformance - OBJECT IDENTIFIER ::= { otpEvaMIB 1 } -otpEvaMIBObjects - OBJECT IDENTIFIER ::= { otpEvaMIB 2 } -otpEvaMIBEvents OBJECT IDENTIFIER ::= { otpEvaMIB 3 } -otpEvaMIBEventsV2 - OBJECT IDENTIFIER ::= { otpEvaMIBEvents 0 } - - --- Datatypes - -AlarmSeverity ::= TEXTUAL-CONVENTION - STATUS current - DESCRIPTION - "The AlarmSeverity defines six severity levels, - which provide an indication of how it is perceived that the - capability of the managed object has been affected. Those - severity levels which represent service affecting conditions - ordered from most severe to least severe are critical, - major, minor and warning. The levels used are as defined - in X.733, ITU Alarm Reporting Function: - - o The Indeterminate severity level indicates that the - severity level cannot be determined. - - o The Critical severity level indicates that a service - affecting condition has occurred and an immediate - corrective action is required. Such a severity can be - reported, for example, when a managed object becomes - totally out of service and its capability must be restored. - - o The Major severity level indicates that a service - affecting condition has developed and an urgent corrective - action is required. Such a severity can be reported, for - example, when there is a severe degradation in the - capability of the managed object and its full capability - must be restored. - - o The Minor severity level indicates the existence of a - non-service affecting fault condition and that corrective - action should be taken in order to prevent a more serious - (for example, service affecting) fault. Such a severity - can be reported, for example, when the detected alarm - condition is not currently degrading the capacity of the - managed object. - - o The Warning severity level indicates the detection of a - potential or impending service affecting fault, before any - significant effects have been felt. Action should be taken - to further diagnose (if necessary) and correct the problem - in order to prevent it from becoming a more serious service - affecting fault. - - When an alarm is cleared, an alarmCleared event is generated. - This event clears the alarm with the currentAlarmFaultId - contained in the event. It is not required that the clearing - of previously reported alarms are reported. Therefore, a managing - system cannot assume that the absence of an alarmedCleared event - for a fault means that the condition that caused the generation - of previous alarms is still present. Managed object definers - shall state if, and under which conditions, the alarmedCleared - event is used. - - The clear value of AlarmSeverity is an action which is used when - a management station wants to clear an active alarm. This is not - possible on all systems, and thus an agent does not have support - write access for this value." - REFERENCE - "X.733, ITU Alarm Reporting Function" - SYNTAX INTEGER { - indeterminate (0), - critical (1), - major (2), - minor (3), - warning (4), - clear (5) -- Written, not read - } - -AlarmClass ::= TEXTUAL-CONVENTION - STATUS current - DESCRIPTION - "The AlarmClass type categorizes the alarm, and is - defined when the alarm is registered. It is as defined in - X.733, ITU Alarm Reporting Function: - - o communications. An alarm of this class is principally - associated with the procedures or processes required - to convey information from one point to another. - - o qos. An alarm of this class is principally associated - with a degradation in the quality of service. - - o processing. An alarm of this class is principally - associated with a software or processing fault. - - o equipment. An alarm of this class is principally - associated with an equipment fault. - - o environmental. An alarm of this class is principally - associated with a condition relating to an enclosure in - with equipment resides." - REFERENCE - "X.733, ITU Alarm Reporting Function" - SYNTAX INTEGER { - unknown (0), - communications (1), - qos (2), - processing (3), - equipment (4), - environmental (5) - } - - --- Managed Objects - -event OBJECT IDENTIFIER ::= { otpEvaMIBObjects 1 } -alarm OBJECT IDENTIFIER ::= { otpEvaMIBObjects 2 } -currentAlarm OBJECT IDENTIFIER ::= { otpEvaMIBObjects 3 } - --- The Event group --- --- The Event group controls the generation of notifications of --- events from the system. - -eventTable OBJECT-TYPE - SYNTAX SEQUENCE OF EventEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A list of events defined by the system. This table is used - to control the sending of traps and to whom the traps are - sent." - ::= { event 1 } - -eventEntry OBJECT-TYPE - SYNTAX EventEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A set of parameters that describe an event to be - generated when certain conditions are met." - INDEX { eventIndex } - ::= { eventTable 1 } - -EventEntry ::= SEQUENCE { - eventIndex Integer32 (1..2147483647), - eventTrapName DisplayString, - eventTreatment INTEGER, - eventCommunity OCTET STRING (SIZE (0..127)), - eventSentTraps Counter32, - eventOwner OwnerString - } - -eventIndex OBJECT-TYPE - SYNTAX Integer32 (1..2147483647) - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "An index that uniquely identifies an entry in the - event table. Each such entry defines one event that - is to be generated when the appropriate conditions - occur. The value for each eventIndex must remain - constant, at least from one re-initialization of the - entity's network management system to the next - re-initialization." - ::= { eventEntry 1 } - -eventTrapName OBJECT-TYPE - SYNTAX DisplayString - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The identifier of the corresponding trap. - NOTE: this should be an OID in SNMPv2, but must be - a string in v1." - ::= { eventEntry 2 } - -eventTreatment OBJECT-TYPE - SYNTAX INTEGER { - none(1), - log(2), - snmpTrap(3), - logAndTrap(4) - } - MAX-ACCESS read-write - STATUS current - DESCRIPTION - "Defines how the system shall treat this event. In the - case of snmpTrap, an SNMP trap is sent to one or more - management stations. In the case of log, the event is - guaranteed to be logged in a log according to some log - strategy. Each such log strategy may define a MIB module - for control and examination of logs." - ::= { eventEntry 3 } - -eventCommunity OBJECT-TYPE - SYNTAX OCTET STRING (SIZE (0..127)) - MAX-ACCESS read-write - STATUS current - DESCRIPTION - "If an SNMP trap is to be sent, it will be sent to - the SNMP community specified by this octet string." - ::= { eventEntry 4 } - -eventSentTraps OBJECT-TYPE - SYNTAX Counter32 - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The number of times this event has been sent as a trap." - ::= { eventEntry 5 } - -eventOwner OBJECT-TYPE - SYNTAX OwnerString - MAX-ACCESS read-write - STATUS current - DESCRIPTION - "The manager entity that 'owns' this event entry, and is - therefore responsible for its contents." - ::= { eventEntry 6 } - -eventTime OBJECT-TYPE - SYNTAX DateAndTime - MAX-ACCESS accessible-for-notify - STATUS current - DESCRIPTION - "This object may be included in a trap definition for an event. - It specifies the time the event was generated." - ::= { event 2 } - - --- The Alarm group --- --- The Alarm group extends the Event group with objects for alarms. - -alarmTable OBJECT-TYPE - SYNTAX SEQUENCE OF AlarmEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "Contains additional information for alarm events." - ::= { alarm 1 } - -alarmEntry OBJECT-TYPE - SYNTAX AlarmEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A set of parameters for alarms." - INDEX { eventIndex } - ::= { alarmTable 1 } - -AlarmEntry ::= SEQUENCE { - alarmClass AlarmClass, - alarmSeverity AlarmSeverity - } - -alarmClass OBJECT-TYPE - SYNTAX AlarmClass - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The class of this alarm." - ::= { alarmEntry 1 } - -alarmSeverity OBJECT-TYPE - SYNTAX AlarmSeverity - MAX-ACCESS read-write - STATUS current - DESCRIPTION - "The perceived severity that shall apply to the - associated alarms." - ::= { alarmEntry 2 } - - --- The CurrentAlarm group --- --- The CurrentAlarm group is a collection of objects for monitoring of --- active alarms in the system. - -numberOfCurrentAlarms OBJECT-TYPE - SYNTAX Gauge32 - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "Number of currently active alarms in the system." - ::= { currentAlarm 1 } - -currentAlarmLastTimeChanged OBJECT-TYPE - SYNTAX DateAndTime - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The time an entry in the currentAlarmTable was changed. - It may be used by a management station as a value to - poll. If the value is changed, the management station - knows that the currentAlarmTable has been updated." - ::= { currentAlarm 2 } - -currentAlarmTable OBJECT-TYPE - SYNTAX SEQUENCE OF CurrentAlarmEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A list of currently active alarms in the system." - ::= { currentAlarm 3 } - -currentAlarmEntry OBJECT-TYPE - SYNTAX CurrentAlarmEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A set of parameters that describe a currently active - alarm." - INDEX { currentAlarmFaultId } - ::= { currentAlarmTable 1 } - -CurrentAlarmEntry ::= SEQUENCE { - currentAlarmFaultId Integer32 (1..2147483647), - currentAlarmEventIndex Integer32 (1..2147483647), - currentAlarmObject OBJECT IDENTIFIER, - currentAlarmCause OBJECT IDENTIFIER, - currentAlarmSeverity AlarmSeverity, - currentAlarmTime DateAndTime, - currentAlarmInformation DisplayString, - currentAlarmExtra1 OBJECT IDENTIFIER, - currentAlarmExtra2 OBJECT IDENTIFIER - } - -currentAlarmFaultId OBJECT-TYPE - SYNTAX Integer32 (1..2147483647) - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "An id that uniquely identifies a fault. Each fault is - represented as one an entry in the currentAlarmTable." - ::= { currentAlarmEntry 1 } - -currentAlarmEventIndex OBJECT-TYPE - SYNTAX Integer32 (1..2147483647) - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "A pointer into the eventTable. Points to the event - corresponding to this alarm." - ::= { currentAlarmEntry 2 } - -currentAlarmObject OBJECT-TYPE - SYNTAX OBJECT IDENTIFIER - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The alarming object." - ::= { currentAlarmEntry 3 } - -currentAlarmCause OBJECT-TYPE - SYNTAX OBJECT IDENTIFIER - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The probable cause of the alarm." - ::= { currentAlarmEntry 4 } - -currentAlarmSeverity OBJECT-TYPE - SYNTAX AlarmSeverity - MAX-ACCESS read-write - STATUS current - DESCRIPTION - "The perceived severity of the fault. A manager can set - this value to clear only. When set to clear, the alarm - is removed from this table, and a 'clearAlarm' event is - generated." - ::= { currentAlarmEntry 5 } - -currentAlarmTime OBJECT-TYPE - SYNTAX DateAndTime - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The time the fault was detected." - ::= { currentAlarmEntry 6 } - -currentAlarmInformation OBJECT-TYPE - SYNTAX DisplayString - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "Additional information pin-pointing the problem." - ::= { currentAlarmEntry 7 } - -currentAlarmExtra1 OBJECT-TYPE - SYNTAX OBJECT IDENTIFIER - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "An extra parameter used for some alarms at their own - discretion. Can be used for example to identify - additional objects in the alarm, or instead of - currentAlarmInformation to pin-point the problem, if the - additional information is defined in some MIB." - ::= { currentAlarmEntry 8 } - -currentAlarmExtra2 OBJECT-TYPE - SYNTAX OBJECT IDENTIFIER - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "An extra parameter used for some alarms at their own - discretion. Can be used for example to identify - additional objects in the alarm, or instead of - currentAlarmInformation to pin-point the problem, if the - additional information is defined in some MIB." - ::= { currentAlarmEntry 9 } - - --- Events - -alarmCleared NOTIFICATION-TYPE - OBJECTS { - currentAlarmEventIndex, - eventTime - } - STATUS current - DESCRIPTION - "This event is sent when an alarm has been cleared, - either by the application or by an operator. Note that the - currentAlarmFaultId is implicitly sent as the instance identifier - for currentAlarmEventIndex." - ::= { otpEvaMIBEventsV2 1 } - - --- conformance information - -otpEvaMIBCompliances - OBJECT IDENTIFIER ::= { otpEvaMIBConformance 1 } -otpEvaMIBGroups - OBJECT IDENTIFIER ::= { otpEvaMIBConformance 2 } - - --- compliance statements - -otpEvaBasicCompliance MODULE-COMPLIANCE - STATUS current - DESCRIPTION - "The compliance statement for SNMPv2 entities which - implement the OTP-EVA-MIB." - MODULE -- this module - MANDATORY-GROUPS { eventGroup, - alarmGroup, - currentAlarmGroup, - evaEventsGroup } - - ::= { otpEvaMIBCompliances 1 } - - --- units of conformance - -eventGroup OBJECT-GROUP - OBJECTS { eventTrapName, - eventTreatment, - eventCommunity, - eventSentTraps, - eventOwner, - eventTime } - STATUS current - DESCRIPTION - "A collection of objects providing basic instrumentation - and control of the events defined in the OTP system." - ::= { otpEvaMIBGroups 1 } - -alarmGroup OBJECT-GROUP - OBJECTS { alarmClass, - alarmSeverity } - STATUS current - DESCRIPTION - "A collection of objects providing basic instrumentation - and control of the alarms defined the OTP system." - ::= { otpEvaMIBGroups 2 } - -currentAlarmGroup OBJECT-GROUP - OBJECTS { numberOfCurrentAlarms, - currentAlarmLastTimeChanged, - currentAlarmEventIndex, - currentAlarmObject, - currentAlarmCause, - currentAlarmSeverity, - currentAlarmTime, - currentAlarmInformation, - currentAlarmExtra1, - currentAlarmExtra2 } - STATUS current - DESCRIPTION - "A collection of objects providing basic instrumentation - of the activa alarm list in the OTP system." - ::= { otpEvaMIBGroups 3 } - -evaEventsGroup NOTIFICATION-GROUP - NOTIFICATIONS { alarmCleared } - STATUS current - DESCRIPTION - "The notification which is generated from EVA." - ::= { otpEvaMIBGroups 4 } - - -END diff --git a/lib/otp_mibs/mibs/OTP-MIB.funcs b/lib/otp_mibs/mibs/OTP-MIB.funcs deleted file mode 100644 index 9f9d69c3d1..0000000000 --- a/lib/otp_mibs/mibs/OTP-MIB.funcs +++ /dev/null @@ -1,2 +0,0 @@ -{erlNodeTable, {otp_mib, erl_node_table, []}}. -{applTable, {otp_mib, appl_table, []}}. diff --git a/lib/otp_mibs/mibs/OTP-MIB.mib b/lib/otp_mibs/mibs/OTP-MIB.mib deleted file mode 100644 index 693319eae4..0000000000 --- a/lib/otp_mibs/mibs/OTP-MIB.mib +++ /dev/null @@ -1,318 +0,0 @@ --- --- %CopyrightBegin% --- --- Copyright Ericsson AB 1996-2016. All Rights Reserved. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- %CopyrightEnd% --- - -OTP-MIB DEFINITIONS ::= BEGIN - -IMPORTS - MODULE-IDENTITY, OBJECT-TYPE, - Counter64, Gauge32, Integer32 - FROM SNMPv2-SMI - TEXTUAL-CONVENTION, DisplayString - FROM SNMPv2-TC - MODULE-COMPLIANCE, OBJECT-GROUP - FROM SNMPv2-CONF - otpModules, otpApplications - FROM OTP-REG - ; - -otpModule MODULE-IDENTITY - LAST-UPDATED "201307160700Z" - ORGANIZATION "Ericsson" - CONTACT-INFO - "Contact: Erlang Support see license agreement for Erlang/OTP." - DESCRIPTION - "This is a MIB for a distributed OTP system, with one SNMP - agent executing at one node only. Each Erlang node in the - OTP system is represented by one row in the erlNodeTable." - - REVISION "201307160700Z" - DESCRIPTION - "Updated various types to be able to reflect larger values. - The objects erlNodeReductions, erlNodeInBytes, erlNodeOutBytes as well - as the type MilliSeconds have been updated to Counter64." - - REVISION "200305090900Z" - DESCRIPTION - "Changed CONTACT-INFO as it was outdated, made it more generic - to avoid such changes in the future." - - REVISION "199712010900Z" - DESCRIPTION - "Converted to v2 SMI and placed in the OTP tree." - - REVISION "199608191700Z" - DESCRIPTION - "The initial revision of MIB module OTP-MIB." - - ::= { otpModules 3 } - -otpMIB OBJECT IDENTIFIER ::= { otpApplications 1 } -otpMIBConformance - OBJECT IDENTIFIER ::= { otpMIB 1 } -otpMIBObjects OBJECT IDENTIFIER ::= { otpMIB 2 } - --- Datatypes - -MilliSeconds ::= TEXTUAL-CONVENTION - STATUS current - DESCRIPTION - "The MilliSeconds type represents a Counter which represents - the time, in milliseconds between two epochs. When objects - are defined which use this type, the description of the object - identifies both of the reference epochs." - SYNTAX Counter64 - - --- Managed Objects - -erlang OBJECT IDENTIFIER ::= { otpMIBObjects 1 } -appls OBJECT IDENTIFIER ::= { otpMIBObjects 2 } - -erlNodeTable OBJECT-TYPE - SYNTAX SEQUENCE OF ErlNodeEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A table with info on each erlang node in the system." - ::= { erlang 1 } - -erlNodeEntry OBJECT-TYPE - SYNTAX ErlNodeEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A conceptual row in the erlNodeTable." - INDEX { erlNodeId } - ::= { erlNodeTable 1 } - -ErlNodeEntry ::= SEQUENCE { - erlNodeId Integer32, - erlNodeName DisplayString, - erlNodeMachine DisplayString, - erlNodeVersion DisplayString, - erlNodeRunQueue Gauge32, - erlNodeRunTime MilliSeconds, - erlNodeWallClock MilliSeconds, - erlNodeReductions Counter64, - erlNodeProcesses Gauge32, - erlNodeInBytes Counter64, - erlNodeOutBytes Counter64 -} - -erlNodeId OBJECT-TYPE - SYNTAX Integer32 - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "An integer that uniquely identifies the erlang node." - ::= { erlNodeEntry 1 } - -erlNodeName OBJECT-TYPE - SYNTAX DisplayString - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The symbolic name of the erlang node." - ::= { erlNodeEntry 2 } - -erlNodeMachine OBJECT-TYPE - SYNTAX DisplayString - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The virtual machine executing the erlang node" - ::= { erlNodeEntry 3 } - -erlNodeVersion OBJECT-TYPE - SYNTAX DisplayString - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The version number of the virtual machine" - ::= { erlNodeEntry 4 } - -erlNodeRunQueue OBJECT-TYPE - SYNTAX Gauge32 - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "Number of processes scheduled to run" - ::= { erlNodeEntry 5 } - -erlNodeRunTime OBJECT-TYPE - SYNTAX MilliSeconds - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "Total cpu time in milliseconds since the - system started" - ::= { erlNodeEntry 6 } - -erlNodeWallClock OBJECT-TYPE - SYNTAX MilliSeconds - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "Total real time in milliseconds since the - system started" - ::= { erlNodeEntry 7 } - -erlNodeReductions OBJECT-TYPE - SYNTAX Counter64 - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "Number of function calls since the system started" - ::= { erlNodeEntry 8 } - -erlNodeProcesses OBJECT-TYPE - SYNTAX Gauge32 - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "Number of running processes in the system." - ::= { erlNodeEntry 9 } - -erlNodeInBytes OBJECT-TYPE - SYNTAX Counter64 - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The total number of bytes delivered to the system" - ::= { erlNodeEntry 10 } - -erlNodeOutBytes OBJECT-TYPE - SYNTAX Counter64 - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The total number of bytes delivered from the system" - ::= { erlNodeEntry 11 } - - -applTable OBJECT-TYPE - SYNTAX SEQUENCE OF ApplEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A table with all currently running applications - for each node." - ::= { appls 1 } - -applEntry OBJECT-TYPE - SYNTAX ApplEntry - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "A conceptual row in the applTable." - INDEX { erlNodeId, applId } - ::= { applTable 1 } - -ApplEntry ::= SEQUENCE { - applId Integer32, - applName DisplayString, - applDescr DisplayString, - applVsn DisplayString -} - -applId OBJECT-TYPE - SYNTAX Integer32 - MAX-ACCESS not-accessible - STATUS current - DESCRIPTION - "An integer that uniquely identifies the application." - ::= { applEntry 1 } - -applName OBJECT-TYPE - SYNTAX DisplayString - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The name of the application." - ::= { applEntry 2 } - -applDescr OBJECT-TYPE - SYNTAX DisplayString - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "A short description of the application." - ::= { applEntry 3 } - -applVsn OBJECT-TYPE - SYNTAX DisplayString - MAX-ACCESS read-only - STATUS current - DESCRIPTION - "The version of the application." - ::= { applEntry 4 } - - --- conformance information - -otpMIBCompliances - OBJECT IDENTIFIER ::= { otpMIBConformance 1 } -otpMIBGroups OBJECT IDENTIFIER ::= { otpMIBConformance 2 } - - --- compliance statements - -otpBasicCompliance MODULE-COMPLIANCE - STATUS current - DESCRIPTION - "The compliance statement for SNMPv2 entities which - implement the OTP-MIB." - MODULE -- this module - MANDATORY-GROUPS { erlGroup, applGroup } - - ::= { otpMIBCompliances 1 } - - --- units of conformance - -erlGroup OBJECT-GROUP - OBJECTS { erlNodeName, - erlNodeMachine, - erlNodeVersion, - erlNodeRunQueue, - erlNodeRunTime, - erlNodeWallClock, - erlNodeReductions, - erlNodeProcesses, - erlNodeInBytes, - erlNodeOutBytes } - STATUS current - DESCRIPTION - "A collection of objects providing basic instrumentation - of the Erlang runtime system." - ::= { otpMIBGroups 1 } - -applGroup OBJECT-GROUP - OBJECTS { applName, - applDescr, - applVsn } - STATUS current - DESCRIPTION - "A collection of objects providing basic instrumentation - of the applications in the OTP system." - ::= { otpMIBGroups 2 } - - -END diff --git a/lib/otp_mibs/mibs/v1/.gitignore b/lib/otp_mibs/mibs/v1/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/otp_mibs/mibs/v1/.gitignore +++ /dev/null diff --git a/lib/otp_mibs/priv/bin/.gitignore b/lib/otp_mibs/priv/bin/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/otp_mibs/priv/bin/.gitignore +++ /dev/null diff --git a/lib/otp_mibs/priv/mibs/.gitignore b/lib/otp_mibs/priv/mibs/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/otp_mibs/priv/mibs/.gitignore +++ /dev/null diff --git a/lib/otp_mibs/priv/obj/.gitignore b/lib/otp_mibs/priv/obj/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/lib/otp_mibs/priv/obj/.gitignore +++ /dev/null diff --git a/lib/otp_mibs/src/Makefile b/lib/otp_mibs/src/Makefile deleted file mode 100644 index 5c7af39c3f..0000000000 --- a/lib/otp_mibs/src/Makefile +++ /dev/null @@ -1,106 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 2003-2016. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# %CopyrightEnd% -# -include $(ERL_TOP)/make/target.mk - -ifeq ($(TYPE),debug) -ERL_COMPILE_FLAGS += -Ddebug -W -endif - -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -# ---------------------------------------------------- -# Application version -# ---------------------------------------------------- -include ../vsn.mk -VSN=$(OTP_MIBS_VSN) - -# ---------------------------------------------------- -# Release directory specification -# ---------------------------------------------------- -RELSYSDIR = $(RELEASE_PATH)/lib/otp_mibs-$(VSN) -# ---------------------------------------------------- -# Target Specs -# ---------------------------------------------------- -MODULES = \ - otp_mib - -INCLUDE=../include - -HRL_FILES = - -INTERNAL_HRL_FILES = - -ERL_FILES = $(MODULES:%=%.erl) - -APP_FILE = otp_mibs.app -APP_SRC = $(APP_FILE).src -APP_TARGET = $(EBIN)/$(APP_FILE) - -APPUP_FILE = otp_mibs.appup -APPUP_SRC = $(APPUP_FILE).src -APPUP_TARGET = $(EBIN)/$(APPUP_FILE) - -TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) - -TARGETS = $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) - -# ---------------------------------------------------- -# FLAGS -# ---------------------------------------------------- -ERL_COMPILE_FLAGS += -I$(INCLUDE) - -# ---------------------------------------------------- -# Targets -# ---------------------------------------------------- - -debug opt: $(TARGETS) - -clean: - rm -f $(TARGET_FILES) - rm -f $(APP_TARGET) - rm -f $(APPUP_TARGET) - rm -f core - -docs: - -# ---------------------------------------------------- -# Special Build Targets -# ---------------------------------------------------- - -$(APP_TARGET): $(APP_SRC) ../vsn.mk - $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ - -$(APPUP_TARGET): $(APPUP_SRC) ../vsn.mk - $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_spec: opt - $(INSTALL_DIR) "$(RELSYSDIR)/src" - $(INSTALL_DATA) $(ERL_FILES) "$(RELSYSDIR)/src" - $(INSTALL_DIR) "$(RELSYSDIR)/ebin" - $(INSTALL_DATA) $(TARGETS) "$(RELSYSDIR)/ebin" - -release_docs_spec: - - diff --git a/lib/otp_mibs/src/otp_mib.erl b/lib/otp_mibs/src/otp_mib.erl deleted file mode 100644 index ca868f2817..0000000000 --- a/lib/otp_mibs/src/otp_mib.erl +++ /dev/null @@ -1,219 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% --module(otp_mib). -%%%----------------------------------------------------------------- -%%% Description: This module implements the OTP-MIB. -%%% The tables are implemented as shadow tables with the module -%%% snmp_shadow_table. -%%%----------------------------------------------------------------- - -%% API --export([load/1, unload/1]). - -%% SNMP instrumentation --export([erl_node_table/1, erl_node_table/3, appl_table/1, appl_table/3]). - -%% SNMP shadow functions --export([update_erl_node_table/0, update_appl_table/0]). - -%% Exported for internal use via rpc --export([get_erl_node/1, get_appls/1]). - --deprecated([{load,1,eventually}, - {unload,1,eventually}]). - -%% Shadow tables --record(erlNodeTable, - {erlNodeId, erlNodeName, erlNodeMachine, erlNodeVersion, - erlNodeRunQueue, - erlNodeRunTime, erlNodeWallClock, erlNodeReductions, - erlNodeProcesses, erlNodeInBytes, erlNodeOutBytes}). - --record(applTable, {key = '_', applName = '_', applDescr = '_', - applVsn = '_'}). - -%% Shadow argument macros --define(erlNodeShadowArgs, - {erlNodeTable, integer, record_info(fields, erlNodeTable), 5000, - fun otp_mib:update_erl_node_table/0}). - --define(applShadowArgs, - {applTable, {integer, integer}, record_info(fields, applTable), - 5000, fun otp_mib:update_appl_table/0}). - -%% Misc --record(erlNodeAlloc, {nodeName, nodeId}). - -%%%========================================================================= -%%% API -%%%========================================================================= - -%%------------------------------------------------------------------------- -%% load(Agent) -> ok | {error, Reason} -%% Agent - pid() | atom() -%% Reason - term() -%% Description: Loads the OTP-MIB -%%------------------------------------------------------------------------- -load(Agent) -> - MibDir = code:priv_dir(otp_mibs) ++ "/mibs", - snmpa:load_mibs(Agent, [MibDir ++ "/OTP-MIB"]). - -%%------------------------------------------------------------------------- -%% unload(Agent) -> ok | {error, Reason} -%% Agent - pid() | atom() -%% Reason - term() -%% Description: Loads the OTP-MIB -%%------------------------------------------------------------------------- -unload(Agent) -> - snmpa:unload_mibs(Agent, ["OTP-MIB"]). - - -%%%========================================================================= -%%% SNMP instrumentation -%%%========================================================================= -erl_node_table(new) -> - Tab = erlNodeAlloc, - Storage = ram_copies, - case lists:member(Tab, mnesia:system_info(tables)) of - true -> - case mnesia:table_info(Tab, storage_type) of - unknown -> - {atomic, ok} = mnesia:add_table_copy(Tab, node(), Storage); - Storage -> - catch delete_all(Tab) - end; - false -> - Nodes = [node()], - Props = [{type, set}, - {attributes, record_info(fields, erlNodeAlloc)}, - {local_content, true}, - {Storage, Nodes}], - {atomic, ok} = mnesia:create_table(Tab, Props) - end, - ok = mnesia:dirty_write({erlNodeAlloc, next_index, 1}), - update_node_alloc([node() | nodes()]), - snmp_shadow_table:table_func(new, ?erlNodeShadowArgs). - -erl_node_table(Op, RowIndex, Cols) -> - snmp_shadow_table:table_func(Op, RowIndex, Cols, ?erlNodeShadowArgs). - - -appl_table(Op) -> - snmp_shadow_table:table_func(Op, ?applShadowArgs). -appl_table(Op, RowIndex, Cols) -> - snmp_shadow_table:table_func(Op, RowIndex, Cols, ?applShadowArgs). - - -%%%========================================================================= -%%% SNMP shadow functions -%%%========================================================================= -update_erl_node_table() -> - delete_all(erlNodeTable), - Nodes = [node() | nodes()], - update_node_alloc(Nodes), - lists:foreach( - fun(Node) -> - [{_,_,Idx}] = mnesia:dirty_read({erlNodeAlloc, Node}), - ErlNode = rpc:call(Node, otp_mib, get_erl_node, [Idx]), - ok = mnesia:dirty_write(ErlNode) - end, Nodes). - -update_appl_table() -> - delete_all(applTable), - Nodes = [node() | nodes()], - update_node_alloc(Nodes), - lists:foreach( - fun(Node) -> - [{_,_,Idx}] = mnesia:dirty_read({erlNodeAlloc, Node}), - Appls = rpc:call(Node, otp_mib, get_appls, [Idx]), - lists:foreach(fun(Appl) -> - ok = mnesia:dirty_write(Appl) - end, Appls) - end, Nodes). - -%%%======================================================================== -%%% Exported for internal use via rpc -%%%======================================================================== -get_erl_node(Id) -> - RunQueue = erlang:statistics(run_queue), - RunTime = element(1, erlang:statistics(runtime)), - WallClock = element(1, erlang:statistics(wall_clock)), - Reductions = element(1, erlang:statistics(reductions)), - Processes = length(processes()), - IO = erlang:statistics(io), - InBytes = element(2, element(1, IO)), - OutBytes = element(2, element(2, IO)), - #erlNodeTable{erlNodeId = truncate_int('Integer32', Id), - erlNodeName = atom_to_list(node()), - erlNodeVersion = erlang:system_info(version), - erlNodeMachine = erlang:system_info(machine), - erlNodeRunQueue = truncate_int('Unsigned32', RunQueue), - erlNodeRunTime = truncate_int('Counter64', RunTime), - erlNodeWallClock = truncate_int('Counter64', WallClock), - erlNodeReductions = truncate_int('Counter64', Reductions), - erlNodeProcesses = truncate_int('Unsigned32', Processes), - erlNodeInBytes = truncate_int('Counter64', InBytes), - erlNodeOutBytes = truncate_int('Counter64', OutBytes)}. - -get_appls(NodeId) -> - element(1, - lists:mapfoldl( - fun({ApplName, ApplDescr, ApplVsn}, ApplId) -> - {#applTable{key = {NodeId, ApplId}, - applName = atom_to_list(ApplName), - applDescr = ApplDescr, - applVsn = ApplVsn}, - ApplId + 1} - end, 1, application:which_applications())). - -%%%======================================================================== -%%% Internal functions -%%%======================================================================== -update_node_alloc([Node | T]) -> - case mnesia:dirty_read({erlNodeAlloc, Node}) of - [] -> - [{_, _, Idx}] = mnesia:dirty_read({erlNodeAlloc, next_index}), - ok = mnesia:dirty_write(#erlNodeAlloc{nodeName = Node, - nodeId = Idx}), - ok = mnesia:dirty_write({erlNodeAlloc, next_index, Idx + 1}); - _ -> - ok - end, - update_node_alloc(T); -update_node_alloc([]) -> ok. - -delete_all(Name) -> delete_all(mnesia:dirty_first(Name), Name). -delete_all('$end_of_table', _Name) -> done; -delete_all(Key, Name) -> - Next = mnesia:dirty_next(Name, Key), - ok = mnesia:dirty_delete({Name, Key}), - delete_all(Next, Name). - -%% This will return a value limited to fit into the specified type. -%% While counter types will be resetted, other integer types will -%% only be restricted to the valid range. -truncate_int('Counter64', Value) when Value < 0 -> 0; -truncate_int('Counter64', Value) -> Value rem 18446744073709551615; -truncate_int('Unsigned32', Value) when Value < 0 -> 0; -truncate_int('Unsigned32', Value) when Value > 4294967295 -> 4294967295; -truncate_int('Unsigned32', Value) -> Value; -truncate_int('Integer32', Value) when Value < -2147483648 -> -2147483648; -truncate_int('Integer32', Value) when Value > 2147483647 -> 2147483647; -truncate_int('Integer32', Value) -> Value. diff --git a/lib/otp_mibs/src/otp_mibs.app.src b/lib/otp_mibs/src/otp_mibs.app.src deleted file mode 100644 index 75ef25c366..0000000000 --- a/lib/otp_mibs/src/otp_mibs.app.src +++ /dev/null @@ -1,30 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2003-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - -{application, otp_mibs, - [{description, "SNMP managment information base for Erlang/OTP nodes."}, - {vsn, "%VSN%"}, - {modules, [otp_mib]}, - {registered, []}, - {applications, [kernel, stdlib, snmp]}, - {env,[]}, - {runtime_dependencies, ["stdlib-2.0","snmp-4.25.1","mnesia-4.12", - "kernel-3.0","erts-6.0"]}]}. - diff --git a/lib/otp_mibs/src/otp_mibs.appup.src b/lib/otp_mibs/src/otp_mibs.appup.src deleted file mode 100644 index 9437ae2222..0000000000 --- a/lib/otp_mibs/src/otp_mibs.appup.src +++ /dev/null @@ -1,22 +0,0 @@ -%% -*- erlang -*- -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2003-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -{"%VSN%", - [{<<".*">>,[{restart_application, otp_mibs}]}], - [{<<".*">>,[{restart_application, otp_mibs}]}] -}. diff --git a/lib/otp_mibs/test/Makefile b/lib/otp_mibs/test/Makefile deleted file mode 100644 index 9736cf8bce..0000000000 --- a/lib/otp_mibs/test/Makefile +++ /dev/null @@ -1,85 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 1997-2016. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# %CopyrightEnd% -# -include $(ERL_TOP)/make/target.mk - -include $(ERL_TOP)/make/$(TARGET)/otp.mk - -# ---------------------------------------------------- -# Target Specs -# ---------------------------------------------------- - -MODULES= otp_mibs_SUITE - -EBIN = . - -HRL_FILES= - -ERL_FILES= $(MODULES:%=%.erl) - -TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) - -SOURCE = $(ERL_FILES) $(HRL_FILES) - -EMAKEFILE=Emakefile - -# ---------------------------------------------------- -# Release directory specification -# ---------------------------------------------------- -RELSYSDIR = $(RELEASE_PATH)/otp_mibs_test - -# ---------------------------------------------------- -# FLAGS -# ---------------------------------------------------- -ERL_MAKE_FLAGS += -ERL_COMPILE_FLAGS += \ - -I$(ERL_TOP)/lib/snmp/include - -# ---------------------------------------------------- -# Targets -# ---------------------------------------------------- - -make_emakefile: - $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES)\ - > $(EMAKEFILE) - -tests debug opt: make_emakefile - erl $(ERL_MAKE_FLAGS) -make - -clean: - rm -f $(EMAKEFILE) - rm -f $(TARGET_FILES) - rm -f core *~ - -docs: - - -# ---------------------------------------------------- -# Release Target -# ---------------------------------------------------- -include $(ERL_TOP)/make/otp_release_targets.mk - -release_spec: - -release_tests_spec: make_emakefile - $(INSTALL_DIR) "$(RELSYSDIR)" - $(INSTALL_DATA) $(EMAKEFILE) $(SOURCE) "$(RELSYSDIR)" - $(INSTALL_DATA) otp_mibs_SUITE.cfg "$(RELSYSDIR)" - -release_docs_spec: diff --git a/lib/otp_mibs/test/otp_mibs_SUITE.cfg b/lib/otp_mibs/test/otp_mibs_SUITE.cfg deleted file mode 100644 index d01cf92104..0000000000 --- a/lib/otp_mibs/test/otp_mibs_SUITE.cfg +++ /dev/null @@ -1,15 +0,0 @@ -%% -*- erlang -*- -{snmp, - [ - {start_agent,true}, - {users, - [ - {otp_mibs_test,[snmpm_user_default,[]]} - ]}, - {managed_agents, - [ - {otp_mibs_test, [otp_mibs_test, {127,0,0,1}, 4000, []]} - ]}, - {agent_sysname,"Test otp_mibs"}, - {mgr_port,5001} - ]}. diff --git a/lib/otp_mibs/test/otp_mibs_SUITE.erl b/lib/otp_mibs/test/otp_mibs_SUITE.erl deleted file mode 100644 index cb3cd28200..0000000000 --- a/lib/otp_mibs/test/otp_mibs_SUITE.erl +++ /dev/null @@ -1,255 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% --module(otp_mibs_SUITE). - -%%----------------------------------------------------------------- -%% This suite can no longer be executed standalone, i.e. it must be -%% executed with common test. The reason is that ct_snmp is used -%% instead of the snmp application directly. The suite requires a -%% config file, otp_mibs_SUITE.cfg, found in the same directory as -%% the suite. -%% -%% Execute with: -%% > ct_run -suite otp_mibs_SUITE -config otp_mibs_SUITE.cfg -%%----------------------------------------------------------------- - --include_lib("common_test/include/ct.hrl"). --include_lib("otp_mibs/include/OTP-MIB.hrl"). --include_lib("snmp/include/snmp_types.hrl"). - -% Test server specific exports --export([all/0, - suite/0, - groups/0, - init_per_group/2, - end_per_group/2, - init_per_suite/1, - end_per_suite/1, - init_per_testcase/2, - end_per_testcase/2]). - -% Test cases must be exported. --export([app/1, appup/1, nt_basic_types/1, nt_high_reduction_count/1]). - --define(TRAP_UDP, 5000). --define(AGENT_UDP, 4000). --define(CONF_FILE_VER, [v2]). --define(SYS_NAME, "Test otp_mibs"). --define(MAX_MSG_SIZE, 484). --define(ENGINE_ID, "mgrEngine"). --define(MGR_PORT, 5001). - -%% Since some cases are only interested in single entries of the OTP-MIB's -%% node table, one row must be chosen. The first row should be sufficient -%% for this. --define(NODE_ENTRY, 1). - -%%--------------------------------------------------------------------- -%% CT setup -%%--------------------------------------------------------------------- - -init_per_testcase(_Case, Config) when is_list(Config) -> - Dog = test_server:timetrap(test_server:minutes(6)), - [{watchdog, Dog}|Config]. - -end_per_testcase(_Case, Config) when is_list(Config) -> - Dog = ?config(watchdog, Config), - test_server:timetrap_cancel(Dog), - Config. - -suite() -> [{ct_hooks,[ts_install_cth]}, {require, snmp_mgr_agent, snmp}]. - -all() -> [{group, app}, {group, node_table}]. - -groups() -> [{app, [], [app, appup]}, - {node_table, [], [nt_basic_types, nt_high_reduction_count]}]. - -init_per_group(_GroupName, Config) -> Config. - -end_per_group(_GroupName, Config) -> Config. - -init_per_suite(Config) -> - ?line application:start(sasl), - ?line application:start(mnesia), - ?line application:start(otp_mibs), - - ok = ct_snmp:start(Config,snmp_mgr_agent), - - %% Load the mibs that should be tested - otp_mib:load(snmp_master_agent), - - Config. - -end_per_suite(Config) -> - PrivDir = ?config(priv_dir, Config), - ConfDir = filename:join(PrivDir,"conf"), - DbDir = filename:join(PrivDir,"db"), - MgrDir = filename:join(PrivDir, "mgr"), - - %% Uload mibs - otp_mib:unload(snmp_master_agent), - - %% Clean up - application:stop(snmp), - application:stop(mnesia), - application:stop(otp_mibs), - - del_dir(ConfDir), - del_dir(DbDir), - (catch del_dir(MgrDir)), - ok. - -%%--------------------------------------------------------------------- -%% Test cases -%%--------------------------------------------------------------------- - -%% Test that the otp_mibs app file is ok -app(Config) when is_list(Config) -> - ok = ?t:app_test(otp_mibs). - -%% Test that the otp_mibs appup file is ok -appup(Config) when is_list(Config) -> - ok = ?t:appup_test(otp_mibs). - -nt_basic_types(suite) -> - []; -nt_basic_types(doc) -> - ["Query every item of the node table and check its variable " - "type and content for sensible values."]; -nt_basic_types(Config) when is_list(Config) -> - ok = otp_mib:update_erl_node_table(), - - NodeNameId = ?erlNodeEntry ++ [?erlNodeName, ?NODE_ENTRY], - {noError, 0, [NodeNameVal]} = snmp_get([NodeNameId]), - #varbind{variabletype = 'OCTET STRING'} = NodeNameVal, - true = is_list(NodeNameVal#varbind.value), - - NodeMachineId = ?erlNodeEntry ++ [?erlNodeMachine, ?NODE_ENTRY], - {noError, 0, [NodeMachineVal]} = snmp_get([NodeMachineId]), - #varbind{variabletype = 'OCTET STRING'} = NodeMachineVal, - true = is_list(NodeMachineVal#varbind.value), - - NodeVersionId = ?erlNodeEntry ++ [?erlNodeVersion, ?NODE_ENTRY], - {noError, 0, [NodeVersionVal]} = snmp_get([NodeVersionId]), - #varbind{variabletype = 'OCTET STRING'} = NodeVersionVal, - true = is_list(NodeVersionVal#varbind.value), - - NodeRunQueueId = ?erlNodeEntry ++ [?erlNodeRunQueue, ?NODE_ENTRY], - {noError, 0, [NodeRunQueueVal]} = snmp_get([NodeRunQueueId]), - #varbind{variabletype = 'Unsigned32'} = NodeRunQueueVal, - true = is_integer(NodeRunQueueVal#varbind.value), - NodeRunQueueVal#varbind.value >= 0, - NodeRunQueueVal#varbind.value =< 4294967295, - - NodeRunTimeId = ?erlNodeEntry ++ [?erlNodeRunTime, ?NODE_ENTRY], - {noError, 0, [NodeRunTimeVal]} = snmp_get([NodeRunTimeId]), - #varbind{variabletype = 'Counter64'} = NodeRunTimeVal, - true = is_integer(NodeRunTimeVal#varbind.value), - NodeRunTimeVal#varbind.value >= 0, - NodeRunTimeVal#varbind.value =< 18446744073709551615, - - NodeWallClockId = ?erlNodeEntry ++ [?erlNodeWallClock, ?NODE_ENTRY], - {noError, 0, [NodeWallClockVal]} = snmp_get([NodeWallClockId]), - #varbind{variabletype = 'Counter64'} = NodeWallClockVal, - true = is_integer(NodeWallClockVal#varbind.value), - NodeWallClockVal#varbind.value >= 0, - NodeWallClockVal#varbind.value =< 18446744073709551615, - - NodeReductionsId = ?erlNodeEntry ++ [?erlNodeReductions, ?NODE_ENTRY], - {noError, 0, [NodeReductionsVal]} = snmp_get([NodeReductionsId]), - #varbind{variabletype = 'Counter64'} = NodeReductionsVal, - true = is_integer(NodeReductionsVal#varbind.value), - NodeReductionsVal#varbind.value >= 0, - NodeReductionsVal#varbind.value =< 18446744073709551615, - - NodeProcessesId = ?erlNodeEntry ++ [?erlNodeProcesses, ?NODE_ENTRY], - {noError, 0, [NodeProcessesVal]} = snmp_get([NodeProcessesId]), - #varbind{variabletype = 'Unsigned32'} = NodeProcessesVal, - true = is_integer(NodeProcessesVal#varbind.value), - NodeProcessesVal#varbind.value >= 0, - NodeProcessesVal#varbind.value =< 4294967295, - - NodeInBytesId = ?erlNodeEntry ++ [?erlNodeInBytes, ?NODE_ENTRY], - {noError, 0, [NodeInBytesVal]} = snmp_get([NodeInBytesId]), - #varbind{variabletype = 'Counter64'} = NodeInBytesVal, - true = is_integer(NodeInBytesVal#varbind.value), - NodeInBytesVal#varbind.value >= 0, - NodeInBytesVal#varbind.value =< 18446744073709551615, - - NodeOutBytesId = ?erlNodeEntry ++ [?erlNodeOutBytes, ?NODE_ENTRY], - {noError, 0, [NodeOutBytesVal]} = snmp_get([NodeOutBytesId]), - #varbind{variabletype = 'Counter64'} = NodeOutBytesVal, - true = is_integer(NodeOutBytesVal#varbind.value), - NodeOutBytesVal#varbind.value >= 0, - NodeOutBytesVal#varbind.value =< 18446744073709551615, - - ok. - -nt_high_reduction_count(suite) -> - []; -nt_high_reduction_count(doc) -> - ["Check that no error occurs when the erlNodeReductions field" - "exceeds the 32bit boundary, this may take about 10min."]; -nt_high_reduction_count(Config) when is_list(Config) -> - NodeReductions = ?erlNodeEntry ++ [?erlNodeReductions, ?NODE_ENTRY], - - BumpFun = fun(F, Limit) -> - case erlang:statistics(reductions) of - {Total, _} when Total < Limit -> - F(F, Limit); - _ -> - ok - end - end, - - ok = otp_mib:update_erl_node_table(), - - {noError, 0, [StartVal]} = snmp_get([NodeReductions]), - #varbind{variabletype = 'Counter64'} = StartVal, - true = is_integer(StartVal#varbind.value), - StartVal#varbind.value >= 0, - case StartVal#varbind.value =< 4294967295 of - true -> - ok = otp_mib:update_erl_node_table(), - BumpFun(BumpFun, 4294967295), - {noError, 0, [EndVal]} = snmp_get([NodeReductions]), - #varbind{variabletype = 'Counter64'} = EndVal, - true = is_integer(EndVal#varbind.value), - EndVal#varbind.value >= 4294967295, - EndVal#varbind.value =< 18446744073709551615; - false -> - %% no need to bump more reductions, since the initial get - %% command already returned successfully with a large value - ok - end. - -%%--------------------------------------------------------------------- -%% Internal functions -%%--------------------------------------------------------------------- - -snmp_get(OIdList) -> - ct_snmp:get_values(otp_mibs_test, OIdList, snmp_mgr_agent). - -del_dir(Dir) -> - io:format("Deleting: ~s~n",[Dir]), - {ok, Files} = file:list_dir(Dir), - FullPathFiles = lists:map(fun(File) -> filename:join(Dir, File) end, Files), - lists:foreach(fun file:delete/1, FullPathFiles), - file:del_dir(Dir). diff --git a/lib/otp_mibs/vsn.mk b/lib/otp_mibs/vsn.mk deleted file mode 100644 index 1b0444afcd..0000000000 --- a/lib/otp_mibs/vsn.mk +++ /dev/null @@ -1,5 +0,0 @@ -OTP_MIBS_VSN = 1.2.1 - -# Note: The branch 'otp_mibs' is defunct as of otp_mibs-1.0.4 and -# should NOT be used again. - diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl index 3f5d9fee3e..ad8fb11beb 100644 --- a/lib/parsetools/test/leex_SUITE.erl +++ b/lib/parsetools/test/leex_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -124,10 +124,6 @@ file(Config) when is_list(Config) -> "Erlang code.\n">>, ?line ok = file:write_file(Filename, Mini), ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = - leex:file(Filename, [{scannerfile,"//"} | Ret]), - ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = - leex:file(Filename, [{includefile,"//"} | Ret]), - ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = leex:file(Filename, [{includefile,"/ /"} | Ret]), LeexPre = filename:join(Dir, "leexinc.hrl"), @@ -191,7 +187,6 @@ compile(Config) when is_list(Config) -> "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" "Erlang code.\n">>, ?line ok = file:write_file(Filename, Mini), - ?line error = leex:compile(Filename, "//", #options{}), ?line ok = leex:compile(Filename, Scannerfile, #options{}), file:delete(Scannerfile), file:delete(Filename), diff --git a/lib/public_key/doc/src/public_key.xml b/lib/public_key/doc/src/public_key.xml index b7589f6653..12bb0b21b0 100644 --- a/lib/public_key/doc/src/public_key.xml +++ b/lib/public_key/doc/src/public_key.xml @@ -286,7 +286,9 @@ entries as ASN.1 DER encoded entities.</fsummary> <desc> <p>Decodes PEM binary data and returns entries as ASN.1 DER encoded entities.</p> - <p>Example <c>{ok, PemBin} = file:read_file("cert.pem").</c></p> + <p>Example <c>{ok, PemBin} = file:read_file("cert.pem"). + PemEntries = public_key:pem_decode(PemBin). + </c></p> </desc> </func> @@ -423,7 +425,7 @@ <p>Available options:</p> <taglist> - <tag>{verify_fun, fun()}</tag> + <tag>{verify_fun, {fun(), InitialUserState::term()}</tag> <item> <p>The fun must be defined as:</p> @@ -541,7 +543,7 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, <tag>{undetermined_details, boolean()}</tag> <item> - <p>Defaults to false. When revocation status can not be + <p>Defaults to false. When revocation status cannot be determined, and this option is set to true, details of why no CRLs where accepted are included in the return value.</p> </item> @@ -736,7 +738,7 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, <note><p> Note that the generated certificates and keys does not provide a formally correct PKIX-trust-chain - and they can not be used to achieve real security. This function is provided for testing purposes only. + and they cannot be used to achieve real security. This function is provided for testing purposes only. </p></note> </desc> </func> @@ -813,7 +815,8 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, <p>The <c>{OtherRefId,term()}</c> is defined by the user and is passed to the <c>match_fun</c>, if defined. If the term in <c>OtherRefId</c> is a binary, it will be converted to a string. </p> - <p>The <c>ip</c> Reference ID takes an <seealso marker="inet:inet#type-ip_address">inet:ip_address()</seealso> + <p>The <c>ip</c> Reference ID takes an + <seealso marker="kernel:inet#type-ip_address">inet:ip_address()</seealso> or an ip address in string format (E.g "10.0.1.1" or "1234::5678:9012") as second element. </p> <p>The options are:</p> diff --git a/lib/public_key/src/pubkey_pbe.erl b/lib/public_key/src/pubkey_pbe.erl index 806f7c5b0f..e6bcedd1b1 100644 --- a/lib/public_key/src/pubkey_pbe.erl +++ b/lib/public_key/src/pubkey_pbe.erl @@ -42,15 +42,14 @@ encode(Data, Password, "DES-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_encrypt(des_cbc, Key, IV, pbe_pad(Data, KeyDevParams)); - encode(Data, Password, "DES-EDE3-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), <<Key1:8/binary, Key2:8/binary, Key3:8/binary>> = Key, crypto:block_encrypt(des3_cbc, [Key1, Key2, Key3], IV, pbe_pad(Data)); - encode(Data, Password, "RC2-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_encrypt(rc2_cbc, Key, IV, pbe_pad(Data, KeyDevParams)). + %%-------------------------------------------------------------------- -spec decode(binary(), string(), string(), term()) -> binary(). %% @@ -59,21 +58,20 @@ encode(Data, Password, "RC2-CBC" = Cipher, KeyDevParams) -> decode(Data, Password,"DES-CBC"= Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_decrypt(des_cbc, Key, IV, Data); - decode(Data, Password,"DES-EDE3-CBC" = Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), <<Key1:8/binary, Key2:8/binary, Key3:8/binary>> = Key, crypto:block_decrypt(des3_cbc, [Key1, Key2, Key3], IV, Data); - decode(Data, Password,"RC2-CBC"= Cipher, KeyDevParams) -> {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), crypto:block_decrypt(rc2_cbc, Key, IV, Data); +decode(Data, Password,"AES-128-CBC"= Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + crypto:block_decrypt(aes_cbc128, Key, IV, Data); +decode(Data, Password,"AES-256-CBC"= Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + crypto:block_decrypt(aes_cbc256, Key, IV, Data). -decode(Data, Password,"AES-128-CBC"= Cipher, IV) -> - %% PKCS5_SALT_LEN is 8 bytes - <<Salt:8/binary,_/binary>> = IV, - {Key, _} = password_to_key_and_iv(Password, Cipher, Salt), - crypto:block_decrypt(aes_cbc128, Key, IV, Data). %%-------------------------------------------------------------------- -spec pbdkdf1(string(), iodata(), integer(), atom()) -> binary(). @@ -131,13 +129,15 @@ password_to_key_and_iv(Password, _Cipher, {#'PBEParameter'{salt = Salt, <<Key:8/binary, IV:8/binary, _/binary>> = pbdkdf1(Password, Salt, Count, Hash), {Key, IV}; -password_to_key_and_iv(Password, Cipher, Salt) -> - KeyLen = derived_key_length(Cipher, undefined), +password_to_key_and_iv(Password, Cipher, KeyDevParams) -> + %% PKCS5_SALT_LEN is 8 bytes + <<Salt:8/binary,_/binary>> = KeyDevParams, + KeyLen = derived_key_length(Cipher, undefined), <<Key:KeyLen/binary, _/binary>> = pem_encrypt(<<>>, Password, Salt, ceiling(KeyLen div 16), <<>>, md5), %% Old PEM encryption does not use standard encryption method - %% pbdkdf1 and uses then salt as IV - {Key, Salt}. + %% pbdkdf1 + {Key, KeyDevParams}. pem_encrypt(_, _, _, 0, Acc, _) -> Acc; pem_encrypt(Prev, Password, Salt, Count, Acc, Hash) -> @@ -267,7 +267,9 @@ derived_key_length(Cipher,_) when (Cipher == ?'des-EDE3-CBC') or (Cipher == "DES-EDE3-CBC") -> 24; derived_key_length(Cipher,_) when (Cipher == "AES-128-CBC") -> - 16. + 16; +derived_key_length(Cipher,_) when (Cipher == "AES-256-CBC") -> + 32. cipher(#'PBES2-params_encryptionScheme'{algorithm = ?'desCBC'}) -> "DES-CBC"; diff --git a/lib/public_key/src/pubkey_pem.erl b/lib/public_key/src/pubkey_pem.erl index d7e5bc3ad8..0fd1453f7c 100644 --- a/lib/public_key/src/pubkey_pem.erl +++ b/lib/public_key/src/pubkey_pem.erl @@ -101,10 +101,10 @@ encode_pem_entry({'PrivateKeyInfo', Der, EncParams}) -> EncDer = encode_encrypted_private_keyinfo(Der, EncParams), StartStr = pem_start('EncryptedPrivateKeyInfo'), [StartStr, "\n", b64encode_and_split(EncDer), "\n", pem_end(StartStr) ,"\n\n"]; -encode_pem_entry({Type, Der, {Cipher, Salt}}) -> +encode_pem_entry({Type, Decrypted, {Cipher, Salt}}) -> StartStr = pem_start(Type), [StartStr,"\n", pem_decrypt(),"\n", pem_decrypt_info(Cipher, Salt),"\n\n", - b64encode_and_split(Der), "\n", pem_end(StartStr) ,"\n\n"]. + b64encode_and_split(Decrypted), "\n", pem_end(StartStr) ,"\n\n"]. decode_pem_entries([], Entries) -> lists:reverse(Entries); diff --git a/lib/public_key/src/public_key.erl b/lib/public_key/src/public_key.erl index d02df27a00..47266c514c 100644 --- a/lib/public_key/src/public_key.erl +++ b/lib/public_key/src/public_key.erl @@ -407,8 +407,7 @@ decrypt_private(CipherText, Options) when is_binary(CipherText), is_list(Options) -> - Padding = proplists:get_value(rsa_pad, Options, rsa_pkcs1_padding), - crypto:private_decrypt(rsa, CipherText, format_rsa_private_key(Key), Padding). + crypto:private_decrypt(rsa, CipherText, format_rsa_private_key(Key), default_options(Options)). %%-------------------------------------------------------------------- %% Description: Public key decryption using the public key. @@ -429,8 +428,7 @@ decrypt_public(CipherText, Key) -> PlainText :: binary() . decrypt_public(CipherText, #'RSAPublicKey'{modulus = N, publicExponent = E}, Options) when is_binary(CipherText), is_list(Options) -> - Padding = proplists:get_value(rsa_pad, Options, rsa_pkcs1_padding), - crypto:public_decrypt(rsa, CipherText,[E, N], Padding). + crypto:public_decrypt(rsa, CipherText,[E, N], default_options(Options)). %%-------------------------------------------------------------------- %% Description: Public key encryption using the public key. @@ -452,8 +450,7 @@ encrypt_public(PlainText, Key) -> CipherText :: binary() . encrypt_public(PlainText, #'RSAPublicKey'{modulus=N,publicExponent=E}, Options) when is_binary(PlainText), is_list(Options) -> - Padding = proplists:get_value(rsa_pad, Options, rsa_pkcs1_padding), - crypto:public_encrypt(rsa, PlainText, [E,N], Padding). + crypto:public_encrypt(rsa, PlainText, [E,N], default_options(Options)). %%-------------------------------------------------------------------- %% @@ -481,8 +478,7 @@ encrypt_private(PlainText, when is_binary(PlainText), is_integer(N), is_integer(E), is_integer(D), is_list(Options) -> - Padding = proplists:get_value(rsa_pad, Options, rsa_pkcs1_padding), - crypto:private_encrypt(rsa, PlainText, format_rsa_private_key(Key), Padding). + crypto:private_encrypt(rsa, PlainText, format_rsa_private_key(Key), default_options(Options)). %%-------------------------------------------------------------------- %% Description: List available group sizes among the pre-computed dh groups @@ -1235,6 +1231,33 @@ pkix_test_root_cert(Name, Opts) -> %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- +default_options([]) -> + [{rsa_padding, rsa_pkcs1_padding}]; +default_options(Opts) -> + case proplists:get_value(rsa_pad, Opts) of + undefined -> + case proplists:get_value(rsa_padding, Opts) of + undefined -> + case lists:dropwhile(fun erlang:is_tuple/1, Opts) of + [Pad|_] -> + set_padding(Pad, Opts); + [] -> + set_padding(rsa_pkcs1_padding, Opts) + end; + Pad -> + set_padding(Pad, Opts) + end; + Pad -> + set_padding(Pad, Opts) + end. + +set_padding(Pad, Opts) -> + [{rsa_padding,Pad} | [{T,V} || {T,V} <- Opts, + T =/= rsa_padding, + T =/= rsa_pad] + ]. + + format_sign_key(Key = #'RSAPrivateKey'{}) -> {rsa, format_rsa_private_key(Key)}; format_sign_key(#'DSAPrivateKey'{p = P, q = Q, g = G, x = X}) -> diff --git a/lib/public_key/test/pbe_SUITE.erl b/lib/public_key/test/pbe_SUITE.erl index 523c9e2515..1136267411 100644 --- a/lib/public_key/test/pbe_SUITE.erl +++ b/lib/public_key/test/pbe_SUITE.erl @@ -37,7 +37,7 @@ all() -> [ pbdkdf1, pbdkdf2, - old_enc, + old_pbe, pbes1, pbes2]. @@ -197,23 +197,11 @@ pbdkdf2(Config) when is_list(Config) -> = pubkey_pbe:pbdkdf2("pass\0word", "sa\0lt", 4096, 16, fun crypto:hmac/4, sha, 20). -old_enc() -> - [{doc,"Tests encode/decode RSA key encrypted with different ciphers using old PEM encryption scheme"}]. -old_enc(Config) when is_list(Config) -> - Datadir = proplists:get_value(data_dir, Config), - %% key generated with ssh-keygen -N hello_aes -f old_aes_128_cbc_enc_key.pem - {ok, PemAesCbc} = file:read_file(filename:join(Datadir, "old_aes_128_cbc_enc_key.pem")), - - PemAesCbcEntry = public_key:pem_decode(PemAesCbc), - ct:print("Pem entry: ~p" , [PemAesCbcEntry]), - [{'RSAPrivateKey', _, {"AES-128-CBC",_}} = PubAesCbcEntry] = PemAesCbcEntry, - #'RSAPrivateKey'{} = public_key:pem_entry_decode(PubAesCbcEntry, "hello_aes"). - pbes1() -> [{doc,"Tests encode/decode EncryptedPrivateKeyInfo encrypted with different ciphers using PBES1"}]. pbes1(Config) when is_list(Config) -> decode_encode_key_file("pbes1_des_cbc_md5_enc_key.pem", "password", "DES-CBC", Config). - + pbes2() -> [{doc,"Tests encode/decode EncryptedPrivateKeyInfo encrypted with different ciphers using PBES2"}]. pbes2(Config) when is_list(Config) -> @@ -225,13 +213,33 @@ pbes2(Config) when is_list(Config) -> false -> ok end. +old_pbe() -> + [{doc,"Tests encode/decode with old format used before PBE"}]. +old_pbe(Config) when is_list(Config) -> + Datadir = proplists:get_value(data_dir, Config), + % key generated with ssh-keygen -N hello_aes -f old_aes_128_cbc.pem + {ok, PemAes128Cbc} = file:read_file(filename:join(Datadir, "old_aes_128_cbc.pem")), + + PemAes128CbcEntries = public_key:pem_decode(PemAes128Cbc), + ct:print("Pem entry: ~p" , [PemAes128CbcEntries]), + [{'RSAPrivateKey', _, {"AES-128-CBC",_}} = Aes128CbcEntry] = PemAes128CbcEntries, + #'RSAPrivateKey'{} = Key = public_key:pem_entry_decode(Aes128CbcEntry, "hello_aes"), + + %% Converted with openssl rsa -in old_aes_128_cbc.pem -out old_aes_256_cbc.pem -aes256 + {ok, PemAes256Cbc} = file:read_file(filename:join(Datadir, "old_aes_256_cbc.pem")), + + PemAes256CbcEntries = public_key:pem_decode(PemAes256Cbc), + ct:print("Pem entry: ~p" , [PemAes256CbcEntries]), + [{'RSAPrivateKey', _, {"AES-256-CBC",_}} = Aes256CbcEntry] = PemAes256CbcEntries, + Key = public_key:pem_entry_decode(Aes256CbcEntry, "hello_aes"). + decode_encode_key_file(File, Password, Cipher, Config) -> Datadir = proplists:get_value(data_dir, Config), {ok, PemKey} = file:read_file(filename:join(Datadir, File)), PemEntry = public_key:pem_decode(PemKey), - ct:print("Pem entry: ~p" , [PemEntry]), + ct:pal("Pem entry: ~p" , [PemEntry]), [{Asn1Type, _, {Cipher,_} = CipherInfo} = PubEntry] = PemEntry, #'RSAPrivateKey'{} = KeyInfo = public_key:pem_entry_decode(PubEntry, Password), PemKey1 = public_key:pem_encode([public_key:pem_entry_encode(Asn1Type, KeyInfo, {CipherInfo, Password})]), diff --git a/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc_enc_key.pem b/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc.pem index 34c7543f30..34c7543f30 100644 --- a/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc_enc_key.pem +++ b/lib/public_key/test/pbe_SUITE_data/old_aes_128_cbc.pem diff --git a/lib/public_key/test/pbe_SUITE_data/old_aes_256_cbc.pem b/lib/public_key/test/pbe_SUITE_data/old_aes_256_cbc.pem new file mode 100644 index 0000000000..e6aec2869d --- /dev/null +++ b/lib/public_key/test/pbe_SUITE_data/old_aes_256_cbc.pem @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,ABDA22398E511E9465983E1A50706044 + +XhIcPOb6pTWL++pgeeTH5rsx0tackhllVqyXyOfbYMBJnVFRhQ/V/1MDg3Jt4wD1 +Nerhcv5srHeiwmf+vwXwDFOzvFzLVM1jFMUJe/2XloYFX4TBiLZAF/zekQA3uPY6 +DKJuBuO5vVSZ0VlxGpu3jphIAwxbssfZkZmryCP3b1/oX5Y3Em/wEWW3RduaeWFu +Z2nTsPH3yNmHkuqFF3cq0aZs+VxtjcuYo0gbkN5hNVgOoOVxIBzw7DsgBAkdXvr3 +LRCMGg7Y2pVthA963s0xkN37XtZEiYbydoLnzHlW/Kx5QSvED8/Fn94Lcdy5NsQN +7nYWWgYZRH39Wnsi0BrTZv399U5rBe7DnpStKWPn2Sa8Bdu3CX2oLajM1cZjAp1X +y6vuasK7SoZ8rWpcsHQpV1HyNBTl/uRU5nrYc/KGD8l2x7tdNidGx2Hey/6O4H5v +rxL1TB4PlzDYwCsReuyLbEjyuZQ10j0SK/SFzikHuvD2IEpM/oUaPdVFqcW8/sjw +VhcsupAf4EXfsi2dJBylmDjI2W7h9XwBDLzQN+69DtkBmvimE5CguTITf8MAHQQ6 +Q1vYF2ij7W675tj9ulksRPaMxSsI03luai/Jrieh0mPqGEenGEEC5QU0XPOvsfyw +GMYaBUbdYrMpbHM3wFPxM2wRlXf4gX5BhZKRGZX7OaEs54pfglyQtTPuZmD0VcAp +EWHq70G9mbuBlhbMX1rKAomuDmIvgLeLRUpAFf59Qr8KoJSLD1S8TJB3uEPk6i39 +4GnylbpmqS4gv/OIc6WTeOeUZTAD3A77HBwSlELPk3/s1d/MLyfciYClOBEuZ6NB +FXEKCGCEC786zJA678gLEaa2XPNkEM+2gjzNFqtYMIn5ehAq/HRRsFvW+wkTbee4 +z+qe5HbVKAQ3EOfbidvYrDaGd7HvHVG8zosl+O61iIFs04lLEMDFXBIdvIgEncOK +Rq3yXdpBKMg89aoZLniaPobSvuvdjNOMzW6EKlb5FKZduCiR68MEZ+rLHYHTwE3W +Z5+TCbrbV2F6WQpq3zqnB14wGu8igEb5Veq+N2vMkx4iTMTUyCty1SwIjj4NidM1 +dJM7Ighdal6tQ6hIwbDfpIPsY4eGH/UrdVZ0SkxuDR2s76cZ8nFX3lJ/BNwTZLKo +IqAC4NjUOv3ID+0Q6Lz+sxLCi5pLYUf0E+s4pgi1BYAOu+BF3GwxyqnqVoq/Fs5D +LXxuY0946YM+WcrYzke4mq3MPx6QQYj04H5KJ2mzxtnbZJrfLF23PVRVhvgKSjyV +I3/zgJ16fV2H/fb26oCpTNbb11pQvhorkLwdvpwtM+go7dJGebAi1762Nbj/CqnW +fbBPxPRvNPZn6pEodJ/L/APhvGv1K7eC9THj66H7Kmeoq8Lz74idhywP9I3QS0ZO +15ORbTDjuiRYNJPxxu79A3/tWMUlprJ9ljhI/0DXRB0M3UGic52D/32Q64I7eewy +qRNS/3C3ejDShIRBDFTdDkM3s/42LySXJjmjU9bpZY4POQ3kOaJb3EzSvbzTyXzu +3FiHvDQY+b8XwbxtE/kTMaAPQZ7TtWOao7SRi7J94MvCQ5/tbakFP2suM8psnigC +-----END RSA PRIVATE KEY----- diff --git a/lib/reltool/doc/src/reltool.xml b/lib/reltool/doc/src/reltool.xml index 5afbad0ba8..136ec2ed69 100644 --- a/lib/reltool/doc/src/reltool.xml +++ b/lib/reltool/doc/src/reltool.xml @@ -503,6 +503,7 @@ sys() = {root_dir, root_dir()} | {incl_cond, incl_cond()} | {boot_rel, boot_rel()} | {rel, rel_name(), rel_vsn(), [rel_app()]} + | {rel, rel_name(), rel_vsn(), [rel_app()], [rel_opt()]} | {relocatable, relocatable()} | {app_file, app_file()} | {debug_info, debug_info()} @@ -534,6 +535,7 @@ rel_app() = app_name() | {app_name(), app_type()} | {app_name(), [incl_app()]} | {app_name(), app_type(), [incl_app()]} +rel_opt() = {load_dot_erlang, boolean()} app_name() = atom() app_type() = permanent | transient | temporary | load | none app_vsn() = string() diff --git a/lib/reltool/doc/src/reltool_examples.xml b/lib/reltool/doc/src/reltool_examples.xml index 2a103119e6..3888b643a2 100644 --- a/lib/reltool/doc/src/reltool_examples.xml +++ b/lib/reltool/doc/src/reltool_examples.xml @@ -160,7 +160,7 @@ Eshell V9.0 (abort with ^G) {mod,erts_internal,[]}, {mod,erts_literal_area_collector,[]}, {mod,init,[]}, - {mod,otp_ring0,...}, + {mod,erl_init,...}, {mod,...}, {...}|...]}]}, {app,compiler, @@ -313,9 +313,12 @@ Erlang/OTP 20 [erts-10.0] [source-c13b302] [64-bit] [smp:4:4] [ds:4:4:10] [async [hipe] [kernel-poll:false] Eshell V10.0 (abort with ^G) 1> -1> {ok, Server} = reltool:start_server([{config, {sys, [{boot_rel, "NAME"}, - {rel, "NAME", "VSN", - [sasl]}]}}]). +1> {ok, Server} = reltool:start_server([{config, + {sys, + [{boot_rel, "NAME"}, + {rel, "NAME", "VSN", + [sasl], + [{load_dot_erlang, false}]}]}}]). {ok,<0.1288.0>} 2> 2> reltool:get_config(Server). @@ -331,7 +334,7 @@ Eshell V10.0 (abort with ^G) {ok,{script,{"NAME","VSN"}, [{preLoaded,[erl_prim_loader,erl_tracer,erlang, erts_code_purger,erts_dirty_process_signal_handler, - erts_internal,erts_literal_area_collector,init,otp_ring0, + erts_internal,erts_literal_area_collector,init,erl_init, prim_eval,prim_file,prim_inet,prim_zip,zlib]}, {progress,preloaded}, {path,["$ROOT/lib/kernel-5.2/ebin", diff --git a/lib/reltool/src/reltool.hrl b/lib/reltool/src/reltool.hrl index d133762818..892aaf8649 100644 --- a/lib/reltool/src/reltool.hrl +++ b/lib/reltool/src/reltool.hrl @@ -61,6 +61,7 @@ | {app_name(), app_type()} | {app_name(), [incl_app()]} | {app_name(), app_type(), [incl_app()]}. +-type rel_opt() :: {load_dot_erlang, boolean()}. -type mod() :: {incl_cond, incl_cond()} | {debug_info, debug_info()}. -type app() :: {vsn, app_vsn()} @@ -92,6 +93,8 @@ | {lib_dirs, [lib_dir()]} | {boot_rel, boot_rel()} | {rel, rel_name(), rel_vsn(), [rel_app()]} + | {rel, rel_name(), rel_vsn(), + [rel_app()], [rel_opt()]} | {relocatable, relocatable()} | {erts, app()} | {escript, escript_file(), [escript()]} diff --git a/lib/reltool/src/reltool_server.erl b/lib/reltool/src/reltool_server.erl index af71b0cf2a..2de8000fd8 100644 --- a/lib/reltool/src/reltool_server.erl +++ b/lib/reltool/src/reltool_server.erl @@ -526,7 +526,7 @@ analyse(#state{sys=Sys} = S, Apps, Status) -> %% is included in a release (rel spec - see apps_in_rels above). %% Then initiate the same for each module, and check that there %% are no duplicated module names (in different applications) - %% where we can not decide which one to use. + %% where we cannot decide which one to use. %% Write all #app to app_tab and all #mod to mod_tab. Status2 = apps_init_is_included(S, Apps, RelApps, Status), @@ -1483,6 +1483,18 @@ decode(#sys{rels = Rels} = Sys, [{rel, Name, Vsn, RelApps} | SysKeyVals]) Rel = #rel{name = Name, vsn = Vsn, rel_apps = []}, Rel2 = decode(Rel, RelApps), decode(Sys#sys{rels = [Rel2 | Rels]}, SysKeyVals); +decode(#sys{rels = Rels} = Sys, [{rel, Name, Vsn, RelApps, Opts} | SysKeyVals]) + when is_list(Name), is_list(Vsn), is_list(RelApps), is_list(Opts) -> + Rel1 = lists:foldl(fun(Opt, Rel0) -> + case Opt of + {load_dot_erlang, Value} when is_boolean(Value) -> + Rel0#rel{load_dot_erlang = Value}; + _ -> + reltool_utils:throw_error("Illegal rel option: ~tp", [Opt]) + end + end, #rel{name = Name, vsn = Vsn, rel_apps = []}, Opts), + Rel2 = decode(Rel1, RelApps), + decode(Sys#sys{rels = [Rel2 | Rels]}, SysKeyVals); decode(#sys{} = Sys, [{Key, Val} | KeyVals]) -> Sys3 = case Key of diff --git a/lib/reltool/src/reltool_target.erl b/lib/reltool/src/reltool_target.erl index 64834ecc1d..dfa62479a0 100644 --- a/lib/reltool/src/reltool_target.erl +++ b/lib/reltool/src/reltool_target.erl @@ -108,12 +108,8 @@ do_gen_config(#sys{root_dir = RootDir, emit(incl_cond, A#app.incl_cond, undefined, InclDefs)} || A <- Apps, A#app.is_escript], DefaultRels = reltool_utils:default_rels(), - RelsItems = - [{rel, R#rel.name, R#rel.vsn, do_gen_config(R, InclDefs)} || - R <- Rels], - DefaultRelsItems = - [{rel, R#rel.name, R#rel.vsn, do_gen_config(R, InclDefs)} || - R <- DefaultRels], + RelsItems = [do_gen_config(R, InclDefs) || R <- Rels], + DefaultRelsItems = [do_gen_config(R, InclDefs) || R <- DefaultRels], RelsItems2 = case InclDefs of true -> RelsItems; @@ -201,11 +197,20 @@ do_gen_config(#mod{name = Name, _ -> [] end; -do_gen_config(#rel{name = _Name, - vsn = _Vsn, - rel_apps = RelApps}, - InclDefs) -> - [do_gen_config(RA, InclDefs) || RA <- RelApps]; +do_gen_config(#rel{name = Name, + vsn = Vsn, + rel_apps = RelApps, + load_dot_erlang = LoadDotErlang}, + InclDefs) -> + RelAppsConfig = [do_gen_config(RA, InclDefs) || RA <- RelApps], + if + LoadDotErlang =:= false -> + {rel, Name, Vsn, RelAppsConfig, [{load_dot_erlang, false}]}; + InclDefs =:= true -> + {rel, Name, Vsn, RelAppsConfig, [{load_dot_erlang, true}]}; + LoadDotErlang =:= true -> + {rel, Name, Vsn, RelAppsConfig} + end; do_gen_config(#rel_app{name = Name, app_type = Type, incl_apps = InclApps}, diff --git a/lib/reltool/test/reltool_server_SUITE.erl b/lib/reltool/test/reltool_server_SUITE.erl index 9ee61b595d..bb092e8bbf 100644 --- a/lib/reltool/test/reltool_server_SUITE.erl +++ b/lib/reltool/test/reltool_server_SUITE.erl @@ -102,6 +102,7 @@ all() -> create_release, create_release_sort, create_script, + create_script_without_dot_erlang, create_script_sort, create_target, create_target_unicode, @@ -171,7 +172,7 @@ break(_Config) -> start_server(_Config) -> {ok, Pid} = ?msym({ok, _}, reltool:start_server([])), - Libs = lists:sort(erl_libs()), + Libs = reltool_test_lib:erl_libs(), StrippedDefault = case Libs of [] -> {sys, []}; @@ -185,7 +186,7 @@ start_server(_Config) -> %% Start a server process and check that it does not crash set_config(_Config) -> - Libs = lists:sort(erl_libs()), + Libs = reltool_test_lib:erl_libs(), Default = {sys, [ @@ -219,7 +220,15 @@ get_config(_Config) -> StdLibDir = filename:join(LibDir,"stdlib-"++StdVsn), SaslLibDir = filename:join(LibDir,"sasl-"++SaslVsn), - Sys = {sys,[{incl_cond, exclude}, + Libs = reltool_test_lib:erl_libs(), + LibDirs = + case Libs of + [] -> []; + _ -> [{lib_dirs,Libs}] + end, + + Sys = {sys,LibDirs ++ + [{incl_cond, exclude}, {app,kernel,[{incl_cond,include}]}, {app,sasl,[{incl_cond,include},{vsn,SaslVsn}]}, {app,stdlib,[{incl_cond,include},{lib_dir,StdLibDir}]}]}, @@ -228,13 +237,27 @@ get_config(_Config) -> ?m({ok, Sys}, reltool:get_config(Pid,false,false)), %% Include derived info - ?msym({ok,{sys,[{incl_cond, exclude}, - {erts,[]}, - {app,kernel,[{incl_cond,include},{mod,_,[]}|_]}, - {app,sasl,[{incl_cond,include},{vsn,SaslVsn},{mod,_,[]}|_]}, - {app,stdlib,[{incl_cond,include},{lib_dir,StdLibDir}, - {mod,_,[]}|_]}]}}, - reltool:get_config(Pid,false,true)), + case Libs of + [] -> + ?msym({ok,{sys,[{incl_cond, exclude}, + {erts,[]}, + {app,kernel,[{incl_cond,include},{mod,_,[]}|_]}, + {app,sasl,[{incl_cond,include},{vsn,SaslVsn}, + {mod,_,[]}|_]}, + {app,stdlib,[{incl_cond,include},{lib_dir,StdLibDir}, + {mod,_,[]}|_]}]}}, + reltool:get_config(Pid,false,true)); + _ -> + ?msym({ok,{sys,[{lib_dirs,Libs}, + {incl_cond, exclude}, + {erts,[]}, + {app,kernel,[{incl_cond,include},{mod,_,[]}|_]}, + {app,sasl,[{incl_cond,include},{vsn,SaslVsn}, + {mod,_,[]}|_]}, + {app,stdlib,[{incl_cond,include},{lib_dir,StdLibDir}, + {mod,_,[]}|_]}]}}, + reltool:get_config(Pid,false,true)) + end, %% Include defaults ?msym({ok,{sys,[{root_dir,_}, @@ -248,9 +271,9 @@ get_config(_Config) -> {app,stdlib,[{incl_cond,include},{vsn,undefined}, {lib_dir,StdLibDir}]}, {boot_rel,"start_clean"}, - {rel,"no_dot_erlang","1.0",[]}, - {rel,"start_clean","1.0",[]}, - {rel,"start_sasl","1.0",[sasl]}, + {rel,"no_dot_erlang","1.0",[],[{load_dot_erlang,false}]}, + {rel,"start_clean","1.0",[],[{load_dot_erlang,true}]}, + {rel,"start_sasl","1.0",[sasl],[{load_dot_erlang,true}]}, {emu_name,"beam"}, {relocatable,true}, {profile,development}, @@ -279,9 +302,9 @@ get_config(_Config) -> {app,stdlib,[{incl_cond,include},{vsn,StdVsn}, {lib_dir,StdLibDir},{mod,_,[]}|_]}, {boot_rel,"start_clean"}, - {rel,"no_dot_erlang","1.0",[]}, - {rel,"start_clean","1.0",[]}, - {rel,"start_sasl","1.0",[sasl]}, + {rel,"no_dot_erlang","1.0",[],[{load_dot_erlang,false}]}, + {rel,"start_clean","1.0",[],[{load_dot_erlang,true}]}, + {rel,"start_sasl","1.0",[sasl],[{load_dot_erlang,true}]}, {emu_name,"beam"}, {relocatable,true}, {profile,development}, @@ -305,11 +328,11 @@ get_config(_Config) -> %% OTP-9135, test that app_file option can be set to all | keep | strip otp_9135(_Config) -> - Libs = lists:sort(erl_libs()), + Libs = reltool_test_lib:erl_libs(), StrippedDefaultSys = case Libs of [] -> []; - _ -> {lib_dirs, Libs} + _ -> [{lib_dirs, Libs}] end, Config1 = {sys,[{app_file, keep}]}, % this is the default @@ -549,6 +572,32 @@ create_script(_Config) -> ?m(equal, diff_script(OrigScript, Script)), + %% A release defaults to load_dot_erlang == true + {script, {RelName, RelVsn}, ScriptInstructions} = Script, + ?m(true, lists:member({apply,{c,erlangrc,[]}}, ScriptInstructions)), + + %% Stop server + ?m(ok, reltool:stop(Pid)), + ok. + +create_script_without_dot_erlang(_Config) -> + %% Configure the server + RelName = "Just testing", + RelVsn = "1.0", + Config = + {sys, + [ + {lib_dirs, []}, + {boot_rel, RelName}, + {rel, RelName, RelVsn, [stdlib, kernel], [{load_dot_erlang, false}]} + ]}, + {ok, Pid} = ?msym({ok, _}, reltool:start_server([{config, Config}])), + + %% Confirm that load_dot_erlang == false was used + {ok, Script} = ?msym({ok, _}, reltool:get_script(Pid, RelName)), + {script, {RelName, RelVsn}, ScriptInstructions} = Script, + ?m(false, lists:member({apply,{c,erlangrc,[]}}, ScriptInstructions)), + %% Stop server ?m(ok, reltool:stop(Pid)), ok. @@ -1039,7 +1088,7 @@ create_standalone_app(Config) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Generate standalone system with inlined archived application -%% Check that the inlined app can not be explicitly configured +%% Check that the inlined app cannot be explicitly configured create_standalone_app_clash(Config) -> %% Create archive @@ -1659,7 +1708,7 @@ set_apps_inlined(Config) -> ?m(true, Someapp1#app.is_included), ?m(true, Someapp1#app.is_pre_included), - %% Check that inlined app can not be configured + %% Check that inlined app cannot be configured Someapp2 = Someapp1#app{incl_cond=exclude}, ?msym({error, "Application someapp is inlined in '*escript* someapp'. " @@ -1719,13 +1768,19 @@ set_sys_and_undo(Config) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% load_config_and_undo(Config) -> - Sys1 = {sys,[{incl_cond, exclude}, - {app,kernel,[{incl_cond,include}]}, - {app,sasl,[{incl_cond,include}]}, - {app,stdlib,[{incl_cond,include}]}, - {app,tools,[{incl_cond,include}]}]}, + Sys1 = {sys,Cfg1=[{incl_cond, exclude}, + {app,kernel,[{incl_cond,include}]}, + {app,sasl,[{incl_cond,include}]}, + {app,stdlib,[{incl_cond,include}]}, + {app,tools,[{incl_cond,include}]}]}, {ok, Pid} = ?msym({ok, _}, reltool:start_server([{config, Sys1}])), - ?m({ok, Sys1}, reltool:get_config(Pid)), + Libs = reltool_test_lib:erl_libs(), + Sys11 = + case Libs of + [] -> Sys1; + _ -> {sys, [{lib_dirs, Libs}|Cfg1]} + end, + ?m({ok, Sys11}, reltool:get_config(Pid)), ?m({ok,[]}, reltool_server:get_status(Pid)), %% Get app and mod @@ -1780,13 +1835,19 @@ load_config_and_undo(Config) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Test that load_config is properly rolled back if it fails load_config_fail(_Config) -> - Sys1 = {sys,[{incl_cond, exclude}, - {app,kernel,[{incl_cond,include}]}, - {app,sasl,[{incl_cond,include}]}, - {app,stdlib,[{incl_cond,include}]}, - {app,tools,[{incl_cond,include}]}]}, + Sys1 = {sys,Cfg1=[{incl_cond, exclude}, + {app,kernel,[{incl_cond,include}]}, + {app,sasl,[{incl_cond,include}]}, + {app,stdlib,[{incl_cond,include}]}, + {app,tools,[{incl_cond,include}]}]}, {ok, Pid} = ?msym({ok, _}, reltool:start_server([{config, Sys1}])), - ?m({ok, Sys1}, reltool:get_config(Pid)), + Libs = reltool_test_lib:erl_libs(), + Sys11 = + case Libs of + [] -> Sys1; + _ -> {sys, [{lib_dirs, Libs}|Cfg1]} + end, + ?m({ok, Sys11}, reltool:get_config(Pid)), ?m({ok,[]}, reltool_server:get_status(Pid)), %% Get app and mod @@ -1804,7 +1865,7 @@ load_config_fail(_Config) -> reltool_server:load_config(Pid,Sys2)), %% Check that a rollback is done to the old configuration - ?m({ok, Sys1}, reltool:get_config(Pid,false,false)), + ?m({ok, Sys11}, reltool:get_config(Pid,false,false)), %% and that tools is not changed (i.e. that the new configuration %% is not applied) @@ -2074,25 +2135,42 @@ gen_rel_files(_Config) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% save_config(Config) -> PrivDir = ?config(priv_dir,Config), - Sys = {sys,[{incl_cond, exclude}, - {app,kernel,[{incl_cond,include}]}, - {app,sasl,[{incl_cond,include}]}, - {app,stdlib,[{incl_cond,include}]}]}, + Sys = {sys,Cfg=[{incl_cond, exclude}, + {app,kernel,[{incl_cond,include}]}, + {app,sasl,[{incl_cond,include}]}, + {app,stdlib,[{incl_cond,include}]}]}, {ok, Pid} = ?msym({ok, _}, reltool:start_server([{config, Sys}])), - ?m({ok, Sys}, reltool:get_config(Pid)), + Libs = reltool_test_lib:erl_libs(), + Sys1 = + case Libs of + [] -> Sys; + _ -> {sys, [{lib_dirs, Libs}|Cfg]} + end, + ?m({ok, Sys1}, reltool:get_config(Pid)), Simple = filename:join(PrivDir,"save_simple.reltool"), ?m(ok, reltool_server:save_config(Pid,Simple,false,false)), - ?m({ok,[Sys]}, file:consult(Simple)), + ?m({ok,[Sys1]}, file:consult(Simple)), Derivates = filename:join(PrivDir,"save_derivates.reltool"), ?m(ok, reltool_server:save_config(Pid,Derivates,false,true)), - ?msym({ok,[{sys,[{incl_cond, exclude}, - {erts,[]}, - {app,kernel,[{incl_cond,include},{mod,_,[]}|_]}, - {app,sasl,[{incl_cond,include},{mod,_,[]}|_]}, - {app,stdlib,[{incl_cond,include},{mod,_,[]}|_]}]}]}, - file:consult(Derivates)), + case Libs of + [] -> + ?msym({ok,[{sys,[{incl_cond, exclude}, + {erts,[]}, + {app,kernel,[{incl_cond,include},{mod,_,[]}|_]}, + {app,sasl,[{incl_cond,include},{mod,_,[]}|_]}, + {app,stdlib,[{incl_cond,include},{mod,_,[]}|_]}]}]}, + file:consult(Derivates)); + _ -> + ?msym({ok,[{sys,[{lib_dirs,Libs}, + {incl_cond, exclude}, + {erts,[]}, + {app,kernel,[{incl_cond,include},{mod,_,[]}|_]}, + {app,sasl,[{incl_cond,include},{mod,_,[]}|_]}, + {app,stdlib,[{incl_cond,include},{mod,_,[]}|_]}]}]}, + file:consult(Derivates)) + end, Defaults = filename:join(PrivDir,"save_defaults.reltool"), ?m(ok, reltool_server:save_config(Pid,Defaults,true,false)), @@ -2107,9 +2185,9 @@ save_config(Config) -> {app,stdlib,[{incl_cond,include},{vsn,undefined}, {lib_dir,undefined}]}, {boot_rel,"start_clean"}, - {rel,"no_dot_erlang","1.0",[]}, - {rel,"start_clean","1.0",[]}, - {rel,"start_sasl","1.0",[sasl]}, + {rel,"no_dot_erlang","1.0",[],[{load_dot_erlang,false}]}, + {rel,"start_clean","1.0",[],[{load_dot_erlang,true}]}, + {rel,"start_sasl","1.0",[sasl],[{load_dot_erlang,true}]}, {emu_name,"beam"}, {relocatable,true}, {profile,development}, @@ -2148,9 +2226,9 @@ save_config(Config) -> {app,stdlib,[{incl_cond,include},{vsn,StdVsn}, {lib_dir,StdLibDir},{mod,_,[]}|_]}, {boot_rel,"start_clean"}, - {rel,"no_dot_erlang","1.0",[]}, - {rel,"start_clean","1.0",[]}, - {rel,"start_sasl","1.0",[sasl]}, + {rel,"no_dot_erlang","1.0",[],[{load_dot_erlang,false}]}, + {rel,"start_clean","1.0",[],[{load_dot_erlang,true}]}, + {rel,"start_sasl","1.0",[sasl],[{load_dot_erlang,true}]}, {emu_name,"beam"}, {relocatable,true}, {profile,development}, @@ -2560,9 +2638,6 @@ windows_erl_libs(_Config) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Library functions -erl_libs() -> - reltool_utils:erl_libs(). - datadir(Config) -> %% Removes the trailing slash... filename:nativename(?config(data_dir,Config)). diff --git a/lib/reltool/test/reltool_test_lib.erl b/lib/reltool/test/reltool_test_lib.erl index be48ea4726..033d952d0a 100644 --- a/lib/reltool/test/reltool_test_lib.erl +++ b/lib/reltool/test/reltool_test_lib.erl @@ -237,7 +237,8 @@ wait_for_close() -> wait_for_close() end. - +erl_libs() -> + lists:sort([filename:absname(P) || P<-reltool_utils:erl_libs()]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% A small test server, which can be run standalone in a shell diff --git a/lib/reltool/test/reltool_wx_SUITE.erl b/lib/reltool/test/reltool_wx_SUITE.erl index f6f7721762..983c8f6c52 100644 --- a/lib/reltool/test/reltool_wx_SUITE.erl +++ b/lib/reltool/test/reltool_wx_SUITE.erl @@ -74,7 +74,12 @@ start_all_windows(_Config) -> %% Test that server pid can be fetched, and that server is alive {ok, Server} = ?msym({ok,_}, reltool:get_server(SysPid)), ?m(true, erlang:is_process_alive(Server)), - ?m({ok,{sys,[]}}, reltool:get_config(Server)), + Sys = + case reltool_test_lib:erl_libs() of + [] -> []; + Libs -> [{lib_dirs,Libs}] + end, + ?m({ok,{sys,Sys}}, reltool:get_config(Server)), %% Terminate check_no_win_crash(), diff --git a/lib/runtime_tools/c_src/Makefile.in b/lib/runtime_tools/c_src/Makefile.in index 4530a83aee..75b3a98d56 100644 --- a/lib/runtime_tools/c_src/Makefile.in +++ b/lib/runtime_tools/c_src/Makefile.in @@ -36,7 +36,7 @@ CC = $(DED_CC) CFLAGS = $(DED_CFLAGS) -I./ LD = $(DED_LD) SHELL = /bin/sh -LIBS = $(DED_LIBS) +LIBS = $(DED_LIBS) @LIBS@ LDFLAGS += $(DED_LDFLAGS) TRACE_LIBNAME = dyntrace trace_file_drv trace_ip_drv @@ -58,7 +58,7 @@ TYPE_FLAGS = $(CFLAGS) endif endif -ALL_CFLAGS = @DEFS@ $(TYPE_FLAGS) $(TRACE_DRV_INCLUDES) \ +ALL_CFLAGS = @DEFS@ @ERTS_CONFIG_H_IDIR@ $(TYPE_FLAGS) $(TRACE_DRV_INCLUDES) \ -I$(OBJDIR) -I$(ERL_TOP)/erts/emulator/$(TARGET) ROOTDIR = $(ERL_TOP)/lib @@ -75,7 +75,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/runtime_tools-$(VSN) # Misc Macros # ---------------------------------------------------- -TRACE_LIBS = $(foreach LIB, $(TRACE_LIBNAME), $(LIBDIR)/$(LIB)$(TYPEMARKER).@DED_EXT@) +TRACE_LIBS = $(foreach LIB, $(TRACE_LIBNAME), $(LIBDIR)/$(LIB)$(TYPEMARKER).$(DED_EXT)) # ---------------------------------------------------- # Targets @@ -94,7 +94,7 @@ $(LIBDIR): $(OBJDIR)/%$(TYPEMARKER).o: %.c dyntrace_lttng.h $(V_CC) -c -o $@ $(ALL_CFLAGS) $< -$(LIBDIR)/%$(TYPEMARKER).@DED_EXT@: $(OBJDIR)/%$(TYPEMARKER).o +$(LIBDIR)/%$(TYPEMARKER).$(DED_EXT): $(OBJDIR)/%$(TYPEMARKER).o $(V_LD) $(LDFLAGS) -o $@ $^ $(LIBS) clean: diff --git a/lib/runtime_tools/doc/src/dbg.xml b/lib/runtime_tools/doc/src/dbg.xml index 941a880778..e15fc3efe6 100644 --- a/lib/runtime_tools/doc/src/dbg.xml +++ b/lib/runtime_tools/doc/src/dbg.xml @@ -113,7 +113,7 @@ <p>The imported variables will be replaced by match_spec <c>const</c> expressions, which is consistent with the static scoping for Erlang <c>fun()</c>s. Local or global - function calls can not be in the guard or body of the fun + function calls cannot be in the guard or body of the fun however. Calls to builtin match_spec functions of course is allowed:</p> <pre> @@ -756,7 +756,7 @@ Error: fun containing local erlang function calls ('is_atomm' called in guard)\ <c>cant_add_local_node</c> is returned. </p> <p>If a trace port (see <seealso marker="#trace_port-2"><c>trace_port/2</c></seealso>) is - running on the local node, remote nodes can not be traced with + running on the local node, remote nodes cannot be traced with a tracer process. The error reason <c>cant_trace_remote_pid_to_local_port</c> is returned. A trace port can however be started on the remote node with the diff --git a/lib/runtime_tools/src/system_information.erl b/lib/runtime_tools/src/system_information.erl index 136ee55b54..8f7bfa195b 100644 --- a/lib/runtime_tools/src/system_information.erl +++ b/lib/runtime_tools/src/system_information.erl @@ -400,7 +400,6 @@ os_getenv_erts_specific() -> "ERL_MALLOC_LIB", "ERL_MAX_PORTS", "ERL_MAX_ETS_TABLES", - "ERL_NO_VFORK", "ERL_NO_KERNEL_POLL", "ERL_THREAD_POOL_SIZE", "ERLC_EMULATOR", diff --git a/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat b/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat index a0e3806981..a9399a5d53 100644 --- a/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat +++ b/lib/runtime_tools/test/system_information_SUITE_data/information_test_report.dat @@ -6829,7 +6829,7 @@ {native,false}, {compiler,"4.9.1"}, {md5,"55bb9fddcdf820938be2efee15eccd82"}]}, - {otp_ring0, + {erl_init, [{loaded,true}, {native,false}, {compiler,"4.9.1"}, diff --git a/lib/sasl/src/Makefile b/lib/sasl/src/Makefile index 7338bdf016..fd62588f5c 100644 --- a/lib/sasl/src/Makefile +++ b/lib/sasl/src/Makefile @@ -61,7 +61,11 @@ TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) $(APP_TARGET) $(APPUP_TARGET) # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- + ERL_COMPILE_FLAGS += -I../../stdlib/include -Werror +ifeq ($(USE_ESOCK), yes) +ERL_COMPILE_FLAGS += -DUSE_ESOCK=true +endif # ---------------------------------------------------- diff --git a/lib/sasl/src/rb.erl b/lib/sasl/src/rb.erl index 28829132a1..bef4268d3a 100644 --- a/lib/sasl/src/rb.erl +++ b/lib/sasl/src/rb.erl @@ -890,7 +890,7 @@ read_rep(Fd, FilePosition, Device, Abort, Log) -> handle_bad_form(Date, Msg, Device, Abort, Log) -> io:format("rb: ERROR! A report on bad form was encountered. " ++ - "It can not be printed to the log.~n~n"), + "It cannot be printed to the log.~n~n"), io:format("Details:~n~p ~tp~n~n", [Date,Msg]), case {Abort,Device,open_log_file(Log)} of {true,standard_io,standard_io} -> diff --git a/lib/sasl/src/release_handler.erl b/lib/sasl/src/release_handler.erl index 7570b74c1a..48feac1a21 100644 --- a/lib/sasl/src/release_handler.erl +++ b/lib/sasl/src/release_handler.erl @@ -1120,7 +1120,7 @@ new_emulator_make_hybrid_config(CurrentVsn,ToVsn,TmpVsn,RelDir,Masters) -> {ok,[FC]} -> FC; {error,Error1} -> - io:format("Warning: ~w can not read ~tp: ~tp~n", + io:format("Warning: ~w cannot read ~tp: ~tp~n", [?MODULE,FromFile,Error1]), [] end, @@ -1130,7 +1130,7 @@ new_emulator_make_hybrid_config(CurrentVsn,ToVsn,TmpVsn,RelDir,Masters) -> {ok,[ToConfig]} -> [lists:keyfind(App,1,ToConfig) || App <- [kernel,stdlib,sasl]]; {error,Error2} -> - io:format("Warning: ~w can not read ~tp: ~tp~n", + io:format("Warning: ~w cannot read ~tp: ~tp~n", [?MODULE,ToFile,Error2]), [false,false,false] end, diff --git a/lib/sasl/src/release_handler_1.erl b/lib/sasl/src/release_handler_1.erl index ca97515299..bf18691687 100644 --- a/lib/sasl/src/release_handler_1.erl +++ b/lib/sasl/src/release_handler_1.erl @@ -147,7 +147,7 @@ split_instructions([], Before) -> %% If PrePurgeMethod == soft_purge, the function will succeed %% only if there is no process running old code of any of the %% modules. Else it will throw {error,Mod}, where Mod is the -%% first module found that can not be soft_purged. +%% first module found that cannot be soft_purged. %% %% If PrePurgeMethod == brutal_purge, the function will %% always succeed and return a list of all modules that are diff --git a/lib/sasl/src/systools.erl b/lib/sasl/src/systools.erl index dd1a58c3c1..34eca6679f 100644 --- a/lib/sasl/src/systools.erl +++ b/lib/sasl/src/systools.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -73,7 +73,7 @@ make_tar(RelName, Opt) -> script2boot(File) -> case systools_lib:file_term2binary(File ++ ".script", File ++ ".boot") of {error,Error} -> - io:format(systools_make:format_error(Error)), + io:format("~ts", [systools_make:format_error(Error)]), error; _ -> ok @@ -84,7 +84,7 @@ script2boot(File, Output0, _Opt) -> Output = Output0++".boot", case systools_lib:file_term2binary(Input, Output) of {error,Error} -> - io:format(systools_make:format_error(Error)), + io:format("~ts", [systools_make:format_error(Error)]), error; _ -> ok diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index f085246924..b5a6b44f93 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -33,6 +33,7 @@ -export([read_application/4]). -export([make_hybrid_boot/4]). +-export([preloaded/0]). % Exported just for testing -import(lists, [filter/2, keysort/2, keysearch/3, map/2, reverse/1, append/1, foldl/3, member/2, foreach/2]). @@ -45,6 +46,13 @@ -compile({inline,[{badarg,2}]}). +-ifdef(USE_ESOCK). +-define(ESOCK_MODS, [socket]). +-else. +-define(ESOCK_MODS, []). +-endif. + + %%----------------------------------------------------------------- %% Create a boot script from a release file. %% Options is a list of {path, Path} | silent | local @@ -1562,11 +1570,11 @@ mandatory_modules() -> preloaded() -> %% Sorted - [atomics, counters, erl_prim_loader,erl_tracer,erlang, + [atomics,counters,erl_init,erl_prim_loader,erl_tracer,erlang, erts_code_purger,erts_dirty_process_signal_handler, erts_internal,erts_literal_area_collector, - init,otp_ring0,persistent_term,prim_buffer,prim_eval,prim_file, - prim_inet,prim_zip,zlib]. + init,net,persistent_term,prim_buffer,prim_eval,prim_file, + prim_inet,prim_zip] ++ ?ESOCK_MODS ++ [zlib]. %%______________________________________________________________________ %% Kernel processes; processes that are specially treated by the init diff --git a/lib/sasl/src/systools_relup.erl b/lib/sasl/src/systools_relup.erl index e836d57670..5f1176ec69 100644 --- a/lib/sasl/src/systools_relup.erl +++ b/lib/sasl/src/systools_relup.erl @@ -587,7 +587,7 @@ default(warnings_as_errors) -> false. print_error({error, Mod, Error}) -> S = apply(Mod, format_error, [Error]), - io:format(S, []); + io:format("~ts", [S]); print_error(Other) -> io:format("Error: ~tp~n", [Other]). diff --git a/lib/sasl/test/installer.erl b/lib/sasl/test/installer.erl index e38d0cfa7b..5429008a5f 100644 --- a/lib/sasl/test/installer.erl +++ b/lib/sasl/test/installer.erl @@ -905,7 +905,7 @@ start_client(TestNode,Client,Sname) -> wait_started(TestNode,Node) after 30000 -> ?print([{start_client,failed,Node},net_adm:ping(Node)]), - ?fail({"can not start", Node}) + ?fail({"cannot start", Node}) end. start_client_unix(TestNode,Sname,Node) -> diff --git a/lib/sasl/test/rb_SUITE.erl b/lib/sasl/test/rb_SUITE.erl index 2b6e452d14..e5ca1775d5 100644 --- a/lib/sasl/test/rb_SUITE.erl +++ b/lib/sasl/test/rb_SUITE.erl @@ -423,7 +423,7 @@ start_stop_log(Config) -> StdioResult2 = capture(fun() -> rb:log_list() end), {ok,<<>>} = file:read_file(OutFile), - %% Test that standard_io is used if log file can not be opened + %% Test that standard_io is used if log file cannot be opened ok = rb:start_log(filename:join(nonexistingdir,"newfile.txt")), StdioResult = capture(fun() -> rb:show(1) end), {ok,<<>>} = file:read_file(OutFile), diff --git a/lib/sasl/test/systools_SUITE.erl b/lib/sasl/test/systools_SUITE.erl index 1827974866..6c913850b9 100644 --- a/lib/sasl/test/systools_SUITE.erl +++ b/lib/sasl/test/systools_SUITE.erl @@ -1654,7 +1654,7 @@ abnormal_relup(Config) when is_list(Config) -> ok. -%% make_relup: Check relup can not be created is sasl is not in rel file. +%% make_relup: Check relup cannot be created is sasl is not in rel file. no_sasl_relup(Config) when is_list(Config) -> {ok, OldDir} = file:get_cwd(), {Dir1,Name1} = create_script(latest1_no_sasl,Config), diff --git a/lib/sasl/test/test_lib.hrl b/lib/sasl/test/test_lib.hrl index f5210d4f27..7867d3da39 100644 --- a/lib/sasl/test/test_lib.hrl +++ b/lib/sasl/test/test_lib.hrl @@ -1,3 +1,3 @@ -define(ertsvsn,"4.4"). --define(kernelvsn,"5.3"). --define(stdlibvsn,"3.4"). +-define(kernelvsn,"6.0"). +-define(stdlibvsn,"3.5"). diff --git a/lib/snmp/mibs/Makefile.in b/lib/snmp/mibs/Makefile.in index 2350194077..77893cbdc8 100644 --- a/lib/snmp/mibs/Makefile.in +++ b/lib/snmp/mibs/Makefile.in @@ -41,14 +41,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/snmp-$(VSN) # Common macros # ---------------------------------------------------- -# NOTE: -# The OTP-REG mib actually belongs to another -# application (otp_mibs), and is exported by this -# app. But since that app is built later, we have -# to built it here in order to be able to build -# OTP-SNMPEA-MIB (that needs otpModules and -# otpApplications). -MIBS_A = \ +MIBS = \ RFC1213-MIB \ STANDARD-MIB \ SNMPv2-TM \ @@ -62,16 +55,10 @@ MIBS_A = \ SNMP-VIEW-BASED-ACM-MIB \ SNMP-USM-AES-MIB \ INET-ADDRESS-MIB \ - TRANSPORT-ADDRESS-MIB - -MIBS_B = OTP-SNMPEA-MIB - -BUILD_MIBS = \ - $(MIBS_A) \ + TRANSPORT-ADDRESS-MIB \ OTP-REG \ - $(MIBS_B) - -MIBS = $(MIBS_A) $(MIBS_B) + OTP-TC \ + OTP-SNMPEA-MIB STD_v1_MIB_FILES = \ RFC1155-SMI.mib \ @@ -100,8 +87,8 @@ HRL_FILES = $(SNMP_HRL_TARGET_DIR)/SNMPv2-TC.hrl \ TARGET_FILES = \ $(ERL_TOP)/lib/snmp/bin/snmp-v2tov1 \ - $(BUILD_MIBS:%=$(SNMP_BIN_TARGET_DIR)/%.bin) \ - $(HRL_TARGETS) \ + $(MIBS:%=$(SNMP_BIN_TARGET_DIR)/%.bin) \ + $(HRL_TARGETS) \ $(V1_MIB_FILES) @@ -136,21 +123,18 @@ endif # Targets # ---------------------------------------------------- -OTP_MIBDIR = $(shell if test -d ../../otp_mibs; then echo otp_mibs; \ - else echo sasl; fi) - debug opt: $(TARGET_FILES) $(ERL_TOP)/lib/snmp/bin/snmp-v2tov1: $(ERL_TOP)/lib/snmp/bin/snmp-v2tov1.src $(gen_verbose)$(PERL) -p -e 's?%PERL%?$(PERL)? ' < $< > $@ $(V_at)chmod 755 $@ -$(SNMP_BIN_TARGET_DIR)/OTP-REG.bin: $(ERL_TOP)/lib/$(OTP_MIBDIR)/mibs/OTP-REG.mib - $(snmp_verbose)$(ERLC) -pa $(SNMP_TOOLKIT)/ebin -I $(SNMP_TOOLKIT)/priv/mibs $(SNMP_FLAGS) -o $(SNMP_BIN_TARGET_DIR) $< - # To support parallel make, we'll need explicit dependencies # to ensure that an imported MIB has been compiled when it's needed. +$(SNMP_BIN_TARGET_DIR)/OTP-TC.bin: \ + $(SNMP_BIN_TARGET_DIR)/OTP-REG.bin + $(SNMP_BIN_TARGET_DIR)/STANDARD-MIB.bin: \ $(SNMP_BIN_TARGET_DIR)/RFC1213-MIB.bin @@ -208,8 +192,6 @@ info: @echo "" @echo "TARGET_FILES = $(TARGET_FILES)" @echo "" - @echo "OTP_MIBDIR = $(OTP_MIBDIR)" - @echo "" @echo "SNMP_VSN = $(SNMP_VSN)" @echo "VSN = $(VSN)" @echo "RELSYSDIR = "$(RELSYSDIR)"" diff --git a/lib/otp_mibs/mibs/OTP-REG.mib b/lib/snmp/mibs/OTP-REG.mib index bf1585061c..bf1585061c 100644 --- a/lib/otp_mibs/mibs/OTP-REG.mib +++ b/lib/snmp/mibs/OTP-REG.mib diff --git a/lib/otp_mibs/mibs/OTP-TC.mib b/lib/snmp/mibs/OTP-TC.mib index efe5451f0c..efe5451f0c 100644 --- a/lib/otp_mibs/mibs/OTP-TC.mib +++ b/lib/snmp/mibs/OTP-TC.mib diff --git a/lib/snmp/src/agent/snmpa_agent.erl b/lib/snmp/src/agent/snmpa_agent.erl index a521b3773b..f280260f47 100644 --- a/lib/snmp/src/agent/snmpa_agent.erl +++ b/lib/snmp/src/agent/snmpa_agent.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1794,9 +1794,8 @@ worker_loop(Master) -> GbMaxVBs, Extra) end catch - T:E -> - exit({worker_crash, Req, T, E, - erlang:get_stacktrace()}) + C:E:S -> + exit({worker_crash, Req, C, E, S}) end, Master ! worker_available, HandlePduRes; % For debugging... @@ -1822,9 +1821,8 @@ worker_loop(Master) -> get(net_if)) end catch - T:E -> - exit({worker_crash, Req, T, E, - erlang:get_stacktrace()}) + C:E:S -> + exit({worker_crash, Req, C, E, S}) end, Master ! worker_available, SendTrapRes; % For debugging... @@ -2543,22 +2541,31 @@ process_msg( process_pdu(#pdu{type='get-request', request_id = ReqId, varbinds=Vbs}, _PduMS, Vsn, MibView, _GbMaxVBs) -> ?vtrace("get ~p",[ReqId]), - Res = get_err(do_get(MibView, Vbs, false)), - ?vtrace("get result: " - "~n ~p",[Res]), + OrigRes = do_get(MibView, Vbs, false), + Res = get_err(OrigRes), {ErrStatus, ErrIndex, ResVarbinds} = if Vsn =:= 'version-1' -> validate_get_v1(Res); true -> Res end, - ?vtrace("get final result: " - "~n Error status: ~p" - "~n Error index: ~p" - "~n Varbinds: ~p", - [ErrStatus,ErrIndex,ResVarbinds]), + if + (ErrStatus =/= noError) -> + ?vlog("get final result: " + "~n Error status: ~p" + "~n Error index: ~p" + "~n when" + "~n Original Result: " + "~n ~p", [ErrStatus, ErrIndex, OrigRes]); + true -> + ?vtrace("get final result: " + "~n Error status: ~p" + "~n Error index: ~p" + "~n Varbinds: ~p", + [ErrStatus, ErrIndex, ResVarbinds]) + end, ResponseVarbinds = lists:keysort(#varbind.org_index, ResVarbinds), ?vtrace("response varbinds: " - "~n ~p",[ResponseVarbinds]), + "~n ~p", [ResponseVarbinds]), make_response_pdu(ReqId, ErrStatus, ErrIndex, Vbs, ResponseVarbinds); process_pdu(#pdu{type = 'get-next-request', request_id = ReqId, varbinds = Vbs}, @@ -2566,22 +2573,31 @@ process_pdu(#pdu{type = 'get-next-request', request_id = ReqId, varbinds = Vbs}, ?vtrace("process get-next-request -> entry with" "~n ReqId: ~p" "~n Vbs: ~p" - "~n MibView: ~p",[ReqId, Vbs, MibView]), - Res = get_err(do_get_next(MibView, Vbs)), - ?vtrace("get-next result: " - "~n ~p",[Res]), + "~n MibView: ~p", [ReqId, Vbs, MibView]), + OrigRes = do_get_next(MibView, Vbs), + Res = get_err(OrigRes), {ErrStatus, ErrIndex, ResVarbinds} = if Vsn =:= 'version-1' -> validate_next_v1(Res, MibView); true -> Res end, - ?vtrace("get-next final result -> validation result:" - "~n Error status: ~p" - "~n Error index: ~p" - "~n Varbinds: ~p",[ErrStatus,ErrIndex,ResVarbinds]), + if + (ErrStatus =/= noError) -> + ?vlog("get-next final result: " + "~n Error status: ~p" + "~n Error index: ~p" + "~n when" + "~n Original Result: " + "~n ~p", [ErrStatus, ErrIndex, OrigRes]); + true -> + ?vtrace("get-next final result:" + "~n Error status: ~p" + "~n Error index: ~p" + "~n Varbinds: ~p", [ErrStatus, ErrIndex, ResVarbinds]) + end, ResponseVarbinds = lists:keysort(#varbind.org_index, ResVarbinds), ?vtrace("get-next final result -> response varbinds: " - "~n ~p",[ResponseVarbinds]), + "~n ~p", [ResponseVarbinds]), make_response_pdu(ReqId, ErrStatus, ErrIndex, Vbs, ResponseVarbinds); process_pdu(#pdu{type = 'get-bulk-request', @@ -2590,31 +2606,50 @@ process_pdu(#pdu{type = 'get-bulk-request', error_status = NonRepeaters, error_index = MaxRepetitions}, PduMS, _Vsn, MibView, GbMaxVBs) -> - {ErrStatus, ErrIndex, ResponseVarbinds} = - get_err(do_get_bulk(MibView, NonRepeaters, MaxRepetitions, PduMS, Vbs, - GbMaxVBs)), - ?vtrace("get-bulk final result: " - "~n Error status: ~p" - "~n Error index: ~p" - "~n Respons varbinds: ~p", - [ErrStatus,ErrIndex,ResponseVarbinds]), + OrigRes = do_get_bulk(MibView, NonRepeaters, MaxRepetitions, PduMS, Vbs, + GbMaxVBs), + {ErrStatus, ErrIndex, ResponseVarbinds} = get_err(OrigRes), + if + (ErrStatus =/= noError) -> + ?vlog("get-bulk final result: " + "~n Error Status: ~p" + "~n Error Index: ~p" + "~n when" + "~n Original Result: " + "~n ~p", [ErrStatus, ErrIndex, OrigRes]); + true -> + ?vtrace("get-bulk final result: " + "~n Error status: ~p" + "~n Error index: ~p" + "~n Response Varbinds: ~p", + [ErrStatus, ErrIndex, ResponseVarbinds]) + end, make_response_pdu(ReqId, ErrStatus, ErrIndex, Vbs, ResponseVarbinds); process_pdu(#pdu{type = 'set-request', request_id = ReqId, varbinds = Vbs}, - _PduMS, Vsn, MibView, _GbMaxVbs)-> - Res = do_set(MibView, Vbs), - ?vtrace("set result: " - "~n ~p",[Res]), + _PduMS, Vsn, MibView, _GbMaxVbs) -> + OrigRes = do_set(MibView, Vbs), {ErrStatus, ErrIndex} = if - Vsn =:= 'version-1' -> validate_err(v2_to_v1, Res); - true -> Res + Vsn =:= 'version-1' -> validate_err(v2_to_v1, OrigRes); + true -> OrigRes end, - ?vtrace("set final result: " - "~n Error status: ~p" - "~n Error index: ~p",[ErrStatus,ErrIndex]), + if + (ErrStatus =/= noError) -> + ?vlog("set final result: " + "~n Error Status: ~p" + "~n Error Index: ~p" + "~n when" + "~n Original Result: " + "~n ~p", [ErrStatus, ErrIndex, OrigRes]); + true -> + ?vtrace("set final result: " + "~n Error Status: ~p" + "~n Error Index: ~p", [ErrStatus, ErrIndex]) + end, make_response_pdu(ReqId, ErrStatus, ErrIndex, Vbs, Vbs). + %%----------------------------------------------------------------- %% Transform a value == noSuchInstance | noSuchObject or a %% Counter64 type to a noSuchName error for the whole pdu. diff --git a/lib/snmp/src/agent/snmpa_local_db.erl b/lib/snmp/src/agent/snmpa_local_db.erl index eb67b9cd6f..f481641242 100644 --- a/lib/snmp/src/agent/snmpa_local_db.erl +++ b/lib/snmp/src/agent/snmpa_local_db.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -147,12 +147,13 @@ init([Prio, DbDir, DbInitError, Opts]) -> do_init(Prio, DbDir, DbInitError, Opts) -> process_flag(priority, Prio), process_flag(trap_exit, true), - put(sname,ldb), - put(verbosity,get_opt(verbosity, Opts, ?default_verbosity)), + put(sname, get_opt(sname, Opts, ldb)), + put(verbosity, get_opt(verbosity, Opts, ?default_verbosity)), ?vlog("starting",[]), Dets = dets_open(DbDir, DbInitError, Opts), Ets = ets:new(?ETS_TAB, [set, protected]), ?vdebug("started",[]), + put(started, snmp_misc:formated_timestamp()), {ok, #state{dets = Dets, ets = Ets}}. dets_open(DbDir, DbInitError, Opts) -> @@ -625,7 +626,7 @@ handle_info(Info, State) -> terminate(Reason, State) -> - ?vlog("terminate: ~p",[Reason]), + ?vlog("terminate: ~p", [Reason]), close(State). diff --git a/lib/snmp/src/agent/snmpa_set_lib.erl b/lib/snmp/src/agent/snmpa_set_lib.erl index 3dcf49cbe6..94120f4c7d 100644 --- a/lib/snmp/src/agent/snmpa_set_lib.erl +++ b/lib/snmp/src/agent/snmpa_set_lib.erl @@ -46,9 +46,9 @@ %%* 6) IF value is outside the acceptable range THEN wrongValue. %% 7) IF variable does not exist and could not ever be created %% THEN noCreation. -%% 8) IF variable can not be created now THEN inconsistentName. -%% 9) IF value can not be set now THEN inconsistentValue. -%%* 9) IF instances of the variable can not be modified THEN notWritable. +%% 8) IF variable cannot be created now THEN inconsistentName. +%% 9) IF value cannot be set now THEN inconsistentValue. +%%* 9) IF instances of the variable cannot be modified THEN notWritable. %% 10) IF an unavailable resource is needed THEN resourceUnavailable. %% 11) IF any other error THEN genErr. %% 12) Otherwise ok! diff --git a/lib/snmp/src/agent/snmpa_trap.erl b/lib/snmp/src/agent/snmpa_trap.erl index 31805c3bee..d04b6a206e 100644 --- a/lib/snmp/src/agent/snmpa_trap.erl +++ b/lib/snmp/src/agent/snmpa_trap.erl @@ -830,11 +830,11 @@ do_send_v1_trap(Enter, Spec, V1Res, NVbs, ExtraInfo, NetIf, SysUpTime) -> case lists:keyfind(transportDomainUdpIpv4, 1, Transports) of false -> ?vtrace( - "snmpa_trap: can not send v1 trap " + "snmpa_trap: cannot send v1 trap " "without IPv4 domain: ~p", [Transports]), user_err( - "snmpa_trap: can not send v1 trap " + "snmpa_trap: cannot send v1 trap " "without IPv4 domain: ~p", [Transports]); DomainAddr -> diff --git a/lib/snmp/src/misc/snmp_misc.erl b/lib/snmp/src/misc/snmp_misc.erl index 1f847b7a29..39254503ac 100644 --- a/lib/snmp/src/misc/snmp_misc.erl +++ b/lib/snmp/src/misc/snmp_misc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2015. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -64,8 +64,18 @@ strip_extension_from_filename/2, str_xor/2, time/3, - - verify_behaviour/2 + + verify_behaviour/2, + + %% These are used both for debugging (verbosity printouts) + %% and other such "utility" operations. + format_timestamp/1, format_timestamp/2, + format_short_timestamp/1, format_short_timestamp/2, + format_long_timestamp/1, format_long_timestamp/2, + formated_timestamp/0, + formated_short_timestamp/0, + formated_long_timestamp/0 + ]). @@ -112,6 +122,102 @@ now(sec) -> erlang:monotonic_time(seconds). + +%% --------------------------------------------------------------------------- +%% # formated_timstamp/0, formated_timstamp/1 +%% # format_short_timstamp/0, format_short_timstamp/1 +%% # format_long_timstamp/0, format_long_timstamp/1 +%% +%% Create a formatted timestamp. Short means that it will not include +%% the date in the formatted timestamp. Also it will only include millis. +%% --------------------------------------------------------------------------- + +formated_timestamp() -> + formated_long_timestamp(). + +formated_short_timestamp() -> + format_short_timestamp(os:timestamp()). + +formated_long_timestamp() -> + format_long_timestamp(os:timestamp()). + + +%% --------------------------------------------------------------------------- +%% # format_timstamp/1, format_timstamp/2 +%% # format_short_timstamp/1, format_short_timstamp/2 +%% # format_long_timstamp/1, format_long_timstamp/2 +%% +%% Formats the provided timestamp. Short means that it will not include +%% the date in the formatted timestamp. +%% --------------------------------------------------------------------------- + +-spec format_timestamp(Now :: erlang:timestamp()) -> + string(). + +format_timestamp(Now) -> + format_long_timestamp(Now). + +-spec format_short_timestamp(Now :: erlang:timestamp()) -> + string(). + +format_short_timestamp(Now) -> + N2T = fun(N) -> calendar:now_to_local_time(N) end, + format_timestamp(short, Now, N2T). + +-spec format_long_timestamp(Now :: erlang:timestamp()) -> + string(). + +format_long_timestamp(Now) -> + N2T = fun(N) -> calendar:now_to_local_time(N) end, + format_timestamp(long, Now, N2T). + +-spec format_timestamp(Now :: erlang:timestamp(), + N2T :: function()) -> + string(). + +format_timestamp(Now, N2T) when is_tuple(Now) andalso is_function(N2T) -> + format_long_timestamp(Now, N2T). + +-spec format_short_timestamp(Now :: erlang:timestamp(), + N2T :: function()) -> + string(). + +format_short_timestamp(Now, N2T) when is_tuple(Now) andalso is_function(N2T) -> + format_timestamp(short, Now, N2T). + +-spec format_long_timestamp(Now :: erlang:timestamp(), + N2T :: function()) -> + string(). + +format_long_timestamp(Now, N2T) when is_tuple(Now) andalso is_function(N2T) -> + format_timestamp(long, Now, N2T). + +format_timestamp(Format, {_N1, _N2, N3} = Now, N2T) -> + {Date, Time} = N2T(Now), + do_format_timestamp(Format, Date, Time, N3). + +do_format_timestamp(short, _Date, Time, N3) -> + do_format_short_timestamp(Time, N3); +do_format_timestamp(long, Date, Time, N3) -> + do_format_long_timestamp(Date, Time, N3). + +do_format_long_timestamp(Date, Time, N3) -> + {YYYY,MM,DD} = Date, + {Hour,Min,Sec} = Time, + FormatDate = + io_lib:format("~.4w-~.2.0w-~.2.0w ~.2.0w:~.2.0w:~.2.0w.~.3.0w", + [YYYY, MM, DD, Hour, Min, Sec, N3 div 1000]), + lists:flatten(FormatDate). + +do_format_short_timestamp(Time, N3) -> + {Hour,Min,Sec} = Time, + FormatDate = + io_lib:format("~.2.0w:~.2.0w:~.2.0w.~.3.0w", + [Hour, Min, Sec, N3 div 1000]), + lists:flatten(FormatDate). + + + is_crypto_supported(Alg) -> %% The 'try catch' handles the case when 'crypto' is %% not present in the system (or not started). diff --git a/lib/snmp/src/misc/snmp_verbosity.erl b/lib/snmp/src/misc/snmp_verbosity.erl index edfb52a474..9b2676d048 100644 --- a/lib/snmp/src/misc/snmp_verbosity.erl +++ b/lib/snmp/src/misc/snmp_verbosity.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2015. All Rights Reserved. +%% Copyright Ericsson AB 2000-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -70,16 +70,7 @@ print2(_Verbosity,Format,Arguments) -> timestamp() -> - format_timestamp(os:timestamp()). - -format_timestamp({_N1, _N2, N3} = Now) -> - {Date, Time} = calendar:now_to_datetime(Now), - {YYYY,MM,DD} = Date, - {Hour,Min,Sec} = Time, - FormatDate = - io_lib:format("~.4w:~.2.0w:~.2.0w ~.2.0w:~.2.0w:~.2.0w ~w", - [YYYY,MM,DD,Hour,Min,Sec,round(N3/1000)]), - lists:flatten(FormatDate). + snmp_misc:formated_timestamp(). process_args([], Acc) -> lists:reverse(Acc); @@ -155,7 +146,8 @@ image_of_sname(mgr) -> "MGR"; image_of_sname(mgr_misc) -> "MGR_MISC"; image_of_sname(undefined) -> ""; -image_of_sname(V) -> lists:flatten(io_lib:format("~p",[V])). +image_of_sname(N) when is_list(N) -> N; % Used in testing +image_of_sname(N) -> lists:flatten(io_lib:format("~p", [N])). validate(info) -> info; diff --git a/lib/snmp/test/snmp_agent_test.erl b/lib/snmp/test/snmp_agent_test.erl index f9c18af6ea..71e3fa3b9a 100644 --- a/lib/snmp/test/snmp_agent_test.erl +++ b/lib/snmp/test/snmp_agent_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2018. All Rights Reserved. +%% Copyright Ericsson AB 2003-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1116,15 +1116,15 @@ init_ms(Config, Opts) when is_list(Config) -> Opts1 = [MasterAgentVerbosity, MibsVerbosity, SymStoreVerbosity | Opts], [{vsn, v1} | start_v1_agent(Config, Opts1)]. -init_size_check_mse(Config) when is_list(Config) -> - MibStorage = {mib_storage, [{module, snmpa_mib_storage_ets}]}, - init_size_check_ms(Config, [MibStorage]). +%% init_size_check_mse(Config) when is_list(Config) -> +%% MibStorage = {mib_storage, [{module, snmpa_mib_storage_ets}]}, +%% init_size_check_ms(Config, [MibStorage]). -init_size_check_msd(Config) when is_list(Config) -> - AgentDbDir = ?GCONF(agent_db_dir, Config), - MibStorage = {mib_storage, [{module, snmpa_mib_storage_dets}, - {options, [{dir, AgentDbDir}]}]}, - init_size_check_ms(Config, [MibStorage]). +%% init_size_check_msd(Config) when is_list(Config) -> +%% AgentDbDir = ?GCONF(agent_db_dir, Config), +%% MibStorage = {mib_storage, [{module, snmpa_mib_storage_dets}, +%% {options, [{dir, AgentDbDir}]}]}, +%% init_size_check_ms(Config, [MibStorage]). init_size_check_msm(Config) when is_list(Config) -> ?line AgentNode = ?GCONF(snmp_master, Config), @@ -5146,12 +5146,21 @@ snmp_framework_mib_3(Config) when is_list(Config) -> %% Req. SNMP-FRAMEWORK-MIB snmp_framework_mib_test() -> ?line ["agentEngine"] = get_req(1, [[snmpEngineID,0]]), + T1 = snmp_misc:now(ms), ?line [EngineTime] = get_req(2, [[snmpEngineTime,0]]), + T2 = snmp_misc:now(ms), ?SLEEP(5000), + T3 = snmp_misc:now(ms), ?line [EngineTime2] = get_req(3, [[snmpEngineTime,0]]), - ?DBG("snmp_framework_mib -> time(s): " - "~n EngineTime 1 = ~p" - "~n EngineTime 2 = ~p", [EngineTime, EngineTime2]), + T4 = snmp_misc:now(ms), + ?PRINT2("snmp_framework_mib -> time(s): " + "~n EngineTime 1: ~p" + "~n Time to acquire: ~w ms" + "~n EngineTime 2: ~p" + "~n Time to acquire: ~w ms" + "~n => (5 sec sleep between get(snmpEngineTime))" + "~n Total time to acquire: ~w ms", + [EngineTime, T2-T1, EngineTime2, T4-T3, T4-T1]), if (EngineTime+7) < EngineTime2 -> ?line ?FAIL({too_large_diff, EngineTime, EngineTime2}); @@ -5160,11 +5169,18 @@ snmp_framework_mib_test() -> true -> ok end, + T5 = snmp_misc:now(ms), ?line case get_req(4, [[snmpEngineBoots,0]]) of [Boots] when is_integer(Boots) -> + T6 = snmp_misc:now(ms), + ?PRINT2("snmp_framework_mib -> " + "~n boots: ~p" + "~n Time to acquire: ~w ms", [Boots, T6-T5]), ok; Else -> - ?FAIL(Else) + ?PRINT2("snmp_framework_mib -> failed get proper boots:" + "~n ~p", [Else]), + ?FAIL({invalid_boots, Else}) end, ok. diff --git a/lib/snmp/test/snmp_agent_test_lib.erl b/lib/snmp/test/snmp_agent_test_lib.erl index c19c88528f..6defdadb5a 100644 --- a/lib/snmp/test/snmp_agent_test_lib.erl +++ b/lib/snmp/test/snmp_agent_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2016. All Rights Reserved. +%% Copyright Ericsson AB 2005-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -358,22 +358,22 @@ run(Mod, Func, Args, Opts) -> "~n StdM: ~p", [M,Vsn,Dir,User,SecLevel,EngineID,CtxEngineID,Community,StdM]), case snmp_test_mgr:start([%% {agent, snmp_test_lib:hostname()}, - {packet_server_debug,true}, - {debug,true}, - {agent, get(master_host)}, - {ipfamily, get(ipfamily)}, - {agent_udp, 4000}, - {trap_udp, 5000}, - {recbuf,65535}, + {packet_server_debug, true}, + {debug, true}, + {agent, get(master_host)}, + {ipfamily, get(ipfamily)}, + {agent_udp, 4000}, + {trap_udp, 5000}, + {recbuf, 65535}, quiet, Vsn, - {community, Community}, - {user, User}, - {sec_level, SecLevel}, - {engine_id, EngineID}, - {context_engine_id, CtxEngineID}, - {dir, Dir}, - {mibs, mibs(StdM, M)}]) of + {community, Community}, + {user, User}, + {sec_level, SecLevel}, + {engine_id, EngineID}, + {context_engine_id, CtxEngineID}, + {dir, Dir}, + {mibs, mibs(StdM, M)}]) of {ok, _Pid} -> case (catch apply(Mod, Func, Args)) of {'EXIT', Reason} -> @@ -383,10 +383,18 @@ run(Mod, Func, Args, Opts) -> catch snmp_test_mgr:stop(), Res end; + + {error, Reason} -> + ?EPRINT2("Failed starting (test) manager: " + "~n ~p", [Reason]), + catch snmp_test_mgr:stop(), + ?line ?FAIL({mgr_start_error, Reason}); + Err -> - io:format("Error starting manager: ~p\n", [Err]), + ?EPRINT2("Failed starting (test) manager: " + "~n ~p", [Err]), catch snmp_test_mgr:stop(), - ?line ?FAIL({mgr_start, Err}) + ?line ?FAIL({mgr_start_failure, Err}) end. @@ -464,20 +472,24 @@ start_agent(Config, Vsns, Opts) -> process_flag(trap_exit,true), + ?PRINT2("start_agent -> try start snmp app supervisor", []), {ok, AppSup} = snmp_app_sup:start_link(), unlink(AppSup), ?DBG("start_agent -> snmp app supervisor: ~p", [AppSup]), - ?DBG("start_agent -> start master agent",[]), + ?PRINT2("start_agent -> try start master agent",[]), ?line Sup = start_sup(Env), - - ?DBG("start_agent -> unlink from supervisor", []), ?line unlink(Sup), + ?DBG("start_agent -> snmp supervisor: ~p", [Sup]), + + ?PRINT2("start_agent -> try (rpc) start sub agent on ~p", [SaNode]), ?line SaDir = ?config(sa_dir, Config), - ?DBG("start_agent -> (rpc) start sub on ~p", [SaNode]), ?line {ok, Sub} = start_sub_sup(SaNode, SaDir), - ?DBG("start_agent -> done",[]), - ?line [{snmp_sup, {Sup, self()}}, {snmp_sub, Sub} | Config]. + ?DBG("start_agent -> done", []), + + ?line [{snmp_app_sup, AppSup}, + {snmp_sup, {Sup, self()}}, + {snmp_sub, Sub} | Config]. app_agent_env_init(Env0, Opts) -> @@ -670,35 +682,52 @@ merge_agent_options([{Key, _Value} = Opt|Opts], Options) -> stop_agent(Config) when is_list(Config) -> - ?LOG("stop_agent -> entry with" - "~n Config: ~p",[Config]), - - {Sup, Par} = ?config(snmp_sup, Config), - ?DBG("stop_agent -> attempt to stop (sup) ~p" - "~n Sup: ~p" - "~n Par: ~p", - [Sup, - (catch process_info(Sup)), - (catch process_info(Par))]), - - _Info = agent_info(Sup), - ?DBG("stop_agent -> Agent info: " - "~n ~p", [_Info]), - - stop_sup(Sup, Par), - - {Sup2, Par2} = ?config(snmp_sub, Config), - ?DBG("stop_agent -> attempt to stop (sub) ~p" - "~n Sup2: ~p" - "~n Par2: ~p", - [Sup2, - (catch process_info(Sup2)), - (catch process_info(Par2))]), - stop_sup(Sup2, Par2), - - ?DBG("stop_agent -> done - now cleanup config", []), - C1 = lists:keydelete(snmp_sup, 1, Config), - lists:keydelete(snmp_sub, 1, C1). + ?PRINT2("stop_agent -> entry with" + "~n Config: ~p",[Config]), + + + %% Stop the sub-agent (the agent supervisor) + {SubSup, SubPar} = ?config(snmp_sub, Config), + ?PRINT2("stop_agent -> attempt to stop sub agent (~p)" + "~n Sub Sup info: " + "~n ~p" + "~n Sub Par info: " + "~n ~p", + [SubSup, + (catch process_info(SubSup)), + (catch process_info(SubPar))]), + stop_sup(SubSup, SubPar), + Config2 = lists:keydelete(snmp_sub, 1, Config), + + + %% Stop the master-agent (the top agent supervisor) + {MasterSup, MasterPar} = ?config(snmp_sup, Config), + ?PRINT2("stop_agent -> attempt to stop master agent (~p)" + "~n Master Sup: " + "~n ~p" + "~n Master Par: " + "~n ~p" + "~n Agent Info: " + "~n ~p", + [MasterSup, + (catch process_info(MasterSup)), + (catch process_info(MasterPar)), + agent_info(MasterSup)]), + stop_sup(MasterSup, MasterPar), + Config3 = lists:keydelete(snmp_sup, 1, Config2), + + + %% Stop the top supervisor (of the snmp app) + AppSup = ?config(snmp_app_sup, Config), + ?PRINT2("stop_agent -> attempt to app sup ~p" + "~n App Sup: ~p", + [AppSup, + (catch process_info(AppSup))]), + Config4 = lists:keydelete(snmp_app_sup, 1, Config3), + + + ?PRINT2("stop_agent -> done", []), + Config4. start_sup(Env) -> @@ -728,7 +757,6 @@ stop_sup(Pid, _) -> ?LOG("stop_sup -> attempt to stop ~p", [Pid]), Ref = erlang:monitor(process, Pid), ?LOG("stop_sup -> Ref: ~p", [Ref]), - %% Pid ! {'EXIT', Parent, shutdown}, % usch exit(Pid, kill), await_stopped(Pid, Ref). @@ -864,13 +892,15 @@ expect(Mod, Line, Type, Enterp, Generic, Specific, ExpVBs) -> expect2(Mod, Line, Fun). expect2(Mod, Line, F) -> - io:format("EXPECT for ~w:~w~n", [Mod, Line]), + io_format_expect("for ~w:~w", [Mod, Line]), case F() of {error, Reason} -> - io:format("EXPECT failed at ~w:~w => ~n~p~n", [Mod, Line, Reason]), + io_format_expect("failed at ~w:~w => " + "~n ~p", [Mod, Line, Reason]), throw({error, {expect, Mod, Line, Reason}}); Else -> - io:format("EXPECT result for ~w:~w => ~n~p~n", [Mod, Line, Else]), + io_format_expect("result for ~w:~w => " + "~n ~p", [Mod, Line, Else]), Else end. @@ -899,20 +929,27 @@ receive_trap(To) -> end. +io_format_expect(F) -> + io_format_expect(F, []). + +io_format_expect(F, A) -> + ?PRINT2("EXPECT " ++ F, A). + + do_expect(Expect) when is_atom(Expect) -> do_expect({Expect, get_timeout()}); do_expect({any_pdu, To}) when is_integer(To) orelse (To =:= infinity) -> - io:format("EXPECT any PDU~n", []), + io_format_expect("any PDU"), receive_pdu(To); do_expect({any_trap, To}) -> - io:format("EXPECT any TRAP within ~w~n", [To]), + io_format_expect("any TRAP within ~w", [To]), receive_trap(To); do_expect({timeout, To}) -> - io:format("EXPECT nothing within ~w~n", [To]), + io_format_expect("nothing within ~w", [To]), receive X -> {error, {unexpected, X}} @@ -924,16 +961,16 @@ do_expect({timeout, To}) -> do_expect({Err, To}) when (is_atom(Err) andalso ((is_integer(To) andalso To > 0) orelse (To =:= infinity))) -> - io:format("EXPECT error ~w within ~w~n", [Err, To]), + io_format_expect("error ~w within ~w", [Err, To]), do_expect({{error, Err}, To}); do_expect({error, Err}) when is_atom(Err) -> Check = fun(_, R) -> R end, - io:format("EXPECT error ~w~n", [Err]), + io_format_expect("error ~w", [Err]), do_expect2(Check, any, Err, any, any, get_timeout()); do_expect({{error, Err}, To}) -> Check = fun(_, R) -> R end, - io:format("EXPECT error ~w within ~w~n", [Err, To]), + io_format_expect("error ~w within ~w", [Err, To]), do_expect2(Check, any, Err, any, any, To); %% exp_varbinds() -> [exp_varbind()] @@ -943,25 +980,23 @@ do_expect({{error, Err}, To}) -> %% ExpVBs -> exp_varbinds() | {VbsCondition, exp_varbinds()} do_expect(ExpVBs) -> Check = fun(_, R) -> R end, - io:format("EXPECT 'get-response'" - "~n with" - "~n Varbinds: ~p~n", [ExpVBs]), + io_format_expect("'get-response'" + "~n with" + "~n Varbinds: ~p", [ExpVBs]), do_expect2(Check, 'get-response', noError, 0, ExpVBs, get_timeout()). do_expect(v2trap, ExpVBs) -> Check = fun(_, R) -> R end, - io:format("EXPECT 'snmpv2-trap'" - "~n with" - "~n Varbinds: ~p~n", [ExpVBs]), + io_format_expect("'snmpv2-trap' with" + "~n Varbinds: ~p", [ExpVBs]), do_expect2(Check, 'snmpv2-trap', noError, 0, ExpVBs, get_timeout()); do_expect(report, ExpVBs) -> Check = fun(_, R) -> R end, - io:format("EXPECT 'report'" - "~n with" - "~n Varbinds: ~p~n", [ExpVBs]), + io_format_expect("'report' with" + "~n Varbinds: ~p", [ExpVBs]), do_expect2(Check, 'report', noError, 0, ExpVBs, get_timeout()); @@ -970,9 +1005,8 @@ do_expect(inform, ExpVBs) -> do_expect({inform, false}, ExpVBs) -> Check = fun(_, R) -> R end, - io:format("EXPECT 'inform-request' (false)" - "~n with" - "~n Varbinds: ~p~n", [ExpVBs]), + io_format_expect("'inform-request' (false) with" + "~n Varbinds: ~p", [ExpVBs]), do_expect2(Check, 'inform-request', noError, 0, ExpVBs, get_timeout()); do_expect({inform, true}, ExpVBs) -> @@ -986,9 +1020,8 @@ do_expect({inform, true}, ExpVBs) -> (_, Err) -> Err end, - io:format("EXPECT 'inform-request' (true)" - "~n with" - "~n Varbinds: ~p~n", [ExpVBs]), + io_format_expect("'inform-request' (true) with" + "~n Varbinds: ~p", [ExpVBs]), do_expect2(Check, 'inform-request', noError, 0, ExpVBs, get_timeout()); do_expect({inform, {error, EStat, EIdx}}, ExpVBs) @@ -1003,11 +1036,10 @@ do_expect({inform, {error, EStat, EIdx}}, ExpVBs) (_, Err) -> Err end, - io:format("EXPECT 'inform-request' (error)" - "~n with" - "~n Error Status: ~p" - "~n Error Index: ~p" - "~n Varbinds: ~p~n", [EStat, EIdx, ExpVBs]), + io_format_expect("'inform-request' (error) with" + "~n Error Status: ~p" + "~n Error Index: ~p" + "~n Varbinds: ~p", [EStat, EIdx, ExpVBs]), do_expect2(Check, 'inform-request', noError, 0, ExpVBs, get_timeout()). @@ -1018,26 +1050,23 @@ do_expect(Err, Idx, ExpVBs, To) when is_atom(Err) andalso (is_integer(Idx) orelse is_list(Idx) orelse (Idx == any)) -> Check = fun(_, R) -> R end, - io:format("EXPECT 'get-response'" - "~n with" - "~n Error: ~p" - "~n Index: ~p" - "~n Varbinds: ~p" - "~n within ~w~n", [Err, Idx, ExpVBs, To]), + io_format_expect("'get-response' withing ~w ms with" + "~n Error: ~p" + "~n Index: ~p" + "~n Varbinds: ~p", [To, Err, Idx, ExpVBs]), do_expect2(Check, 'get-response', Err, Idx, ExpVBs, To). do_expect(Type, Enterp, Generic, Specific, ExpVBs) -> - do_expect(Type, Enterp, Generic, Specific, ExpVBs, 3500). + do_expect(Type, Enterp, Generic, Specific, ExpVBs, get_timeout()). do_expect(trap, Enterp, Generic, Specific, ExpVBs, To) -> - io:format("EXPECT trap" - "~n with" - "~n Enterp: ~w" - "~n Generic: ~w" - "~n Specific: ~w" - "~n Varbinds: ~w" - "~n within ~w~n", [Enterp, Generic, Specific, ExpVBs, To]), + io_format_expect("trap within ~w ms with" + "~n Enterp: ~w" + "~n Generic: ~w" + "~n Specific: ~w" + "~n Varbinds: ~w", + [To, Enterp, Generic, Specific, ExpVBs]), PureE = purify_oid(Enterp), case receive_trap(To) of #trappdu{enterprise = PureE, @@ -1072,46 +1101,46 @@ do_expect2(Check, Type, Err, Idx, ExpVBs, To) #pdu{type = Type, error_status = Err, error_index = Idx} when ExpVBs =:= any -> - io:format("EXPECT received expected pdu (1)~n", []), + io_format_expect("received expected pdu (1)"), ok; #pdu{type = Type, request_id = ReqId, error_status = Err2, error_index = Idx} when ExpVBs =:= any -> - io:format("EXPECT received expected pdu with " - "unexpected error status (2): " - "~n Error Status: ~p~n", [Err2]), + io_format_expect("received expected pdu with " + "unexpected error status (2): " + "~n Error Status: ~p", [Err2]), {error, {unexpected_error_status, Err, Err2, ReqId}}; #pdu{error_status = Err} when (Type =:= any) andalso (Idx =:= any) andalso (ExpVBs =:= any) -> - io:format("EXPECT received expected pdu (3)~n", []), + io_format_expect("received expected pdu (3)"), ok; #pdu{request_id = ReqId, error_status = Err2} when (Type =:= any) andalso (Idx =:= any) andalso (ExpVBs =:= any) -> - io:format("EXPECT received expected pdu with " - "unexpected error status (4): " - "~n Error Status: ~p~n", [Err2]), + io_format_expect("received expected pdu with " + "unexpected error status (4): " + "~n Error Status: ~p", [Err2]), {error, {unexpected_error_status, Err, Err2, ReqId}}; #pdu{type = Type, error_status = Err} when (Idx =:= any) andalso (ExpVBs =:= any) -> - io:format("EXPECT received expected pdu (5)~n", []), + io_format_expect("received expected pdu (5)", []), ok; #pdu{type = Type, request_id = ReqId, error_status = Err2} when (Idx =:= any) andalso (ExpVBs =:= any) -> - io:format("EXPECT received expected pdu with " - "unexpected error status (6): " - "~n Error Status: ~p~n", [Err2]), + io_format_expect("received expected pdu with " + "unexpected error status (6): " + "~n Error Status: ~p", [Err2]), {error, {unexpected_error_status, Err, Err2, ReqId}}; #pdu{type = Type, @@ -1120,13 +1149,13 @@ do_expect2(Check, Type, Err, Idx, ExpVBs, To) error_index = EI} when is_list(Idx) andalso (ExpVBs =:= any) -> case lists:member(EI, Idx) of true -> - io:format("EXPECT received expected pdu with " - "expected error index (7)~n", []), + io_format_expect("received expected pdu with " + "expected error index (7)"), ok; false -> - io:format("EXPECT received expected pdu with " - "unexpected error index (8): " - "~n Error Index: ~p~n", [EI]), + io_format_expect("received expected pdu with " + "unexpected error index (8): " + "~n Error Index: ~p", [EI]), {error, {unexpected_error_index, EI, Idx, ReqId}} end; @@ -1136,15 +1165,15 @@ do_expect2(Check, Type, Err, Idx, ExpVBs, To) error_index = EI} when is_list(Idx) andalso (ExpVBs =:= any) -> case lists:member(EI, Idx) of true -> - io:format("EXPECT received expected pdu with " - "unexpected error status (9): " - "~n Error Status: ~p~n", [Err2]), + io_format_expect("received expected pdu with " + "unexpected error status (9): " + "~n Error Status: ~p", [Err2]), {error, {unexpected_error_status, Err, Err2, ReqId}}; false -> - io:format("EXPECT received expected pdu with " - "unexpected error (10): " - "~n Error Status: ~p" - "~n Error index: ~p~n", [Err2, EI]), + io_format_expect("received expected pdu with " + "unexpected error (10): " + "~n Error Status: ~p" + "~n Error index: ~p", [Err2, EI]), {error, {unexpected_error, {Err, Idx}, {Err2, EI}, ReqId}} end; @@ -1152,12 +1181,12 @@ do_expect2(Check, Type, Err, Idx, ExpVBs, To) request_id = ReqId, error_status = Err2, error_index = Idx2} when ExpVBs =:= any -> - io:format("EXPECT received unexpected pdu with (11) " - "~n Type: ~p" - "~n ReqId: ~p" - "~n Errot status: ~p" - "~n Error index: ~p" - "~n", [Type2, ReqId, Err2, Idx2]), + io_format_expect("received unexpected pdu with (11) " + "~n Type: ~p" + "~n ReqId: ~p" + "~n Errot status: ~p" + "~n Error index: ~p", + [Type2, ReqId, Err2, Idx2]), {error, {unexpected_pdu, {Type, Err, Idx}, {Type2, Err2, Idx2}, ReqId}}; @@ -1166,26 +1195,26 @@ do_expect2(Check, Type, Err, Idx, ExpVBs, To) error_status = Err, error_index = Idx, varbinds = VBs} = PDU -> - io:format("EXPECT received pdu (12): " - "~n [exp] Type: ~p" - "~n [exp] Error Status: ~p" - "~n [exp] Error Index: ~p" - "~n VBs: ~p" - "~nwhen" - "~n ExpVBs: ~p" - "~n", [Type, Err, Idx, VBs, ExpVBs]), + io_format_expect("received pdu (12): " + "~n [exp] Type: ~p" + "~n [exp] Error Status: ~p" + "~n [exp] Error Index: ~p" + "~n VBs: ~p" + "~nwhen" + "~n ExpVBs: ~p", + [Type, Err, Idx, VBs, ExpVBs]), Check(PDU, check_vbs(purify_oids(ExpVBs), VBs)); #pdu{type = Type, error_status = Err, varbinds = VBs} = PDU when Idx =:= any -> - io:format("EXPECT received pdu (13): " - "~n [exp] Type: ~p" - "~n [exp] Error Status: ~p" - "~n VBs: ~p" - "~nwhen" - "~n ExpVBs: ~p" - "~n", [Type, Err, VBs, ExpVBs]), + io_format_expect("received pdu (13): " + "~n [exp] Type: ~p" + "~n [exp] Error Status: ~p" + "~n VBs: ~p" + "~nwhen" + "~n ExpVBs: ~p", + [Type, Err, VBs, ExpVBs]), Check(PDU, check_vbs(purify_oids(ExpVBs), VBs)); #pdu{type = Type, @@ -1193,15 +1222,15 @@ do_expect2(Check, Type, Err, Idx, ExpVBs, To) error_status = Err, error_index = EI, varbinds = VBs} = PDU when is_list(Idx) -> - io:format("EXPECT received pdu (14): " - "~n [exp] Type: ~p" - "~n ReqId: ~p" - "~n [exp] Error Status: ~p" - "~n [exp] Error Index: ~p" - "~n VBs: ~p" - "~nwhen" - "~n ExpVBs: ~p" - "~n", [Type, ReqId, Err, EI, VBs, ExpVBs]), + io_format_expect("received pdu (14): " + "~n [exp] Type: ~p" + "~n ReqId: ~p" + "~n [exp] Error Status: ~p" + "~n [exp] Error Index: ~p" + "~n VBs: ~p" + "~nwhen" + "~n ExpVBs: ~p", + [Type, ReqId, Err, EI, VBs, ExpVBs]), PureVBs = purify_oids(ExpVBs), case lists:member(EI, Idx) of true -> @@ -1215,13 +1244,13 @@ do_expect2(Check, Type, Err, Idx, ExpVBs, To) error_status = Err2, error_index = Idx2, varbinds = VBs2} -> - io:format("EXPECT received unexpected pdu with (15) " - "~n Type: ~p" - "~n ReqId: ~p" - "~n Errot status: ~p" - "~n Error index: ~p" - "~n Varbinds: ~p" - "~n", [Type2, ReqId, Err2, Idx2, VBs2]), + io_format_expect("received unexpected pdu with (15) " + "~n Type: ~p" + "~n ReqId: ~p" + "~n Errot status: ~p" + "~n Error index: ~p" + "~n Varbinds: ~p", + [Type2, ReqId, Err2, Idx2, VBs2]), {error, {unexpected_pdu, {Type, Err, Idx, purify_oids(ExpVBs)}, @@ -1229,9 +1258,8 @@ do_expect2(Check, Type, Err, Idx, ExpVBs, To) ReqId}}; Error -> - io:format("EXPECT received error (16): " - "~n Error: ~p" - "~n", [Error]), + io_format_expect("received error (16): " + "~n Error: ~p", [Error]), Error end. diff --git a/lib/snmp/test/snmp_compiler_test.erl b/lib/snmp/test/snmp_compiler_test.erl index 0a7b729d1f..a28f925a22 100644 --- a/lib/snmp/test/snmp_compiler_test.erl +++ b/lib/snmp/test/snmp_compiler_test.erl @@ -226,22 +226,20 @@ agent_capabilities(Config) when is_list(Config) -> put(tname,agent_capabilities), p("starting with Config: ~p~n", [Config]), - SnmpPrivDir = code:priv_dir(snmp), + SnmpPrivDir = which_priv_dir(snmp), SnmpMibsDir = join(SnmpPrivDir, "mibs"), - OtpMibsPrivDir = code:priv_dir(otp_mibs), - OtpMibsMibsDir = join(OtpMibsPrivDir, "mibs"), Dir = ?config(mib_dir, Config), AcMib = join(Dir,"AC-TEST-MIB.mib"), ?line {ok, MibFile1} = snmpc:compile(AcMib, [options, version, - {i, [SnmpMibsDir, OtpMibsMibsDir]}, + {i, [SnmpMibsDir]}, {outdir, Dir}, {verbosity, trace}]), ?line {ok, Mib1} = snmp_misc:read_mib(MibFile1), ?line {ok, MibFile2} = snmpc:compile(AcMib, [options, version, agent_capabilities, - {i, [SnmpMibsDir, OtpMibsMibsDir]}, + {i, [SnmpMibsDir]}, {outdir, Dir}, {verbosity, trace}]), ?line {ok, Mib2} = snmp_misc:read_mib(MibFile2), @@ -269,28 +267,26 @@ module_compliance(Config) when is_list(Config) -> put(tname,module_compliance), p("starting with Config: ~p~n", [Config]), - SnmpPrivDir = code:priv_dir(snmp), - SnmpMibsDir = join(SnmpPrivDir, "mibs"), - OtpMibsPrivDir = code:priv_dir(otp_mibs), - OtpMibsMibsDir = join(OtpMibsPrivDir, "mibs"), - Dir = ?config(mib_dir, Config), - AcMib = join(Dir,"MC-TEST-MIB.mib"), + SnmpPrivDir = which_priv_dir(snmp), + SnmpMibsDir = join(SnmpPrivDir, "mibs"), + Dir = ?config(mib_dir, Config), + AcMib = join(Dir,"MC-TEST-MIB.mib"), ?line {ok, MibFile1} = snmpc:compile(AcMib, [options, version, - {i, [SnmpMibsDir, OtpMibsMibsDir]}, + {i, [SnmpMibsDir]}, {outdir, Dir}, {verbosity, trace}]), ?line {ok, Mib1} = snmp_misc:read_mib(MibFile1), ?line {ok, MibFile2} = snmpc:compile(AcMib, [options, version, module_compliance, - {i, [SnmpMibsDir, OtpMibsMibsDir]}, + {i, [SnmpMibsDir]}, {outdir, Dir}, {verbosity, trace}]), ?line {ok, Mib2} = snmp_misc:read_mib(MibFile2), MEDiff = Mib2#mib.mes -- Mib1#mib.mes, %% This is a rather pathetic test, but it is somthing... - io:format("agent_capabilities -> " + io:format("module_compliance -> " "~n MEDiff: ~p" "~n Mib1: ~p" "~n Mib2: ~p" @@ -731,6 +727,15 @@ check_desc(Desc1, Desc2) -> exit({'description not equal', Desc1, Desc2}). +which_priv_dir(App) -> + case code:priv_dir(App) of + Dir when is_list(Dir) -> + Dir; + {error, Reason} -> + exit({App, priv_dir_not_found, Reason}) + end. + + %% join(Comp) -> %% filename:join(Comp). diff --git a/lib/snmp/test/snmp_manager_test.erl b/lib/snmp/test/snmp_manager_test.erl index bb9b05b89f..5b0ebf8647 100644 --- a/lib/snmp/test/snmp_manager_test.erl +++ b/lib/snmp/test/snmp_manager_test.erl @@ -204,10 +204,15 @@ init_per_testcase(Case, Config) when is_list(Config) -> Result = case lists:member(Case, DeprecatedApiCases) of true -> - %% ?SKIP(api_no_longer_supported); {skip, api_no_longer_supported}; false -> - init_per_testcase2(Case, Config) + try init_per_testcase2(Case, Config) + catch + C:{skip, _} = E:_ when ((C =:= throw) orelse (C =:= exit)) -> + E; + C:E:_ when ((C =:= throw) orelse (C =:= exit)) -> + {skip, {catched, C, E}} + end end, p(Case, "init_per_testcase end when" "~n Nodes: ~p" @@ -326,9 +331,25 @@ init_per_testcase3(Case, Config) -> true -> Config end, + %% We don't need to try catch this (init_agent) + %% since we have a try catch "higher up"... Conf2 = init_agent(Conf1), - Conf3 = init_manager(AutoInform, Conf2), - Conf4 = init_mgr_user(Conf3), + Conf3 = try init_manager(AutoInform, Conf2) + catch AC:AE:_ -> + %% Ouch we need to clean up: + %% The init_agent starts an agent node! + init_per_testcase_fail_agent_cleanup(Conf2), + throw({skip, {manager_init_failed, AC, AE}}) + end, + Conf4 = try init_mgr_user(Conf3) + catch MC:ME:_ -> + %% Ouch we need to clean up: + %% The init_agent starts an agent node! + %% The init_magager starts an manager node! + init_per_testcase_fail_manager_cleanup(Conf3), + init_per_testcase_fail_agent_cleanup(Conf3), + throw({skip, {manager_user_init_failed, MC, ME}}) + end, case lists:member(Case, ApiCases02 ++ ApiCases03) of true -> init_mgr_user_data2(Conf4); @@ -339,6 +360,12 @@ init_per_testcase3(Case, Config) -> Config end. +init_per_testcase_fail_manager_cleanup(Conf) -> + (catch fin_manager(Conf)). + +init_per_testcase_fail_agent_cleanup(Conf) -> + (catch fin_agent(Conf)). + end_per_testcase(Case, Config) when is_list(Config) -> p(Case, "end_per_testcase begin when" "~n Nodes: ~p~n~n", [erlang:nodes()]), @@ -993,18 +1020,40 @@ notify_started02(Config) when is_list(Config) -> {config, [{verbosity, log}, {dir, ConfDir}, {db_dir, DbDir}]}], p("start snmpm client process"), - Pid1 = ns02_loop1_start(), + NumIterations = 5, + Pid1 = ns02_client_start(NumIterations), + + p("start snmpm ctrl (starter) process"), + Pid2 = ns02_ctrl_start(Opts, NumIterations), + + %% On a reasonably fast machine, one iteration takes approx 4 seconds. + %% We measure the first iteration, and then we wait for the remaining + %% ones (4 in this case). + ApproxStartTime = + case ns02_client_await_approx_runtime(Pid1) of + {ok, T} -> + T; + {error, Reason} -> + %% Attempt cleanup just in case + exit(Pid1, kill), + exit(Pid2, kill), + ?FAIL(Reason); + {skip, Reason} -> + %% Attempt cleanup just in case + exit(Pid1, kill), + exit(Pid2, kill), + ?SKIP(Reason) + end, - p("start snmpm starter process"), - Pid2 = ns02_loop2_start(Opts), - - p("await snmpm client process exit"), + p("await snmpm client process exit (max ~p+10000 msec)", [ApproxStartTime]), receive {'EXIT', Pid1, normal} -> ok; {'EXIT', Pid1, Reason1} -> - ?FAIL(Reason1) - after 25000 -> + ?FAIL({client, Reason1}) + after ApproxStartTime + 10000 -> + exit(Pid1, kill), + exit(Pid2, kill), ?FAIL(timeout) end, @@ -1013,8 +1062,9 @@ notify_started02(Config) when is_list(Config) -> {'EXIT', Pid2, normal} -> ok; {'EXIT', Pid2, Reason2} -> - ?FAIL(Reason2) + ?FAIL({ctrl, Reason2}) after 5000 -> + exit(Pid2, kill), ?FAIL(timeout) end, @@ -1022,26 +1072,63 @@ notify_started02(Config) when is_list(Config) -> ok. -ns02_loop1_start() -> - spawn_link(fun() -> ns02_loop1() end). +ns02_client_start(N) -> + Self = self(), + spawn_link(fun() -> ns02_client(Self, N) end). + +ns02_client_await_approx_runtime(Pid) -> + receive + {?MODULE, client_time, Time} -> + {ok, Time}; + {'EXIT', Pid, Reason} -> + p("client (~p) failed: " + "~n ~p", [Pid, Reason]), + {error, Reason} + + after 15000 -> + %% Either something is *really* wrong or this machine + %% is dog slow. Either way, this is a skip-reason... + {skip, approx_runtime_timeout} + end. + -ns02_loop1() -> - put(tname,ns02_loop1), +ns02_client(Parent, N) when is_pid(Parent) -> + put(tname, ns02_client), p("starting"), - ns02_loop1(dummy, snmpm:notify_started(?NS_TIMEOUT), 5). + ns02_client_loop(Parent, + dummy, snmpm:notify_started(?NS_TIMEOUT), + snmp_misc:now(ms), undefined, + N). -ns02_loop1(_Ref, _Pid, 0) -> - p("done"), +ns02_client_loop(_Parent, _Ref, _Pid, _Begin, _End, 0) -> + %% p("loop -> done"), exit(normal); -ns02_loop1(Ref, Pid, N) -> - p("entry when" - "~n Ref: ~p" - "~n Pid: ~p" - "~n N: ~p", [Ref, Pid, N]), +ns02_client_loop(Parent, Ref, Pid, Begin, End, N) + when is_pid(Parent) andalso is_integer(Begin) andalso is_integer(End) -> + %% p("loop -> [~w] inform parent: ~w, ~w => ~w", [N, Begin, End, End-Begin]), + Parent ! {?MODULE, client_time, N*(End-Begin)}, + ns02_client_loop(undefined, Ref, Pid, snmp_misc:now(ms), undefined, N); +ns02_client_loop(Parent, Ref, Pid, Begin, End, N) + when is_integer(Begin) andalso is_integer(End) -> + %% p("loop -> [~w] entry when" + %% "~n Ref: ~p" + %% "~n Pid: ~p" + %% "~n Begin: ~p" + %% "~n End: ~p", [N, Ref, Pid, Begin, End]), + ns02_client_loop(Parent, Ref, Pid, snmp_misc:now(ms), undefined, N); +ns02_client_loop(Parent, Ref, Pid, Begin, End, N) -> + %% p("loop(await message) -> [~w] entry when" + %% "~n Ref: ~p" + %% "~n Pid: ~p" + %% "~n Begin: ~p" + %% "~n End: ~p", [N, Ref, Pid, Begin, End]), receive {snmpm_started, Pid} -> p("received expected started message (~w)", [N]), - ns02_loop1(snmpm:monitor(), dummy, N); + ns02_client_loop(Parent, + snmpm:monitor(), dummy, + Begin, End, + N); {snmpm_start_timeout, Pid} -> p("unexpected timout"), ?FAIL({unexpected_start_timeout, Pid}); @@ -1049,24 +1136,24 @@ ns02_loop1(Ref, Pid, N) -> p("received expected DOWN message (~w) with" "~n Obj: ~p" "~n Reason: ~p", [N, Obj, Reason]), - ns02_loop1(dummy, snmpm:notify_started(?NS_TIMEOUT), N-1) - after 10000 -> - ?FAIL(timeout) + ns02_client_loop(Parent, + dummy, snmpm:notify_started(?NS_TIMEOUT), + Begin, snmp_misc:now(ms), + N-1) end. - -ns02_loop2_start(Opts) -> - spawn_link(fun() -> ns02_loop2(Opts) end). +ns02_ctrl_start(Opts, N) -> + spawn_link(fun() -> ns02_ctrl(Opts, N) end). -ns02_loop2(Opts) -> - put(tname,ns02_loop2), +ns02_ctrl(Opts, N) -> + put(tname, ns02_ctrl), p("starting"), - ns02_loop2(Opts, 5). + ns02_ctrl_loop(Opts, N). -ns02_loop2(_Opts, 0) -> +ns02_ctrl_loop(_Opts, 0) -> p("done"), exit(normal); -ns02_loop2(Opts, N) -> +ns02_ctrl_loop(Opts, N) -> p("entry when N: ~p", [N]), ?SLEEP(2000), p("start manager"), @@ -1074,7 +1161,7 @@ ns02_loop2(Opts, N) -> ?SLEEP(2000), p("stop manager"), snmpm:stop(), - ns02_loop2(Opts, N-1). + ns02_ctrl_loop(Opts, N-1). %%====================================================================== @@ -5416,15 +5503,14 @@ init_manager(AutoInform, Config) -> start_manager(Node, Vsns, Conf) end catch - T:E -> - StackTrace = ?STACK(), + C:E:S -> p("Failure during manager start: " - "~n Error Type: ~p" - "~n Error: ~p" - "~n StackTrace: ~p", [T, E, StackTrace]), + "~n Error Class: ~p" + "~n Error: ~p" + "~n StackTrace: ~p", [C, E, S]), %% And now, *try* to cleanup (catch stop_node(Node)), - ?FAIL({failed_starting_manager, T, E, StackTrace}) + ?FAIL({failed_starting_manager, C, E, S}) end. fin_manager(Config) -> @@ -5432,7 +5518,7 @@ fin_manager(Config) -> StopMgrRes = stop_manager(Node), StopCryptoRes = fin_crypto(Node), StopNode = stop_node(Node), - p("fin_agent -> stop apps and (mgr node ~p) node results: " + p("fin_manager -> stop apps and (mgr node ~p) node results: " "~n SNMP Mgr: ~p" "~n Crypto: ~p" "~n Node: ~p", @@ -5498,15 +5584,14 @@ init_agent(Config) -> start_agent(Node, Vsns, Conf) end catch - T:E -> - StackTrace = ?STACK(), + C:E:S -> p("Failure during agent start: " - "~n Error Type: ~p" - "~n Error: ~p" - "~n StackTrace: ~p", [T, E, StackTrace]), + "~n Error Class: ~p" + "~n Error: ~p" + "~n StackTrace: ~p", [C, E, S]), %% And now, *try* to cleanup (catch stop_node(Node)), - ?FAIL({failed_starting_agent, T, E, StackTrace}) + ?FAIL({failed_starting_agent, C, E, S}) end. diff --git a/lib/snmp/test/snmp_test_lib.erl b/lib/snmp/test/snmp_test_lib.erl index 290f1bc31a..a483690653 100644 --- a/lib/snmp/test/snmp_test_lib.erl +++ b/lib/snmp/test/snmp_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2015. All Rights Reserved. +%% Copyright Ericsson AB 2002-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -41,7 +41,7 @@ -export([watchdog/3, watchdog_start/1, watchdog_start/2, watchdog_stop/1]). -export([del_dir/1]). -export([cover/1]). --export([p/2, print/5, formated_timestamp/0]). +-export([p/2, print1/2, print2/2, print/5, formated_timestamp/0]). %% ---------------------------------------------------------------------- @@ -58,12 +58,67 @@ from(H, [H | T]) -> T; from(H, [_ | T]) -> from(H, T); from(_H, []) -> []. +%% localhost() -> +%% {ok, Ip} = snmp_misc:ip(net_adm:localhost()), +%% Ip. +%% localhost(Family) -> +%% {ok, Ip} = snmp_misc:ip(net_adm:localhost(), Family), +%% Ip. + localhost() -> - {ok, Ip} = snmp_misc:ip(net_adm:localhost()), - Ip. + localhost(inet). + localhost(Family) -> - {ok, Ip} = snmp_misc:ip(net_adm:localhost(), Family), - Ip. + case inet:getaddr(net_adm:localhost(), Family) of + {ok, {127, _, _, _}} when (Family =:= inet) -> + %% Ouch, we need to use something else + case inet:getifaddrs() of + {ok, IfList} -> + which_addr(Family, IfList); + {error, Reason1} -> + fail({getifaddrs, Reason1}, ?MODULE, ?LINE) + end; + {ok, {0, _, _, _, _, _, _, _}} when (Family =:= inet6) -> + %% Ouch, we need to use something else + case inet:getifaddrs() of + {ok, IfList} -> + which_addr(Family, IfList); + {error, Reason1} -> + fail({getifaddrs, Reason1}, ?MODULE, ?LINE) + end; + {ok, Addr} -> + Addr; + {error, Reason2} -> + fail({getaddr, Reason2}, ?MODULE, ?LINE) + end. + +which_addr(_Family, []) -> + fail(no_valid_addr, ?MODULE, ?LINE); +which_addr(Family, [{"lo", _} | IfList]) -> + which_addr(Family, IfList); +which_addr(Family, [{"docker" ++ _, _} | IfList]) -> + which_addr(Family, IfList); +which_addr(Family, [{"br-" ++ _, _} | IfList]) -> + which_addr(Family, IfList); +which_addr(Family, [{_Name, IfOpts} | IfList]) -> + case which_addr2(Family, IfOpts) of + {ok, Addr} -> + Addr; + {error, _} -> + which_addr(Family, IfList) + end. + +which_addr2(_Family, []) -> + {error, not_found}; +which_addr2(Family, [{addr, Addr}|_]) + when (Family =:= inet) andalso (size(Addr) =:= 4) -> + {ok, Addr}; +which_addr2(Family, [{addr, Addr}|_]) + when (Family =:= inet6) andalso (size(Addr) =:= 8) -> + {ok, Addr}; +which_addr2(Family, [_|IfOpts]) -> + which_addr2(Family, IfOpts). + sz(L) when is_list(L) -> length(L); @@ -605,19 +660,30 @@ p(Mod, Case) when is_atom(Mod) andalso is_atom(Case) -> p(F, A) when is_list(F) andalso is_list(A) -> io:format(user, F ++ "~n", A). +%% This is just a bog standard printout, with a (formatted) timestamp +%% prefix and a newline after. +%% print1 - prints to both standard_io and user. +%% print2 - prints to just standard_io. + +print_format(F, A) -> + FTS = snmp_test_lib:formated_timestamp(), + io_lib:format("[~s] " ++ F ++ "~n", [FTS | A]). + +print1(F, A) -> + S = print_format(F, A), + io:format("~s", [S]), + io:format(user, "~s", [S]). + +print2(F, A) -> + S = print_format(F, A), + io:format("~s", [S]). + + print(Prefix, Module, Line, Format, Args) -> io:format("*** [~s] ~s ~p ~p ~p:~p *** " ++ Format ++ "~n", [formated_timestamp(), Prefix, node(), self(), Module, Line|Args]). formated_timestamp() -> - format_timestamp(os:timestamp()). - -format_timestamp({_N1, _N2, N3} = Now) -> - {Date, Time} = calendar:now_to_datetime(Now), - {YYYY,MM,DD} = Date, - {Hour,Min,Sec} = Time, - FormatDate = - io_lib:format("~.4w:~.2.0w:~.2.0w ~.2.0w:~.2.0w:~.2.0w ~w", - [YYYY,MM,DD,Hour,Min,Sec,round(N3/1000)]), - lists:flatten(FormatDate). + snmp_misc:formated_timestamp(). + diff --git a/lib/snmp/test/snmp_test_lib.hrl b/lib/snmp/test/snmp_test_lib.hrl index 7acebee1f1..335f3fff3c 100644 --- a/lib/snmp/test/snmp_test_lib.hrl +++ b/lib/snmp/test/snmp_test_lib.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2015. All Rights Reserved. +%% Copyright Ericsson AB 2002-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -127,24 +127,31 @@ -endif. -ifdef(snmp_debug). --define(DBG(F,A),?PRINT("DBG",F,A)). +-define(DBG(F,A), ?PRINT("DBG", F, A)). -else. --define(DBG(F,A),ok). +-define(DBG(F,A), ok). -endif. -ifdef(snmp_log). --define(LOG(F,A),?PRINT("LOG",F,A)). +-define(LOG(F,A), ?PRINT("LOG", F, A)). -else. --define(LOG(F,A),ok). +-define(LOG(F,A), ok). -endif. -ifdef(snmp_error). --define(ERR(F,A),?PRINT("ERR",F,A)). +-define(ERR(F,A), ?PRINT("ERR", F, A)). -else. --define(ERR(F,A),ok). +-define(ERR(F,A), ok). -endif. --define(INF(F,A),?PRINT("INF",F,A)). +-define(INF(F,A), ?PRINT("INF", F, A)). -define(PRINT(P,F,A), - snmp_test_lib:print(P,?MODULE,?LINE,F,A)). + snmp_test_lib:print(P, ?MODULE, ?LINE, F, A)). + +-define(PRINT1(F, A), snmp_test_lib:print1(F, A)). +-define(EPRINT1(F, A), ?PRINT1("<ERROR> " ++ F, A)). + +-define(PRINT2(F, A), snmp_test_lib:print2(F, A)). +-define(EPRINT2(F, A), ?PRINT2("<ERROR> " ++ F, A)). + diff --git a/lib/snmp/test/snmp_test_mgr.erl b/lib/snmp/test/snmp_test_mgr.erl index 36637d5cf4..73a4d56084 100644 --- a/lib/snmp/test/snmp_test_mgr.erl +++ b/lib/snmp/test/snmp_test_mgr.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2015. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,8 +20,10 @@ -module(snmp_test_mgr). + %%---------------------------------------------------------------------- -%% This module implements a simple SNMP manager for Erlang. +%% This module implements a simple SNMP manager for Erlang. Its used +%% during by the agent test suite. %%---------------------------------------------------------------------- %% c(snmp_test_mgr). @@ -49,16 +51,17 @@ -include_lib("snmp/include/snmp_types.hrl"). -include_lib("snmp/include/STANDARD-MIB.hrl"). - --record(state,{dbg = true, - quiet, - parent, - timeout = 3500, - print_traps = true, - mini_mib, - packet_server, - last_sent_pdu, - last_received_pdu}). +-include("snmp_test_lib.hrl"). + +-record(state, {dbg = true, + quiet, + parent, + timeout = 3500, + print_traps = true, + mini_mib, + packet_server, + last_sent_pdu, + last_received_pdu}). -define(SERVER, ?MODULE). -define(PACK_SERV, snmp_test_mgr_misc). @@ -197,27 +200,28 @@ init({Options, CallerPid}) -> put(debug, get_value(debug, Options, false)), d("init -> (~p) extract options",[self()]), PacksDbg = get_value(packet_server_debug, Options, false), - io:format("[~w] ~p -> PacksDbg: ~p~n", [?MODULE, self(), PacksDbg]), + print("[~w] ~p -> PacksDbg: ~p~n", [?MODULE, self(), PacksDbg]), RecBufSz = get_value(recbuf, Options, 1024), - io:format("[~w] ~p -> RecBufSz: ~p~n", [?MODULE, self(), RecBufSz]), + print("[~w] ~p -> RecBufSz: ~p~n", [?MODULE, self(), RecBufSz]), Mibs = get_value(mibs, Options, []), - io:format("[~w] ~p -> Mibs: ~p~n", [?MODULE, self(), Mibs]), + print("[~w] ~p -> Mibs: ~p~n", [?MODULE, self(), Mibs]), Udp = get_value(agent_udp, Options, 4000), - io:format("[~w] ~p -> Udp: ~p~n", [?MODULE, self(), Udp]), + print("[~w] ~p -> Udp: ~p~n", [?MODULE, self(), Udp]), User = get_value(user, Options, "initial"), - io:format("[~w] ~p -> User: ~p~n", [?MODULE, self(), User]), + print("[~w] ~p -> User: ~p~n", [?MODULE, self(), User]), EngineId = get_value(engine_id, Options, "agentEngine"), - io:format("[~w] ~p -> EngineId: ~p~n", [?MODULE, self(), EngineId]), + print("[~w] ~p -> EngineId: ~p~n", [?MODULE, self(), EngineId]), CtxEngineId = get_value(context_engine_id, Options, EngineId), - io:format("[~w] ~p -> CtxEngineId: ~p~n", [?MODULE, self(), CtxEngineId]), + print("[~w] ~p -> CtxEngineId: ~p~n", [?MODULE, self(), CtxEngineId]), TrapUdp = get_value(trap_udp, Options, 5000), - io:format("[~w] ~p -> TrapUdp: ~p~n", [?MODULE, self(), TrapUdp]), + print("[~w] ~p -> TrapUdp: ~p~n", [?MODULE, self(), TrapUdp]), Dir = get_value(dir, Options, "."), - io:format("[~w] ~p -> Dir: ~p~n", [?MODULE, self(), Dir]), + print("[~w] ~p -> Dir: ~p~n", [?MODULE, self(), Dir]), SecLevel = get_value(sec_level, Options, noAuthNoPriv), - io:format("[~w] ~p -> SecLevel: ~p~n", [?MODULE, self(), SecLevel]), + print("[~w] ~p -> SecLevel: ~p~n", [?MODULE, self(), SecLevel]), MiniMIB = snmp_mini_mib:create(Mibs), - io:format("[~w] ~p -> MiniMIB: ~p~n", [?MODULE, self(), MiniMIB]), + d("[~w] ~p -> MiniMIB: " + "~n ~p", [?MODULE, self(), MiniMIB]), Version = case lists:member(v2, Options) of true -> 'version-2'; false -> @@ -226,19 +230,19 @@ init({Options, CallerPid}) -> false -> 'version-1' end end, - io:format("[~w] ~p -> Version: ~p~n", [?MODULE, self(), Version]), + print("[~w] ~p -> Version: ~p~n", [?MODULE, self(), Version]), Com = case Version of 'version-3' -> get_value(context, Options, ""); _ -> get_value(community, Options, "public") end, - io:format("[~w] ~p -> Com: ~p~n", [?MODULE, self(), Com]), + print("[~w] ~p -> Com: ~p~n", [?MODULE, self(), Com]), VsnHdrD = {Com, User, EngineId, CtxEngineId, mk_seclevel(SecLevel)}, - io:format("[~w] ~p -> VsnHdrD: ~p~n", [?MODULE, self(), VsnHdrD]), + print("[~w] ~p -> VsnHdrD: ~p~n", [?MODULE, self(), VsnHdrD]), IpFamily = get_value(ipfamily, Options, inet), - io:format("[~w] ~p -> IpFamily: ~p~n", [?MODULE, self(), IpFamily]), + print("[~w] ~p -> IpFamily: ~p~n", [?MODULE, self(), IpFamily]), AgIp = case snmp_misc:assq(agent, Options) of {value, Tuple4} when is_tuple(Tuple4) andalso (size(Tuple4) =:= 4) -> @@ -247,9 +251,9 @@ init({Options, CallerPid}) -> {ok, Ip} = snmp_misc:ip(Host, IpFamily), Ip end, - io:format("[~w] ~p -> AgIp: ~p~n", [?MODULE, self(), AgIp]), + print("[~w] ~p -> AgIp: ~p~n", [?MODULE, self(), AgIp]), Quiet = lists:member(quiet, Options), - io:format("[~w] ~p -> Quiet: ~p~n", [?MODULE, self(), Quiet]), + print("[~w] ~p -> Quiet: ~p~n", [?MODULE, self(), Quiet]), PackServ = start_packet_server( Quiet, Options, CallerPid, AgIp, Udp, TrapUdp, @@ -443,7 +447,8 @@ handle_cast({bulk, Args}, State) -> {noreply, execute_request(bulk, Args, State)}; handle_cast({response, RespPdu}, State) -> - d("handle_cast -> response request with ~p", [RespPdu]), + d("handle_cast -> response request with " + "~n ~p", [RespPdu]), ?PACK_SERV:send_pdu(RespPdu, State#state.packet_server), {noreply, State}; @@ -1126,14 +1131,15 @@ sizeOf(L) when is_list(L) -> sizeOf(B) when is_binary(B) -> size(B). -d(F,A) -> d(get(debug),F,A). +d(F, A) -> d(get(debug), F, A). -d(true,F,A) -> - io:format("*** [~s] MGR_DBG *** " ++ F ++ "~n", - [formated_timestamp()|A]); +d(true, F, A) -> + print(F, A); d(_,_F,_A) -> ok. +print(F, A) -> + ?PRINT2("MGR " ++ F, A). formated_timestamp() -> snmp_test_lib:formated_timestamp(). diff --git a/lib/snmp/test/snmp_test_mgr_misc.erl b/lib/snmp/test/snmp_test_mgr_misc.erl index 274fb5be26..315e3ebd9e 100644 --- a/lib/snmp/test/snmp_test_mgr_misc.erl +++ b/lib/snmp/test/snmp_test_mgr_misc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -38,11 +38,14 @@ -define(SNMP_USE_V3, true). -include_lib("snmp/include/snmp_types.hrl"). +-include_lib("snmp/src/misc/snmp_verbosity.hrl"). +-include("snmp_test_lib.hrl"). %%---------------------------------------------------------------------- %% The InHandler process will receive messages on the form {snmp_pdu, Pdu}. %%---------------------------------------------------------------------- + start_link_packet( InHandler, AgentIp, UdpPort, TrapUdp, VsnHdr, Version, Dir, BufSz) -> start_link_packet( @@ -101,11 +104,11 @@ init_packet( DbgOptions, IpFamily) -> put(sname, mgr_misc), init_debug(DbgOptions), - {ok, UdpId} = - gen_udp:open(TrapUdp, [{recbuf,BufSz}, {reuseaddr, true}, IpFamily]), + UdpOpts = [{recbuf,BufSz}, {reuseaddr, true}, IpFamily], + {ok, UdpId} = gen_udp:open(TrapUdp, UdpOpts), put(msg_id, 1), - proc_lib:init_ack(Parent, self()), init_usm(Version, Dir), + proc_lib:init_ack(Parent, self()), packet_loop(SnmpMgr, UdpId, AgentIp, UdpPort, VsnHdr, Version, []). init_debug(Dbg) when is_atom(Dbg) -> @@ -200,86 +203,24 @@ packet_loop(SnmpMgr, UdpId, AgentIp, UdpPort, VsnHdr, Version, MsgData) -> handle_udp_packet(_V, undefined, UdpId, Ip, UdpPort, Bytes, SnmpMgr, AgentIp) -> - M = (catch snmp_pdus:dec_message_only(Bytes)), - MsgData3 = - case M of - Message when Message#message.version =:= 'version-3' -> - d("handle_udp_packet -> version 3"), - case catch handle_v3_msg(Bytes, Message) of - {ok, NewData, MsgData2} -> - Msg = Message#message{data = NewData}, - case SnmpMgr of - {pdu, Pid} -> - Pdu = get_pdu(Msg), - d("packet_loop -> " - "send pdu to manager (~w): ~p", [Pid, Pdu]), - Pid ! {snmp_pdu, Pdu}; - {msg, Pid} -> - d("packet_loop -> " - "send msg to manager (~w): ~p", [Pid, Msg]), - Pid ! {snmp_msg, Msg, Ip, UdpPort} - end, - MsgData2; - {error, Reason, B} -> - udp_send(UdpId, AgentIp, UdpPort, B), - error("Decoding error. Auto-sending Report.\n" - "Reason: ~w " - "(UDPport: ~w, Ip: ~w)", - [Reason, UdpPort, Ip]), - []; - {error, Reason} -> - error("Decoding error. " - "Bytes: ~w ~n Reason: ~w " - "(UDPport: ~w, Ip: ~w)", - [Bytes, Reason, UdpPort, Ip]), - [] - end; - Message when is_record(Message, message) -> - %% v1 or v2c - d("handle_udp_packet -> version v1 or v2c"), - case catch snmp_pdus:dec_pdu(Message#message.data) of - Pdu when is_record(Pdu, pdu) -> - case SnmpMgr of - {pdu, Pid} -> - d("handle_udp_packet -> " - "send pdu to manager (~w): ~p", - [Pid, Pdu]), - Pid ! {snmp_pdu, Pdu}; - {msg, Pid} -> - d("handle_udp_packet -> " - "send pdu-msg to manager (~w): ~p", - [Pid, Pdu]), - Msg = Message#message{data = Pdu}, - Pid ! {snmp_msg, Msg, Ip, UdpPort} - end; - Pdu when is_record(Pdu, trappdu) -> - case SnmpMgr of - {pdu, Pid} -> - d("handle_udp_packet -> " - "send trap to manager (~w): ~p", - [Pid, Pdu]), - Pid ! {snmp_pdu, Pdu}; - {msg, Pid} -> - d("handle_udp_packet -> " - "send trap-msg to manager (~w): ~p", - [Pid, Pdu]), - Msg = Message#message{data = Pdu}, - Pid ! {snmp_msg, Msg, Ip, UdpPort} - end; - Reason -> - error("Decoding error. " - "Bytes: ~w ~n Reason: ~w " - "(UDPport: ~w, Ip: ~w)", - [Bytes, Reason, UdpPort, Ip]) - end, - []; - Reason -> - error("Decoding error. Bytes: ~w ~n Reason: ~w " - "(UDPport: ~w, Ip: ~w)", - [Bytes, Reason, UdpPort, Ip]), - [] - end, - MsgData3; + try snmp_pdus:dec_message_only(Bytes) of + Message when Message#message.version =:= 'version-3' -> + d("handle_udp_packet -> version 3"), + handle_v3_message(SnmpMgr, UdpId, Ip, UdpPort, AgentIp, + Bytes, Message); + + Message when is_record(Message, message) -> + d("handle_udp_packet -> version 1 or 2"), + handle_v1_or_v2_message(SnmpMgr, UdpId, Ip, UdpPort, AgentIp, + Bytes, Message) + + catch + Class:Error:_ -> + error("Decoding error (~w). Bytes: ~w ~n Error: ~w " + "(UDPport: ~w, Ip: ~w)", + [Class, Bytes, Error, UdpPort, Ip]), + [] + end; handle_udp_packet(V, {DiscoReqMsg, From}, _UdpId, _Ip, _UdpPort, Bytes, _, _AgentIp) -> DiscoRspMsg = (catch snmp_pdus:dec_message(Bytes)), @@ -297,6 +238,88 @@ handle_udp_packet(V, {DiscoReqMsg, From}, _UdpId, _Ip, _UdpPort, [] end. +handle_v3_message(Mgr, UdpId, Ip, UdpPort, AgentIp, + Bytes, Message) -> + try handle_v3_msg(Bytes, Message) of + {ok, NewData, MsgData} -> + Msg = Message#message{data = NewData}, + case Mgr of + {pdu, Pid} -> + Pdu = get_pdu(Msg), + d("handle_v3_message -> send pdu to manager (~p): " + "~n ~p", [Pid, Pdu]), + Pid ! {snmp_pdu, Pdu}; + {msg, Pid} -> + d("handle_v3_message -> send msg to manager (~p): " + "~n ~p", [Pid, Msg]), + Pid ! {snmp_msg, Msg, Ip, UdpPort} + end, + MsgData + + catch + throw:{error, Reason, B}:_ -> + udp_send(UdpId, AgentIp, UdpPort, B), + error("Decoding (v3) error. Auto-sending Report.\n" + "~n Reason: ~w " + "(UDPport: ~w, Ip: ~w)", + [Reason, UdpPort, Ip]), + []; + + throw:{error, Reason}:_ -> + error("Decoding (v3) error. " + "~n Bytes: ~w" + "~n Reason: ~w " + "(UDPport: ~w, Ip: ~w)", + [Bytes, Reason, UdpPort, Ip]), + []; + + Class:Error:_ -> + error("Decoding (v3) error (~w). " + "~n Bytes: ~w" + "~n Error: ~w " + "(UDPport: ~w, Ip: ~w)", + [Class, Bytes, Error, UdpPort, Ip]), + [] + + end. + +handle_v1_or_v2_message(Mgr, _UdpId, Ip, UdpPort, _AgentIp, + Bytes, Message) -> + try snmp_pdus:dec_pdu(Message#message.data) of + Pdu when is_record(Pdu, pdu) -> + case Mgr of + {pdu, Pid} -> + d("handle_v1_or_v2_message -> send pdu to manager (~p): " + "~n ~p", [Pid, Pdu]), + Pid ! {snmp_pdu, Pdu}; + {msg, Pid} -> + d("handle_v1_or_v2_message -> send msg to manager (~p): " + "~n ~p", [Pid, Pdu]), + Msg = Message#message{data = Pdu}, + Pid ! {snmp_msg, Msg, Ip, UdpPort} + end; + Pdu when is_record(Pdu, trappdu) -> + case Mgr of + {pdu, Pid} -> + d("handle_v1_or_v2_message -> send trap-pdu to manager (~p): " + "~n ~p", [Pid, Pdu]), + Pid ! {snmp_pdu, Pdu}; + {msg, Pid} -> + d("handle_v1_or_v2_message -> send trap-msg to manager (~p): " + "~n ~p", [Pid, Pdu]), + Msg = Message#message{data = Pdu}, + Pid ! {snmp_msg, Msg, Ip, UdpPort} + end + + catch + Class:Error:_ -> + error("Decoding (v1 or v2) error (~w): " + "~n Bytes: ~w" + "~n Error: ~w " + "(UDPport: ~w, Ip: ~w)", + [Class, Bytes, Error, UdpPort, Ip]) + end. + %% This function assumes that the agent and the manager (thats us) %% has the same version. @@ -578,18 +601,100 @@ set_pdu(Msg, RePdu) -> init_usm('version-3', Dir) -> + ?vlog("init_usm -> create (and init) fake \"agent\" table", []), ets:new(snmp_agent_table, [set, public, named_table]), ets:insert(snmp_agent_table, {agent_mib_storage, persistent}), - snmpa_local_db:start_link(normal, Dir, [{verbosity,trace}]), + %% The local-db process may *still* be running (from a previous + %% test case), on the way down, but not yet dead. + %% Either way, before we start it, make sure its dead and *gone*! + %% How do we do that without getting hung up? Calling the stop + %% function, will not do since it uses Timeout=infinity. + ?vlog("init_usm -> ensure (old) fake local-db is dead", []), + ensure_local_db_dead(), + ?vlog("init_usm -> try start fake local-db", []), + case snmpa_local_db:start_link(normal, Dir, + [{sname, "MGR-LOCAL-DB"}, + {verbosity, trace}]) of + {ok, Pid} -> + ?vlog("started: ~p" + "~n ~p", [Pid, process_info(Pid)]); + {error, {already_started, Pid}} -> + LDBInfo = process_info(Pid), + ?vlog("already started: ~p" + "~n ~p", [Pid, LDBInfo]), + ?FAIL({still_running, snmpa_local_db, LDBInfo}); + {error, Reason} -> + ?FAIL({failed_starting, snmpa_local_db, Reason}) + end, NameDb = snmpa_agent:db(snmpEngineID), + ?vlog("init_usm -> try set manager engine-id", []), R = snmp_generic:variable_set(NameDb, "mgrEngine"), - io:format("~w:init_usm -> engine-id set result: ~p~n", [?MODULE,R]), + snmp_verbosity:print(info, info, "init_usm -> engine-id set result: ~p", [R]), + ?vlog("init_usm -> try set engine boots (framework-mib)", []), snmp_framework_mib:set_engine_boots(1), + ?vlog("init_usm -> try set engine time (framework-mib)", []), snmp_framework_mib:set_engine_time(1), - snmp_user_based_sm_mib:reconfigure(Dir); + ?vlog("init_usm -> try usm (mib) reconfigure", []), + snmp_user_based_sm_mib:reconfigure(Dir), + ?vlog("init_usm -> done", []), + ok; init_usm(_Vsn, _Dir) -> ok. +ensure_local_db_dead() -> + ensure_dead(whereis(snmpa_local_db), 2000). + +ensure_dead(Pid, Timeout) when is_pid(Pid) -> + MRef = erlang:monitor(process, Pid), + try + begin + ensure_dead_wait(Pid, MRef, Timeout), + ensure_dead_stop(Pid, MRef, Timeout), + ensure_dead_kill(Pid, MRef, Timeout), + exit(failed_stop_local_db) + end + catch + throw:ok -> + ok + end; +ensure_dead(_, _) -> + ?vlog("ensure_dead -> already dead", []), + ok. + +ensure_dead_wait(Pid, MRef, Timeout) -> + receive + {'DOWN', MRef, process, Pid, _Info} -> + ?vlog("ensure_dead_wait -> died peacefully", []), + throw(ok) + after Timeout -> + ?vlog("ensure_dead_wait -> giving up", []), + ok + end. + +ensure_dead_stop(Pid, MRef, Timeout) -> + StopPid = spawn(fun() -> snmpa_local_db:stop() end), + receive + {'DOWN', MRef, process, Pid, _Info} -> + ?vlog("ensure_dead -> dead (stopped)", []), + throw(ok) + after Timeout -> + ?vlog("ensure_dead_stop -> giving up", []), + exit(StopPid, kill), + ok + end. + +ensure_dead_kill(Pid, MRef, Timeout) -> + exit(Pid, kill), + receive + {'DOWN', MRef, process, Pid, _Info} -> + ?vlog("ensure_dead -> dead (killed)", []), + throw(ok) + after Timeout -> + ?vlog("ensure_dead_kill -> giving up", []), + ok + end. + + display_incomming_message(M) -> display_message("Incomming",M). @@ -782,13 +887,13 @@ sz(O) -> {unknown_size, O}. d(F) -> d(F, []). -d(F,A) -> d(get(debug),F,A). +d(F,A) -> d(get(debug), F, A). -d(true,F,A) -> - io:format("*** [~s] MGR_PS_DBG *** " ++ F ++ "~n", - [formated_timestamp()|A]); +d(true, F, A) -> + print(F, A); d(_,_F,_A) -> ok. -formated_timestamp() -> - snmp_test_lib:formated_timestamp(). +print(F, A) -> + ?PRINT2("MGR_PS " ++ F, A). + diff --git a/lib/snmp/test/test-mibs/ALARM-MIB.mib b/lib/snmp/test/test-mibs/ALARM-MIB.mib index 18e43d4b4b..10d2adbff9 100644 --- a/lib/snmp/test/test-mibs/ALARM-MIB.mib +++ b/lib/snmp/test/test-mibs/ALARM-MIB.mib @@ -330,7 +330,7 @@ alarmModelRowStatus OBJECT-TYPE cannot be used as an alarm suppression mechanism. Entries that are notInService will disappear as described in RFC2579. - This row can not be modified while it is being + This row cannot be modified while it is being referenced by a value of alarmActiveModelPointer. In these cases, an error of `inconsistentValue' will be returned to the manager. diff --git a/lib/snmp/test/test-mibs/SNMPv2-TC.mib b/lib/snmp/test/test-mibs/SNMPv2-TC.mib index fd6a728ab5..1d75c4bbd8 100644 --- a/lib/snmp/test/test-mibs/SNMPv2-TC.mib +++ b/lib/snmp/test/test-mibs/SNMPv2-TC.mib @@ -454,7 +454,7 @@ value | | see 1| ->C| ->D associated with this column or that there is no conceptual row for which this column would be accessible in the MIB view used by the retrieval. As - such, the management station can not issue any + such, the management station cannot issue any management protocol set operations to create an instance of this column. @@ -576,7 +576,7 @@ value | | see 1| ->C| ->D associated with this column or that there is no conceptual row for which this column would be accessible in the MIB view used by the retrieval. As - such, the management station can not issue any + such, the management station cannot issue any management protocol set operations to create an instance of this column. diff --git a/lib/snmp/vsn.mk b/lib/snmp/vsn.mk index 4d5a0fbce8..f305497cd3 100644 --- a/lib/snmp/vsn.mk +++ b/lib/snmp/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = snmp -SNMP_VSN = 5.2.12 +SNMP_VSN = 5.3 PRE_VSN = APP_VSN = "$(APPLICATION)-$(SNMP_VSN)$(PRE_VSN)" diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 1bc4f866ce..78990c48f2 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -2995,7 +2995,7 @@ </item> <item> <p> - Fixed internal error on when client and server can not + Fixed internal error on when client and server cannot agree o which authmethod to use.</p> <p> Own Id: OTP-10731 Aux Id: seq12237 </p> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index 1a53a2ea98..3fd6eae423 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -46,7 +46,7 @@ In that encrypted connection one or more channels could be opened with <seealso marker="ssh_connection#session_channel/2">ssh_connection:session_channel/2,4</seealso>. </p> - <p>Each channel is an isolated "pipe" between a client-side process and a server-side process. Thoose process + <p>Each channel is an isolated "pipe" between a client-side process and a server-side process. Those process pairs could handle for example file transfers (sftp) or remote command execution (shell, exec and/or cli). If a custom shell is implemented, the user of the client could execute the special commands remotely. Note that the user is not necessarily a human but probably a system interfacing the SSH app. diff --git a/lib/ssh/src/ssh.app.src b/lib/ssh/src/ssh.app.src index 410061cded..7449405d20 100644 --- a/lib/ssh/src/ssh.app.src +++ b/lib/ssh/src/ssh.app.src @@ -44,10 +44,10 @@ {env, []}, {mod, {ssh_app, []}}, {runtime_dependencies, [ - "crypto-4.2", - "erts-6.0", - "kernel-3.0", - "public_key-1.5.2", - "stdlib-3.3" + "crypto-@OTP-15644@", + "erts-9.0", + "kernel-5.3", + "public_key-1.6.1", + "stdlib-3.4.1" ]}]}. diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 923e9309f4..04453e6ef0 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -396,11 +396,13 @@ recv_mac_size = 0, encrypt = none, %% encrypt algorithm + encrypt_cipher, %% cipher. could be different from the algorithm encrypt_keys, %% encrypt keys encrypt_block_size = 8, encrypt_ctx, decrypt = none, %% decrypt algorithm + decrypt_cipher, %% cipher. could be different from the algorithm decrypt_keys, %% decrypt keys decrypt_block_size = 8, decrypt_ctx, %% Decryption context diff --git a/lib/ssh/src/ssh_sftpd.erl b/lib/ssh/src/ssh_sftpd.erl index aa9ba0f9bb..5ec12e2d04 100644 --- a/lib/ssh/src/ssh_sftpd.erl +++ b/lib/ssh/src/ssh_sftpd.erl @@ -508,7 +508,7 @@ close_our_file({_,Fd}, FileMod, FS0) -> FS1. %%% stat: do the stat -stat(Vsn, ReqId, Data, State, F) -> +stat(_Vsn, ReqId, Data, State, F) -> <<?UINT32(BLen), BPath:BLen/binary, _/binary>> = Data, stat(ReqId, unicode:characters_to_list(BPath), State, F). diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index 9ff20454cd..eaab13433a 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -162,15 +162,15 @@ supported_algorithms(cipher) -> select_crypto_supported( [ {'[email protected]', [{ciphers,chacha20}, {macs,poly1305}]}, - {'[email protected]', [{ciphers,{aes_gcm,256}}]}, - {'aes256-ctr', [{ciphers,{aes_ctr,256}}]}, - {'aes192-ctr', [{ciphers,{aes_ctr,192}}]}, - {'[email protected]', [{ciphers,{aes_gcm,128}}]}, - {'aes128-ctr', [{ciphers,{aes_ctr,128}}]}, - {'AEAD_AES_256_GCM', [{ciphers,{aes_gcm,256}}]}, - {'AEAD_AES_128_GCM', [{ciphers,{aes_gcm,128}}]}, - {'aes128-cbc', [{ciphers,aes_cbc128}]}, - {'3des-cbc', [{ciphers,des3_cbc}]} + {'[email protected]', [{ciphers,aes_256_gcm}]}, + {'aes256-ctr', [{ciphers,aes_256_ctr}]}, + {'aes192-ctr', [{ciphers,aes_192_ctr}]}, + {'[email protected]', [{ciphers,aes_128_gcm}]}, + {'aes128-ctr', [{ciphers,aes_128_ctr}]}, + {'AEAD_AES_256_GCM', [{ciphers,aes_256_gcm}]}, + {'AEAD_AES_128_GCM', [{ciphers,aes_128_gcm}]}, + {'aes128-cbc', [{ciphers,aes_128_cbc}]}, + {'3des-cbc', [{ciphers,des_ede3_cbc}]} ] )); supported_algorithms(mac) -> @@ -179,8 +179,8 @@ supported_algorithms(mac) -> [{'hmac-sha2-256', [{macs,hmac}, {hashs,sha256}]}, {'hmac-sha2-512', [{macs,hmac}, {hashs,sha512}]}, {'hmac-sha1', [{macs,hmac}, {hashs,sha}]}, - {'AEAD_AES_128_GCM', [{ciphers,{aes_gcm,128}}]}, - {'AEAD_AES_256_GCM', [{ciphers,{aes_gcm,256}}]} + {'AEAD_AES_128_GCM', [{ciphers,aes_128_gcm}]}, + {'AEAD_AES_256_GCM', [{ciphers,aes_256_gcm}]} ] )); supported_algorithms(compression) -> @@ -1256,11 +1256,6 @@ get_length(aead, EncryptedBuffer, Ssh) -> end. -pkt_type('AEAD_AES_128_GCM') -> aead; -pkt_type('AEAD_AES_256_GCM') -> aead; -pkt_type('[email protected]') -> aead; -pkt_type(_) -> common. - payload(<<PacketLen:32, PaddingLen:8, PayloadAndPadding/binary>>) -> PayloadLen = PacketLen - PaddingLen - 1, <<Payload:PayloadLen/binary, _/binary>> = PayloadAndPadding, @@ -1323,231 +1318,163 @@ verify(PlainText, HashAlg, Sig, Key, _) -> %%% Unit: bytes --record(cipher_data, { - key_bytes, - iv_bytes, - block_bytes - }). +-record(cipher, { + impl, + key_bytes, + iv_bytes, + block_bytes, + pkt_type = common + }). %%% Start of a more parameterized crypto handling. cipher('AEAD_AES_128_GCM') -> - #cipher_data{key_bytes = 16, - iv_bytes = 12, - block_bytes = 16}; + #cipher{impl = aes_128_gcm, + key_bytes = 16, + iv_bytes = 12, + block_bytes = 16, + pkt_type = aead}; cipher('AEAD_AES_256_GCM') -> - #cipher_data{key_bytes = 32, - iv_bytes = 12, - block_bytes = 16}; + #cipher{impl = aes_256_gcm, + key_bytes = 32, + iv_bytes = 12, + block_bytes = 16, + pkt_type = aead}; cipher('3des-cbc') -> - #cipher_data{key_bytes = 24, - iv_bytes = 8, - block_bytes = 8}; + #cipher{impl = des_ede3_cbc, + key_bytes = 24, + iv_bytes = 8, + block_bytes = 8}; cipher('aes128-cbc') -> - #cipher_data{key_bytes = 16, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_128_cbc, + key_bytes = 16, + iv_bytes = 16, + block_bytes = 16}; cipher('aes128-ctr') -> - #cipher_data{key_bytes = 16, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_128_ctr, + key_bytes = 16, + iv_bytes = 16, + block_bytes = 16}; cipher('aes192-ctr') -> - #cipher_data{key_bytes = 24, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_192_ctr, + key_bytes = 24, + iv_bytes = 16, + block_bytes = 16}; cipher('aes256-ctr') -> - #cipher_data{key_bytes = 32, - iv_bytes = 16, - block_bytes = 16}; + #cipher{impl = aes_256_ctr, + key_bytes = 32, + iv_bytes = 16, + block_bytes = 16}; cipher('[email protected]') -> % FIXME: Verify!! - #cipher_data{key_bytes = 32, - iv_bytes = 12, - block_bytes = 8}. - + #cipher{impl = chacha20_poly1305, + key_bytes = 32, + iv_bytes = 12, + block_bytes = 8, + pkt_type = aead}; + +cipher(_) -> + #cipher{}. + + +pkt_type(SshCipher) -> (cipher(SshCipher))#cipher.pkt_type. + +decrypt_magic(server) -> {"A", "C"}; +decrypt_magic(client) -> {"B", "D"}. + +encrypt_magic(client) -> decrypt_magic(server); +encrypt_magic(server) -> decrypt_magic(client). + encrypt_init(#ssh{encrypt = none} = Ssh) -> {ok, Ssh}; -encrypt_init(#ssh{encrypt = '[email protected]', role = client} = Ssh) -> + +encrypt_init(#ssh{encrypt = '[email protected]', role = Role} = Ssh) -> %% [email protected] uses two independent crypto streams, one (chacha20) %% for the length used in stream mode, and the other (chacha20-poly1305) as AEAD for %% the payload and to MAC the length||payload. %% See draft-josefsson-ssh-chacha20-poly1305-openssh-00 - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "C", 512), + {_, KeyMagic} = encrypt_magic(Role), + <<K2:32/binary,K1:32/binary>> = hash(Ssh, KeyMagic, 8*64), {ok, Ssh#ssh{encrypt_keys = {K1,K2} % encrypt_block_size = 16, %default = 8. What to set it to? 64 (openssl chacha.h) % ctx and iv is setup for each packet }}; -encrypt_init(#ssh{encrypt = '[email protected]', role = server} = Ssh) -> - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "D", 512), - {ok, Ssh#ssh{encrypt_keys = {K1,K2} - % encrypt_block_size = 16, %default = 8. What to set it to? - }}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_128_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:16/binary>> = hash(Ssh, "C", 128), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_128_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:16/binary>> = hash(Ssh, "D", 128), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_256_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:32/binary>> = hash(Ssh, "C", 256), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'AEAD_AES_256_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:32/binary>> = hash(Ssh, "D", 256), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = '3des-cbc', role = client} = Ssh) -> - IV = hash(Ssh, "A", 64), - <<K1:8/binary, K2:8/binary, K3:8/binary>> = hash(Ssh, "C", 192), - {ok, Ssh#ssh{encrypt_keys = {K1,K2,K3}, - encrypt_block_size = 8, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = '3des-cbc', role = server} = Ssh) -> - IV = hash(Ssh, "B", 64), - <<K1:8/binary, K2:8/binary, K3:8/binary>> = hash(Ssh, "D", 192), - {ok, Ssh#ssh{encrypt_keys = {K1,K2,K3}, - encrypt_block_size = 8, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'aes128-cbc', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:16/binary>> = hash(Ssh, "C", 128), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, + +encrypt_init(#ssh{encrypt = SshCipher, role = Role} = Ssh) when SshCipher == 'AEAD_AES_128_GCM'; + SshCipher == 'AEAD_AES_256_GCM' -> + {IvMagic, KeyMagic} = encrypt_magic(Role), + #cipher{impl = CryptoCipher, + key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), + {ok, Ssh#ssh{encrypt_cipher = CryptoCipher, + encrypt_keys = K, + encrypt_block_size = BlockBytes, encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'aes128-cbc', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:16/binary>> = hash(Ssh, "D", 128), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = IV}}; -encrypt_init(#ssh{encrypt = 'aes128-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:16/binary>> = hash(Ssh, "C", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes192-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:24/binary>> = hash(Ssh, "C", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes256-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:32/binary>> = hash(Ssh, "C", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes128-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:16/binary>> = hash(Ssh, "D", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes192-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:24/binary>> = hash(Ssh, "D", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}; -encrypt_init(#ssh{encrypt = 'aes256-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:32/binary>> = hash(Ssh, "D", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{encrypt_keys = K, - encrypt_block_size = 16, - encrypt_ctx = State}}. + +encrypt_init(#ssh{encrypt = SshCipher, role = Role} = Ssh) -> + {IvMagic, KeyMagic} = encrypt_magic(Role), + #cipher{impl = CryptoCipher, + key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), + Ctx0 = crypto:crypto_init(CryptoCipher, K, IV, true), + {ok, Ssh#ssh{encrypt_cipher = CryptoCipher, + encrypt_block_size = BlockBytes, + encrypt_ctx = Ctx0}}. encrypt_final(Ssh) -> - {ok, Ssh#ssh{encrypt = none, + {ok, Ssh#ssh{encrypt = none, encrypt_keys = undefined, encrypt_block_size = 8, encrypt_ctx = undefined }}. + encrypt(#ssh{encrypt = none} = Ssh, Data) -> {Ssh, Data}; + encrypt(#ssh{encrypt = '[email protected]', encrypt_keys = {K1,K2}, send_sequence = Seq} = Ssh, <<LenData:4/binary, PayloadData/binary>>) -> %% Encrypt length IV1 = <<0:8/unit:8, Seq:8/unit:8>>, - {_,EncLen} = crypto:stream_encrypt(crypto:stream_init(chacha20, K1, IV1), - LenData), + EncLen = crypto:crypto_one_time(chacha20, K1, IV1, LenData, true), %% Encrypt payload IV2 = <<1:8/little-unit:8, Seq:8/unit:8>>, - {_,EncPayloadData} = crypto:stream_encrypt(crypto:stream_init(chacha20, K2, IV2), - PayloadData), - + EncPayloadData = crypto:crypto_one_time(chacha20, K2, IV2, PayloadData, true), %% MAC tag - {_,PolyKey} = crypto:stream_encrypt(crypto:stream_init(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>), - <<0:32/unit:8>>), + PolyKey = crypto:crypto_one_time(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>, <<0:32/unit:8>>, true), EncBytes = <<EncLen/binary,EncPayloadData/binary>>, Ctag = crypto:poly1305(PolyKey, EncBytes), %% Result {Ssh, {EncBytes,Ctag}}; -encrypt(#ssh{encrypt = 'AEAD_AES_128_GCM', - encrypt_keys = K, + +encrypt(#ssh{encrypt = SshCipher, + encrypt_cipher = CryptoCipher, + encrypt_keys = K, encrypt_ctx = IV0} = Ssh, - <<LenData:4/binary, PayloadData/binary>>) -> - {Ctext,Ctag} = crypto:block_encrypt(aes_gcm, K, IV0, {LenData,PayloadData}), + <<LenData:4/binary, PayloadData/binary>>) when SshCipher == 'AEAD_AES_128_GCM' ; + SshCipher == 'AEAD_AES_256_GCM' -> + {Ctext,Ctag} = crypto:crypto_one_time_aead(CryptoCipher, K, IV0, PayloadData, LenData, true), IV = next_gcm_iv(IV0), {Ssh#ssh{encrypt_ctx = IV}, {<<LenData/binary,Ctext/binary>>,Ctag}}; -encrypt(#ssh{encrypt = 'AEAD_AES_256_GCM', - encrypt_keys = K, - encrypt_ctx = IV0} = Ssh, - <<LenData:4/binary, PayloadData/binary>>) -> - {Ctext,Ctag} = crypto:block_encrypt(aes_gcm, K, IV0, {LenData,PayloadData}), - IV = next_gcm_iv(IV0), - {Ssh#ssh{encrypt_ctx = IV}, {<<LenData/binary,Ctext/binary>>,Ctag}}; -encrypt(#ssh{encrypt = '3des-cbc', - encrypt_keys = {K1,K2,K3}, - encrypt_ctx = IV0} = Ssh, Data) -> - Enc = crypto:block_encrypt(des3_cbc, [K1,K2,K3], IV0, Data), - IV = crypto:next_iv(des3_cbc, Enc), - {Ssh#ssh{encrypt_ctx = IV}, Enc}; -encrypt(#ssh{encrypt = 'aes128-cbc', - encrypt_keys = K, - encrypt_ctx = IV0} = Ssh, Data) -> - Enc = crypto:block_encrypt(aes_cbc128, K,IV0,Data), - IV = crypto:next_iv(aes_cbc, Enc), - {Ssh#ssh{encrypt_ctx = IV}, Enc}; -encrypt(#ssh{encrypt = 'aes128-ctr', - encrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_encrypt(State0,Data), - {Ssh#ssh{encrypt_ctx = State}, Enc}; -encrypt(#ssh{encrypt = 'aes192-ctr', - encrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_encrypt(State0,Data), - {Ssh#ssh{encrypt_ctx = State}, Enc}; -encrypt(#ssh{encrypt = 'aes256-ctr', - encrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_encrypt(State0,Data), - {Ssh#ssh{encrypt_ctx = State}, Enc}. - + +encrypt(#ssh{encrypt_ctx = Ctx0} = Ssh, Data) -> + Enc = crypto:crypto_update(Ctx0, Data), + {Ssh, Enc}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Decryption @@ -1555,181 +1482,92 @@ encrypt(#ssh{encrypt = 'aes256-ctr', decrypt_init(#ssh{decrypt = none} = Ssh) -> {ok, Ssh}; -decrypt_init(#ssh{decrypt = '[email protected]', role = client} = Ssh) -> - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "D", 512), - {ok, Ssh#ssh{decrypt_keys = {K1,K2} - }}; -decrypt_init(#ssh{decrypt = '[email protected]', role = server} = Ssh) -> - <<K2:32/binary,K1:32/binary>> = hash(Ssh, "C", 512), + +decrypt_init(#ssh{decrypt = '[email protected]', role = Role} = Ssh) -> + {_, KeyMagic} = decrypt_magic(Role), + <<K2:32/binary,K1:32/binary>> = hash(Ssh, KeyMagic, 8*64), {ok, Ssh#ssh{decrypt_keys = {K1,K2} }}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_128_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:16/binary>> = hash(Ssh, "D", 128), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_128_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:16/binary>> = hash(Ssh, "C", 128), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_256_GCM', role = client} = Ssh) -> - IV = hash(Ssh, "B", 12*8), - <<K:32/binary>> = hash(Ssh, "D", 256), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = 'AEAD_AES_256_GCM', role = server} = Ssh) -> - IV = hash(Ssh, "A", 12*8), - <<K:32/binary>> = hash(Ssh, "C", 256), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, + +decrypt_init(#ssh{decrypt = SshCipher, role = Role} = Ssh) when SshCipher == 'AEAD_AES_128_GCM'; + SshCipher == 'AEAD_AES_256_GCM' -> + {IvMagic, KeyMagic} = decrypt_magic(Role), + #cipher{impl = CryptoCipher, + key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), + {ok, Ssh#ssh{decrypt_cipher = CryptoCipher, + decrypt_keys = K, + decrypt_block_size = BlockBytes, decrypt_ctx = IV}}; -decrypt_init(#ssh{decrypt = '3des-cbc', role = client} = Ssh) -> - {IV, KD} = {hash(Ssh, "B", 64), - hash(Ssh, "D", 192)}, - <<K1:8/binary, K2:8/binary, K3:8/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = {K1,K2,K3}, decrypt_ctx = IV, - decrypt_block_size = 8}}; -decrypt_init(#ssh{decrypt = '3des-cbc', role = server} = Ssh) -> - {IV, KD} = {hash(Ssh, "A", 64), - hash(Ssh, "C", 192)}, - <<K1:8/binary, K2:8/binary, K3:8/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = {K1, K2, K3}, decrypt_ctx = IV, - decrypt_block_size = 8}}; -decrypt_init(#ssh{decrypt = 'aes128-cbc', role = client} = Ssh) -> - {IV, KD} = {hash(Ssh, "B", 128), - hash(Ssh, "D", 128)}, - <<K:16/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = K, decrypt_ctx = IV, - decrypt_block_size = 16}}; -decrypt_init(#ssh{decrypt = 'aes128-cbc', role = server} = Ssh) -> - {IV, KD} = {hash(Ssh, "A", 128), - hash(Ssh, "C", 128)}, - <<K:16/binary>> = KD, - {ok, Ssh#ssh{decrypt_keys = K, decrypt_ctx = IV, - decrypt_block_size = 16}}; -decrypt_init(#ssh{decrypt = 'aes128-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:16/binary>> = hash(Ssh, "D", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes192-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:24/binary>> = hash(Ssh, "D", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes256-ctr', role = client} = Ssh) -> - IV = hash(Ssh, "B", 128), - <<K:32/binary>> = hash(Ssh, "D", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes128-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:16/binary>> = hash(Ssh, "C", 128), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes192-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:24/binary>> = hash(Ssh, "C", 192), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}; -decrypt_init(#ssh{decrypt = 'aes256-ctr', role = server} = Ssh) -> - IV = hash(Ssh, "A", 128), - <<K:32/binary>> = hash(Ssh, "C", 256), - State = crypto:stream_init(aes_ctr, K, IV), - {ok, Ssh#ssh{decrypt_keys = K, - decrypt_block_size = 16, - decrypt_ctx = State}}. - - + +decrypt_init(#ssh{decrypt = SshCipher, role = Role} = Ssh) -> + {IvMagic, KeyMagic} = decrypt_magic(Role), + #cipher{impl = CryptoCipher, + key_bytes = KeyBytes, + iv_bytes = IvBytes, + block_bytes = BlockBytes} = cipher(SshCipher), + IV = hash(Ssh, IvMagic, 8*IvBytes), + K = hash(Ssh, KeyMagic, 8*KeyBytes), + Ctx0 = crypto:crypto_init(CryptoCipher, K, IV, false), + {ok, Ssh#ssh{decrypt_cipher = CryptoCipher, + decrypt_block_size = BlockBytes, + decrypt_ctx = Ctx0}}. + + decrypt_final(Ssh) -> {ok, Ssh#ssh {decrypt = none, decrypt_keys = undefined, decrypt_ctx = undefined, decrypt_block_size = 8}}. + decrypt(Ssh, <<>>) -> {Ssh, <<>>}; + decrypt(#ssh{decrypt = '[email protected]', - decrypt_keys = {K1,_K2}, - recv_sequence = Seq} = Ssh, {length,EncryptedLen}) -> - {_State,PacketLenBin} = - crypto:stream_decrypt(crypto:stream_init(chacha20, K1, <<0:8/unit:8, Seq:8/unit:8>>), - EncryptedLen), - {Ssh, PacketLenBin}; -decrypt(#ssh{decrypt = '[email protected]', - decrypt_keys = {_K1,K2}, - recv_sequence = Seq} = Ssh, {AAD,Ctext,Ctag}) -> - %% The length is already decoded and used to divide the input - %% Check the mac (important that it is timing-safe): - {_,PolyKey} = - crypto:stream_encrypt(crypto:stream_init(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>), - <<0:32/unit:8>>), - case equal_const_time(Ctag, crypto:poly1305(PolyKey, <<AAD/binary,Ctext/binary>>)) of - true -> - %% MAC is ok, decode - IV2 = <<1:8/little-unit:8, Seq:8/unit:8>>, - {_,PlainText} = - crypto:stream_decrypt(crypto:stream_init(chacha20,K2,IV2), Ctext), - {Ssh, PlainText}; - false -> - {Ssh,error} + decrypt_keys = {K1,K2}, + recv_sequence = Seq} = Ssh, Data) -> + case Data of + {length,EncryptedLen} -> + %% The length is decrypted separately in a first step + PacketLenBin = crypto:crypto_one_time(chacha20, K1, <<0:8/unit:8, Seq:8/unit:8>>, EncryptedLen, false), + {Ssh, PacketLenBin}; + {AAD,Ctext,Ctag} -> + %% The length is already decrypted and used to divide the input + %% Check the mac (important that it is timing-safe): + PolyKey = crypto:crypto_one_time(chacha20, K2, <<0:8/unit:8,Seq:8/unit:8>>, <<0:32/unit:8>>, false), + case equal_const_time(Ctag, crypto:poly1305(PolyKey, <<AAD/binary,Ctext/binary>>)) of + true -> + %% MAC is ok, decode + IV2 = <<1:8/little-unit:8, Seq:8/unit:8>>, + PlainText = crypto:crypto_one_time(chacha20, K2, IV2, Ctext, false), + {Ssh, PlainText}; + false -> + {Ssh,error} + end end; + decrypt(#ssh{decrypt = none} = Ssh, Data) -> {Ssh, Data}; -decrypt(#ssh{decrypt = 'AEAD_AES_128_GCM', - decrypt_keys = K, - decrypt_ctx = IV0} = Ssh, Data = {_AAD,_Ctext,_Ctag}) -> - Dec = crypto:block_decrypt(aes_gcm, K, IV0, Data), % Dec = PlainText | error - IV = next_gcm_iv(IV0), - {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = 'AEAD_AES_256_GCM', + +decrypt(#ssh{decrypt = SshCipher, + decrypt_cipher = CryptoCipher, decrypt_keys = K, - decrypt_ctx = IV0} = Ssh, Data = {_AAD,_Ctext,_Ctag}) -> - Dec = crypto:block_decrypt(aes_gcm, K, IV0, Data), % Dec = PlainText | error + decrypt_ctx = IV0} = Ssh, {AAD,Ctext,Ctag}) when SshCipher == 'AEAD_AES_128_GCM' ; + SshCipher == 'AEAD_AES_256_GCM' -> + Dec = crypto:crypto_one_time_aead(CryptoCipher, K, IV0, Ctext, AAD, Ctag, false), IV = next_gcm_iv(IV0), {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = '3des-cbc', decrypt_keys = Keys, - decrypt_ctx = IV0} = Ssh, Data) -> - {K1, K2, K3} = Keys, - Dec = crypto:block_decrypt(des3_cbc, [K1,K2,K3], IV0, Data), - IV = crypto:next_iv(des3_cbc, Data), - {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = 'aes128-cbc', decrypt_keys = Key, - decrypt_ctx = IV0} = Ssh, Data) -> - Dec = crypto:block_decrypt(aes_cbc128, Key,IV0,Data), - IV = crypto:next_iv(aes_cbc, Data), - {Ssh#ssh{decrypt_ctx = IV}, Dec}; -decrypt(#ssh{decrypt = 'aes128-ctr', - decrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_decrypt(State0,Data), - {Ssh#ssh{decrypt_ctx = State}, Enc}; -decrypt(#ssh{decrypt = 'aes192-ctr', - decrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_decrypt(State0,Data), - {Ssh#ssh{decrypt_ctx = State}, Enc}; -decrypt(#ssh{decrypt = 'aes256-ctr', - decrypt_ctx = State0} = Ssh, Data) -> - {State, Enc} = crypto:stream_decrypt(State0,Data), - {Ssh#ssh{decrypt_ctx = State}, Enc}. +decrypt(#ssh{decrypt_ctx = Ctx0} = Ssh, Data) -> + Dec = crypto:crypto_update(Ctx0, Data), + {Ssh, Dec}. next_gcm_iv(<<Fixed:32, InvCtr:64>>) -> <<Fixed:32, (InvCtr+1):64>>. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Compression %% @@ -2058,9 +1896,9 @@ compute_key(Algorithm, OthersPublic, MyPrivate, Args) -> dh_bits(#alg{encrypt = Encrypt, send_mac = SendMac}) -> C = cipher(Encrypt), - 8 * lists:max([C#cipher_data.key_bytes, - C#cipher_data.block_bytes, - C#cipher_data.iv_bytes, + 8 * lists:max([C#cipher.key_bytes, + C#cipher.block_bytes, + C#cipher.iv_bytes, mac_key_bytes(SendMac) ]). @@ -2091,40 +1929,13 @@ select_crypto_supported(L) -> crypto_supported(Conditions, Supported) -> lists:all( fun({Tag,CryptoName}) when is_atom(CryptoName) -> - crypto_name_supported(Tag,CryptoName,Supported); - ({Tag,{Name,Len}}) when is_integer(Len) -> - crypto_name_supported(Tag,Name,Supported) andalso - len_supported(Name,Len) + crypto_name_supported(Tag,CryptoName,Supported) end, Conditions). crypto_name_supported(Tag, CryptoName, Supported) -> - Vs = case proplists:get_value(Tag,Supported,[]) of - [] when Tag == curves -> crypto:ec_curves(); - L -> L - end, + Vs = proplists:get_value(Tag,Supported,[]), lists:member(CryptoName, Vs). -len_supported(Name, Len) -> - try - case Name of - aes_ctr -> - {_, <<_/binary>>} = - %% Test encryption - crypto:stream_encrypt(crypto:stream_init(Name, <<0:Len>>, <<0:128>>), <<"">>); - aes_gcm -> - {<<_/binary>>, <<_/binary>>} = - crypto:block_encrypt(Name, - _Key = <<0:Len>>, - _IV = <<0:12/unsigned-unit:8>>, - {<<"AAD">>,"PT"}) - end - of - _ -> true - catch - _:_ -> false - end. - - same(Algs) -> [{client2server,Algs}, {server2client,Algs}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 5de6d52092..9b987dea5a 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -1399,7 +1399,7 @@ rekey_chk(Config, RLdaemon, RLclient) -> Kex1 = ssh_test_lib:get_kex_init(ConnectionRef), %% Make both sides send something: - {ok, SftpPid} = ssh_sftp:start_channel(ConnectionRef), + {ok, _SftpPid} = ssh_sftp:start_channel(ConnectionRef), %% Check rekeying timer:sleep(?REKEY_DATA_TMO), diff --git a/lib/ssh/test/ssh_bench_SUITE.erl b/lib/ssh/test/ssh_bench_SUITE.erl index 2ac4e5636a..880c519a5e 100644 --- a/lib/ssh/test/ssh_bench_SUITE.erl +++ b/lib/ssh/test/ssh_bench_SUITE.erl @@ -178,7 +178,7 @@ gen_data(DataSz) -> connect_measure(Port, Cipher, Mac, Data, Options) -> - AES_GCM = {cipher, + _AES_GCM = {cipher, []}, %% ['[email protected]', %% '[email protected]']}, @@ -187,22 +187,22 @@ connect_measure(Port, Cipher, Mac, Data, Options) -> {none,none} -> [{modify_algorithms,[{prepend, [{cipher,[Cipher]}, {mac,[Mac]}]} -%%% ,{rm,[AES_GCM]} +%%% ,{rm,[_AES_GCM]} ]}]; {none,_} -> [{modify_algorithms,[{prepend, [{cipher,[Cipher]}]} -%%% ,{rm,[AES_GCM]} +%%% ,{rm,[_AES_GCM]} ]}, {preferred_algorithms, [{mac,[Mac]}]}]; {_,none} -> [{modify_algorithms,[{prepend, [{mac,[Mac]}]} -%%% ,{rm,[AES_GCM]} +%%% ,{rm,[_AES_GCM]} ]}, {preferred_algorithms, [{cipher,[Cipher]}]}]; _ -> [{preferred_algorithms, [{cipher,[Cipher]}, {mac,[Mac]}]} -%%% ,{modify_algorithms, [{rm,[AES_GCM]}]} +%%% ,{modify_algorithms, [{rm,[_AES_GCM]}]} ] end, Times = diff --git a/lib/ssh/test/ssh_chan_behaviours_SUITE.erl b/lib/ssh/test/ssh_chan_behaviours_SUITE.erl index 16ed152bcd..103d7253fd 100644 --- a/lib/ssh/test/ssh_chan_behaviours_SUITE.erl +++ b/lib/ssh/test/ssh_chan_behaviours_SUITE.erl @@ -128,8 +128,8 @@ subsystem_client(Config) -> C = proplists:get_value(connref, Config), {ok,ChRef} = ssh_chan_behaviours_client:start_link(C), - IDclt = ?EXPECT({{C,Ch1clt}, {ssh_channel_up,Ch1clt,C}}, {C,Ch1clt}), - IDsrv = ?EXPECT({{_Csrv,Ch1srv}, {ssh_channel_up,Ch1srv,_Csrv}}, {_Csrv,Ch1srv}), + IDclt = ?EXPECT({{C,_Ch1clt}, {ssh_channel_up,_Ch1clt,C}}, {C,_Ch1clt}), + IDsrv = ?EXPECT({{_Csrv,_Ch1srv}, {ssh_channel_up,_Ch1srv,_Csrv}}, {_Csrv,_Ch1srv}), ok = ssh_chan_behaviours_client:stop(ChRef), ?EXPECT({IDclt, {terminate,normal}}, []), % From the proper channel handler diff --git a/lib/ssh/test/ssh_chan_behaviours_client.erl b/lib/ssh/test/ssh_chan_behaviours_client.erl index 15f17733d6..8dd18973ad 100644 --- a/lib/ssh/test/ssh_chan_behaviours_client.erl +++ b/lib/ssh/test/ssh_chan_behaviours_client.erl @@ -94,7 +94,7 @@ handle_ssh_msg({ssh_cm, C, {eof, Ch}}=M, #state{ch=Ch,cm=C} = State) -> ?DBG(State, "eof",[]), {ok, State}; -handle_ssh_msg({ssh_cm, C, {signal, _Ch, _SigNameStr}=Sig} = M, #state{ch=Ch,cm=C} = State) -> +handle_ssh_msg({ssh_cm, C, {signal, Ch, _SigNameStr}=Sig} = M, #state{ch=Ch,cm=C} = State) -> %% Ignore signals according to RFC 4254 section 6.9. tell_parent(M, State), ?DBG(State, "~p",[Sig]), diff --git a/lib/ssh/test/ssh_chan_behaviours_server.erl b/lib/ssh/test/ssh_chan_behaviours_server.erl index 1408675a6e..1d504b1bc6 100644 --- a/lib/ssh/test/ssh_chan_behaviours_server.erl +++ b/lib/ssh/test/ssh_chan_behaviours_server.erl @@ -65,7 +65,7 @@ handle_ssh_msg({ssh_cm, C, {eof, Ch}}=M, #state{ch=Ch,cm=C} = State) -> ?DBG(State, "eof",[]), {ok, State}; -handle_ssh_msg({ssh_cm, C, {signal, _Ch, _SigNameStr}=Sig} = M, #state{ch=Ch,cm=C} = State) -> +handle_ssh_msg({ssh_cm, C, {signal, Ch, _SigNameStr}=Sig} = M, #state{ch=Ch,cm=C} = State) -> %% Ignore signals according to RFC 4254 section 6.9. tell_parent(M, State), ?DBG(State, "~p",[Sig]), diff --git a/lib/ssh/test/ssh_compat_SUITE.erl b/lib/ssh/test/ssh_compat_SUITE.erl index 8e82527c6e..06ed9082cf 100644 --- a/lib/ssh/test/ssh_compat_SUITE.erl +++ b/lib/ssh/test/ssh_compat_SUITE.erl @@ -150,8 +150,7 @@ init_per_group(G, Config0) -> stop_docker(ID), {fail, "Can't contact docker sshd"} catch - Class:Exc -> - ST = erlang:get_stacktrace(), + Class:Exc:ST -> ct:log("common_algs: ~p:~p~n~p",[Class,Exc,ST]), stop_docker(ID), {fail, "Failed during setup"} @@ -160,8 +159,7 @@ init_per_group(G, Config0) -> cant_start_docker -> {skip, "Can't start docker"}; - C:E -> - ST = erlang:get_stacktrace(), + C:E:ST -> ct:log("No ~p~n~p:~p~n~p",[G,C,E,ST]), {skip, "Can't start docker"} end; @@ -1026,8 +1024,7 @@ receive_hello(S) -> Result -> Result catch - Class:Error -> - ST = erlang:get_stacktrace(), + Class:Error:ST -> {error, {Class,Error,ST}} end. @@ -1104,8 +1101,7 @@ sftp_tests_erl_server(Config, ServerIP, ServerPort, ServerRootDir, UserDir) -> call_sftp_in_docker(Config, ServerIP, ServerPort, Cmnds, UserDir), check_local_directory(ServerRootDir) catch - Class:Error -> - ST = erlang:get_stacktrace(), + Class:Error:ST -> {error, {Class,Error,ST}} end. @@ -1133,7 +1129,7 @@ check_local_directory(ServerRootDir) -> check_local_directory(ServerRootDir, SleepTime, N) -> case do_check_local_directory(ServerRootDir) of - {error,Error} when N>0 -> + {error,_Error} when N>0 -> %% Could be that the erlang side is faster and the docker's operations %% are not yet finalized. %% Sleep for a while and retry a few times: @@ -1347,8 +1343,7 @@ one_test_erl_client(SFTP, Id, C) when SFTP==sftp ; SFTP==sftp_async -> catch ssh_sftp:stop_channel(Ch), R catch - Class:Error -> - ST = erlang:get_stacktrace(), + Class:Error:ST -> {error, {SFTP,Id,Class,Error,ST}} end. diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index 778e4a5fc8..6aa587dc7f 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -1124,12 +1124,12 @@ start_our_shell(_User, _Peer) -> ssh_exec_echo(Cmd) -> spawn(fun() -> - io:format("echo "++Cmd ++ "\n") + io:format("echo ~s\n", [Cmd]) end). ssh_exec_echo(Cmd, User) -> spawn(fun() -> - io:format(io_lib:format("echo ~s ~s\n",[User,Cmd])) + io:format("echo ~s ~s\n",[User,Cmd]) end). ssh_exec_echo(Cmd, User, _PeerAddr) -> ssh_exec_echo(Cmd,User). diff --git a/lib/ssh/test/ssh_options_SUITE.erl b/lib/ssh/test/ssh_options_SUITE.erl index 60d0da2a39..bf90f74324 100644 --- a/lib/ssh/test/ssh_options_SUITE.erl +++ b/lib/ssh/test/ssh_options_SUITE.erl @@ -214,7 +214,7 @@ init_per_testcase(_TestCase, Config) -> file:make_dir(UserDir), [{user_dir,UserDir}|Config]. -end_per_testcase(_TestCase, Config) -> +end_per_testcase(_TestCase, _Config) -> ssh:stop(), ok. diff --git a/lib/ssh/test/ssh_protocol_SUITE.erl b/lib/ssh/test/ssh_protocol_SUITE.erl index 3e3e151781..b12ddfeef6 100644 --- a/lib/ssh/test/ssh_protocol_SUITE.erl +++ b/lib/ssh/test/ssh_protocol_SUITE.erl @@ -38,7 +38,11 @@ -define(EXTRA_KEX, 'diffie-hellman-group1-sha1'). -define(CIPHERS, ['aes256-ctr','aes192-ctr','aes128-ctr','aes128-cbc','3des-cbc']). --define(DEFAULT_CIPHERS, [{client2server,?CIPHERS}, {server2client,?CIPHERS}]). +-define(DEFAULT_CIPHERS, (fun() -> Ciphs = filter_supported(cipher, ?CIPHERS), + [{client2server,Ciphs}, {server2client,Ciphs}] + end)() + ). + -define(v(Key, Config), proplists:get_value(Key, Config)). -define(v(Key, Config, Default), proplists:get_value(Key, Config, Default)). diff --git a/lib/ssh/test/ssh_test_lib.erl b/lib/ssh/test/ssh_test_lib.erl index a1a7eebcde..1129303414 100644 --- a/lib/ssh/test/ssh_test_lib.erl +++ b/lib/ssh/test/ssh_test_lib.erl @@ -409,7 +409,7 @@ ct:log("DataDir ~p:~n ~p~n~nSystDir ~p:~n ~p~n~nUserDir ~p:~n ~p",[DataDir, file setup_ecdsa_auth_keys(Size, DataDir, UserDir). setup_eddsa(Alg, DataDir, UserDir) -> - {IdPriv, IdPub, HostPriv, HostPub} = + {IdPriv, _IdPub, HostPriv, HostPub} = case Alg of ed25519 -> {"id_ed25519", "id_ed25519.pub", "ssh_host_ed25519_key", "ssh_host_ed25519_key.pub"}; ed448 -> {"id_ed448", "id_ed448.pub", "ssh_host_ed448_key", "ssh_host_ed448_key.pub"} @@ -970,7 +970,7 @@ expected_state(_) -> false. %%%---------------------------------------------------------------- %%% Return a string with N random characters %%% -random_chars(N) -> [crypto:rand_uniform($a,$z) || _<-lists:duplicate(N,x)]. +random_chars(N) -> [($a-1)+rand:uniform($z-$a) || _<-lists:duplicate(N,x)]. create_random_dir(Config) -> diff --git a/lib/ssh/test/ssh_trpt_test_lib.erl b/lib/ssh/test/ssh_trpt_test_lib.erl index f2c9892f95..3f4df2c986 100644 --- a/lib/ssh/test/ssh_trpt_test_lib.erl +++ b/lib/ssh/test/ssh_trpt_test_lib.erl @@ -570,75 +570,6 @@ receive_binary_msg(S0=#s{}) -> -old_receive_binary_msg(S0=#s{ssh=C0=#ssh{decrypt_block_size = BlockSize, - recv_mac_size = MacSize - } - }) -> - case size(S0#s.encrypted_data_buffer) >= max(8,BlockSize) of - false -> - %% Need more bytes to decode the packet_length field - Remaining = max(8,BlockSize) - size(S0#s.encrypted_data_buffer), - receive_binary_msg( receive_wait(Remaining, S0) ); - true -> - %% Has enough bytes to decode the packet_length field - {_, <<?UINT32(PacketLen), _/binary>>, _} = - ssh_transport:decrypt_blocks(S0#s.encrypted_data_buffer, BlockSize, C0), % FIXME: BlockSize should be at least 4 - - %% FIXME: Check that ((4+PacketLen) rem BlockSize) == 0 ? - - S1 = if - PacketLen > ?SSH_MAX_PACKET_SIZE -> - fail({too_large_message,PacketLen},S0); % FIXME: disconnect - - ((4+PacketLen) rem BlockSize) =/= 0 -> - fail(bad_packet_length_modulo, S0); % FIXME: disconnect - - size(S0#s.encrypted_data_buffer) >= (4 + PacketLen + MacSize) -> - %% has the whole packet - S0; - - true -> - %% need more bytes to get have the whole packet - Remaining = (4 + PacketLen + MacSize) - size(S0#s.encrypted_data_buffer), - receive_wait(Remaining, S0) - end, - - %% Decrypt all, including the packet_length part (re-use the initial #ssh{}) - {C1, SshPacket = <<?UINT32(_),?BYTE(PadLen),Tail/binary>>, EncRest} = - ssh_transport:decrypt_blocks(S1#s.encrypted_data_buffer, PacketLen+4, C0), - - PayloadLen = PacketLen - 1 - PadLen, - <<CompressedPayload:PayloadLen/binary, _Padding:PadLen/binary>> = Tail, - - {C2, Payload} = ssh_transport:decompress(C1, CompressedPayload), - - <<Mac:MacSize/binary, Rest/binary>> = EncRest, - - case {ssh_transport:is_valid_mac(Mac, SshPacket, C2), - catch ssh_message:decode(set_prefix_if_trouble(Payload,S1))} - of - {false, _} -> fail(bad_mac,S1); - {_, {'EXIT',_}} -> fail(decode_failed,S1); - - {true, Msg} -> - C3 = case Msg of - #ssh_msg_kexinit{} -> - ssh_transport:key_init(opposite_role(C2), C2, Payload); - _ -> - C2 - end, - S2 = opt(print_messages, S1, - fun(X) when X==true;X==detail -> {"Recv~n~s~n",[format_msg(Msg)]} end), - S3 = opt(print_messages, S2, - fun(detail) -> {"decrypted bytes ~p~n",[SshPacket]} end), - S3#s{ssh = inc_recv_seq_num(C3), - encrypted_data_buffer = Rest, - return_value = Msg - } - end - end. - - set_prefix_if_trouble(Msg = <<?BYTE(Op),_/binary>>, #s{alg=#alg{kex=Kex}}) when Op == 30; Op == 31 diff --git a/lib/ssl/doc/src/Makefile b/lib/ssl/doc/src/Makefile index 7cf251d8f9..064131944c 100644 --- a/lib/ssl/doc/src/Makefile +++ b/lib/ssl/doc/src/Makefile @@ -47,6 +47,7 @@ XML_CHAPTER_FILES = \ ssl_protocol.xml \ using_ssl.xml \ ssl_distribution.xml \ + standards_compliance.xml \ notes.xml BOOK_FILES = book.xml diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index cee6752020..f0231da2ad 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -843,7 +843,7 @@ <p> TLS sessions must be registered with SNI if provided, so that sessions where client hostname verification would - fail can not connect reusing a session created when the + fail cannot connect reusing a session created when the server name verification succeeded.</p> <p> Own Id: OTP-14632</p> @@ -1021,7 +1021,7 @@ public_key:pkix_verify_hostname/2 to verify the hostname of the connection with the server certificates specified hostname during certificate path validation. The user may - explicitly disables it. Also if the hostname can not be + explicitly disables it. Also if the hostname cannot be derived from the first argument to connect or is not supplied by the server name indication option, the check will not be performed.</p> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index c448d345de..d626748af6 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -145,7 +145,11 @@ </datatype> <datatype> - <name name="legacy_version"/> + <name name="tls_legacy_version"/> + </datatype> + + <datatype> + <name name="dtls_legacy_version"/> </datatype> <datatype> @@ -195,11 +199,15 @@ <datatype> <name name="signature_algs"/> </datatype> - + <datatype> <name name="sign_algo"/> </datatype> - + + <datatype> + <name name="sign_scheme"/> + </datatype> + <datatype> <name name="kex_algo"/> </datatype> @@ -332,7 +340,30 @@ matters.</p> </desc> </datatype> - + + <datatype> + <name name="signature_schemes"/> + <desc> + <p> + In addition to the signature_algorithms extension from TLS 1.2, + <url href="http://www.ietf.org/rfc/rfc8446.txt#section-4.2.3">TLS 1.3 + (RFC 5246 Section 4.2.3)</url>adds the signature_algorithms_cert extension + which enables having special requirements on the signatures used in the + certificates that differs from the requirements on digital signatures as a whole. + If this is not required this extension is not needed. + </p> + <p> + The client will send a signature_algorithms_cert extension (ClientHello), + if TLS version 1.3 or later is used, and the signature_algs_cert option is + explicitly specified. By default, only the signature_algs extension is sent. + </p> + <p> + The signature schemes shall be ordered according to the client's preference + (favorite choice first). + </p> + </desc> + </datatype> + <datatype> <name name="secure_renegotiation"/> <desc><p>Specifies if to reject renegotiation attempt that does @@ -472,7 +503,7 @@ marker="public_key:public_key#pkix_path_validation-3">public_key:pkix_path_valid <item>check is only performed on the peer certificate.</item> <tag><c>best_effort</c></tag> - <item>if certificate revocation status can not be determined + <item>if certificate revocation status cannot be determined it will be accepted as valid.</item> </taglist> @@ -605,10 +636,19 @@ fun(srp, Username :: string(), UserState :: term()) -> </desc> </datatype> - <datatype> - <name name="log_alert"/> - <desc><p>If set to <c>false</c>, error reports are not displayed.</p> - </desc> + <datatype> + <name name="log_alert"/> + <desc><p>If set to <c>false</c>, error reports are not displayed. + Deprecated in OTP 22, use {log_level, <seealso marker="#type-logging_level">logging_level()</seealso>} instead.</p> + </desc> + </datatype> + + <datatype> + <name name="logging_level"/> + <desc><p>Specifies the log level for TLS/DTLS. At verbosity level <c>notice</c> and above error reports are + displayed in TLS/DTLS. The level <c>debug</c> triggers verbose logging of TLS/DTLS protocol + messages.</p> + </desc> </datatype> <datatype> @@ -849,7 +889,6 @@ fun(srp, Username :: string(), UserState :: term()) -> </desc> </datatype> - <datatype_title>TLS/DTLS OPTION DESCRIPTIONS - SERVER </datatype_title> @@ -863,8 +902,7 @@ fun(srp, Username :: string(), UserState :: term()) -> is supplied it overrides option <c>cacertfile</c>.</p> </desc> </datatype> - - + <datatype> <name name="server_cafile"/> <desc><p>Path to a file containing PEM-encoded CA @@ -892,7 +930,6 @@ fun(srp, Username :: string(), UserState :: term()) -> default parameters are used.</p> </desc> </datatype> - <datatype> <name name="server_verify_type"/> diff --git a/lib/ssl/doc/src/ssl_app.xml b/lib/ssl/doc/src/ssl_app.xml index 893919aeb4..b05caf44ea 100644 --- a/lib/ssl/doc/src/ssl_app.xml +++ b/lib/ssl/doc/src/ssl_app.xml @@ -35,45 +35,10 @@ <description> <p> - The ssl application is an implementation of the SSL/TLS/DTLS protocol in Erlang. + The ssl application is an implementation of the SSL, TLS and DTLS protocols in Erlang. </p> - <list type="bulleted"> - <item>Supported SSL/TLS/DTLS-versions are SSL-3.0, TLS-1.0, - TLS-1.1, TLS-1.2, DTLS-1.0 (based on TLS-1.1), DTLS-1.2 (based on TLS-1.2)</item> - <item>For security reasons SSL-2.0 is not supported. - Interoperability with SSL-2.0 enabled clients dropped. (OTP 21) </item> - <item>For security reasons SSL-3.0 is no longer supported by default, - but can be configured. (OTP 19) </item> - <item>For security reasons RSA key exchange cipher suites are no longer supported by default, - but can be configured. (OTP 21) </item> - <item>For security reasons DES cipher suites are no longer supported by default, - but can be configured. (OTP 20) </item> - <item>For security reasons 3DES cipher suites are no longer supported by default, - but can be configured. (OTP 21) </item> - <item> Renegotiation Indication Extension <url href="http://www.ietf.org/rfc/rfc5746.txt">RFC 5746</url> is supported - </item> - <item>Ephemeral Diffie-Hellman cipher suites are supported, - but not Diffie Hellman Certificates cipher suites.</item> - <item>Elliptic Curve cipher suites are supported if the Crypto - application supports it and named curves are used. - </item> - <item>Export cipher suites are not supported as the - U.S. lifted its export restrictions in early 2000.</item> - <item>IDEA cipher suites are not supported as they have - become deprecated by the latest TLS specification so it is not - motivated to implement them.</item> - <item>Compression is not supported.</item> - <item>CRL validation is supported.</item> - <item>Policy certificate extensions are not supported.</item> - <item>'Server Name Indication' extension - (<url href="http://www.ietf.org/rfc/rfc6066.txt">RFC 6066</url>) is supported.</item> - <item>Application Layer Protocol Negotiation (ALPN) and its successor Next Protocol Negotiation (NPN) - are supported. </item> - <item>It is possible to use Pre-Shared Key (PSK) and Secure Remote Password (SRP) - cipher suites, but they are not enabled by default. - </item> - </list> - </description> + <p>For current statement of standards compliance see the <seealso marker="standards_compliance">User's Guide</seealso>.</p> + </description> <section> <title>DEPENDENCIES</title> diff --git a/lib/ssl/doc/src/ssl_distribution.xml b/lib/ssl/doc/src/ssl_distribution.xml index e14f3f90dc..1774bd8f77 100644 --- a/lib/ssl/doc/src/ssl_distribution.xml +++ b/lib/ssl/doc/src/ssl_distribution.xml @@ -191,7 +191,7 @@ Eshell V5.0 (abort with ^G) Any available SSL/TLS option can be specified in an options file, but note that options that take a <c>fun()</c> has to use the syntax <c>fun Mod:Func/Arity</c> since a function - body can not be compiled when consulting a file. + body cannot be compiled when consulting a file. </p> <p> Do not tamper with the socket options diff --git a/lib/ssl/doc/src/standards_compliance.xml b/lib/ssl/doc/src/standards_compliance.xml new file mode 100644 index 0000000000..ca98385f85 --- /dev/null +++ b/lib/ssl/doc/src/standards_compliance.xml @@ -0,0 +1,2338 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2015</year> + <year>2019</year> + <holder>Ericsson AB, All Rights Reserved</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + The Initial Developer of the Original Code is Ericsson AB. + </legalnotice> + + <title>Standards Compliance</title> + <prepared>OTP team</prepared> + <docno></docno> + <date>2019-03-20</date> + <rev>A</rev> + <file>standards_compliance.xml</file> + </header> + + <section> + <title>Purpose</title> + <p>This section describes the current state of standards compliance of the ssl application.</p> + </section> + + <section> + <title>Common (pre TLS 1.3)</title> + <list type="bulleted"> + <item>For security reasons RSA key exchange cipher suites are no longer supported by default, + but can be configured. (OTP 21) + </item> + <item>For security reasons DES cipher suites are no longer supported by default, + but can be configured. (OTP 20) + </item> + <item>For security reasons 3DES cipher suites are no longer supported by default, + but can be configured. (OTP 21) + </item> + <item>Renegotiation Indication Extension <url href="http://www.ietf.org/rfc/rfc5746.txt">RFC 5746</url> is supported + </item> + <item>Ephemeral Diffie-Hellman cipher suites are supported, + but not Diffie Hellman Certificates cipher suites. + </item> + <item>Elliptic Curve cipher suites are supported if the Crypto + application supports it and named curves are used. + </item> + <item>Export cipher suites are not supported as the + U.S. lifted its export restrictions in early 2000. + </item> + <item>IDEA cipher suites are not supported as they have + become deprecated by the TLS 1.2 specification so it is not + motivated to implement them. + </item> + <item>Compression is not supported. + </item> + </list> + </section> + + <section> + <title>Common</title> + <list type="bulleted"> + <item>CRL validation is supported.</item> + <item>Policy certificate extensions are not supported.</item> + <item>'Server Name Indication' extension + (<url href="http://www.ietf.org/rfc/rfc6066.txt">RFC 6066</url>) is supported.</item> + <item>Application Layer Protocol Negotiation (ALPN) and its successor Next Protocol Negotiation (NPN) are supported. </item> + <item>It is possible to use Pre-Shared Key (PSK) and Secure Remote Password (SRP) + cipher suites, but they are not enabled by default. + </item> + </list> + </section> + + + <section> + <title>SSL 2.0</title> + <p>For security reasons SSL-2.0 is not supported. Interoperability with SSL-2.0 enabled clients dropped. (OTP 21)</p> + </section> + + <section> + <title>SSL 3.0</title> + <p>For security reasons SSL-3.0 is no longer supported by default, but can be configured. (OTP 19)</p> + </section> + + <section> + <title>TLS 1.0</title> + <p>For security reasons TLS-1.0 is no longer supported by default, but can be configured. (OTP 22)</p> + </section> + + <section> + <title>TLS 1.1</title> + <p>For security reasons TLS-1.1 is no longer supported by default, but can be configured. (OTP 22)</p> + </section> + + <section> + <title>TLS 1.2</title> + <p>Supported</p> + </section> + + <section> + <title>DTLS 1.0</title> + <p>For security reasons DTLS-1.0 (based on TLS 1.1) is no longer supported by default, but can be configured. (OTP 22)</p> + </section> + + <section> + <title>DTLS 1.2</title> + <p>Supported (based on TLS 1.2)</p> + </section> + + <section> + <title>DTLS 1.3</title> + <p>Not yet supported</p> + </section> + + <section> + <title>TLS 1.3</title> + <p>OTP-22 introduces basic support for TLS 1.3 on the server side. Basic functionality + covers a simple TLS 1.3 handshake with support of the mandatory extensions + (supported_groups, signature_algorithms, key_share, supported_versions and + signature_algorithms_cert). The server supports a selective set of cryptographic algorithms:</p> + <list type="bulleted"> + <item>Key Exchange: ECDHE</item> + <item>Groups: all standard groups supported for the Diffie-Hellman key exchange</item> + <item>Ciphers: TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, + TLS_CHACHA20_POLY1305_SHA256 and TLS_AES_128_CCM_SHA256</item> + <item>Signature Algorithms: RSA and RSA PSS</item> + <item>Certificates: currently only certificates with RSA keys are supported</item> + </list> + <p>Other notable features:</p> + <list type="bulleted"> + <item>The server supports the HelloRetryRequest mechanism</item> + <item>PSK and session resumption not supported</item> + <item>Early data and 0-RTT not supported</item> + <item>Key and Initialization Vector Update not supported</item> + </list> + <p>For more detailed information see the + <seealso marker="#soc_table">Standards Compliance</seealso> below.</p> + <warning><p>Note that the client side is not yet functional. It is planned to be released + later in OTP-22.</p></warning> + + <p> The following table describes the current state of standards compliance for TLS 1.3.</p> + <p>(<em>C</em> = Compliant, <em>NC</em> = Non-Compliant, <em>PC</em> = Partially-Compliant, + <em>NA</em> = Not Applicable)</p> + <marker id="soc_table"/> + <table> + <row> + <cell align="left" valign="middle"><em>Section</em></cell> + <cell align="left" valign="middle"><em>Feature</em></cell> + <cell align="left" valign="middle"><em>State</em></cell> + <cell align="left" valign="middle"><em>Since</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-1.2"> + 1.3. Updates Affecting TLS 1.2 + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Version downgrade protection mechanism</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">RSASSA-PSS signature schemes</cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (ClientHello) extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2"> + 2. Protocol Overview + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">(EC)DHE</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">PSK-only</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">PSK with (EC)DHE</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2.1"> + 2.1. Incorrect DHE share + </url> + </cell> + <cell align="left" valign="middle">HelloRetryRequest</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2.2"> + 2.2. Resumption and Pre-Shared Key (PSK) + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2.3"> + 2.3. 0-RTT Data + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.1"> + 4.1.1. Cryptographic Negotiation + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key extension</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.2"> + 4.1.2. Client Hello + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">padding (RFC7685)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">psk_key_exchange_modes (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">cookie (RFC8446) </cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">post_handshake_auth (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">padding (RFC7685)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">psk_key_exchange_modes (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">cookie (RFC8446) </cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">post_handshake_auth (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.3"> + 4.1.3. Server Hello + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Version downgrade protection</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Version downgrade protection</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.4"> + 4.1.4. Hello Retry Request + </url> + </cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">cookie (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.1"> + 4.2.1. Supported Versions + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.2"> + 4.2.2. Cookie + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.3"> + 4.2.3. Signature Algorithms + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp256r1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp384r1_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp521r1_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed25519</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed448</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_sha1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha512</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp256r1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp384r1_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp521r1_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha512</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed25519</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed448</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_sha1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.4"> + 4.2.4. Certificate Authorities + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.5"> + 4.2.5. OID Filters + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.6"> + 4.2.6. Post-Handshake Client Authentication + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.7"> + 4.2.7. Supported Groups + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp256r1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp384r1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp521r1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x25519</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x448</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe2048</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe3072</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe4096</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe6144</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe8192</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp256r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp384r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp521r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x25519</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x448</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe2048</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe3072</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe4096</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe6144</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe8192</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.8"> + 4.2.8. Key Share + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.9"> + 4.2.9. Pre-Shared Key Exchange Modes + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.10"> + 4.2.10. Early Data Indication + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11"> + 4.2.11. Pre-Shared Key Extension + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11.1"> + 4.2.11.1. Ticket Age + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11.2"> + 4.2.11.2. PSK Binder + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11.3"> + 4.2.11.3. Processing Order + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.3.1"> + 4.3.1. Encrypted Extensions + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.3.2"> + 4.3.2. Certificate Request + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.1"> + 4.4.1. The Transcript Hash + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2"> + 4.4.2. Certificate + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.1"> + 4.4.2.1. OCSP Status and SCT Extensions + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.2"> + 4.4.2.2. Server Certificate Selection + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate type MUST be X.509v3</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate's public key is compatible</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">The certificate MUST allow the key to be used for signing</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name and certificate_authorities are used</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate type MUST be X.509v3</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate's public key is compatible</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">The certificate MUST allow the key to be used for signing</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name and certificate_authorities are used</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.3"> + 4.4.2.3. Client Certificate Selection + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.4"> + 4.4.2.4. Receiving a Certificate Message + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.3"> + 4.4.3. Certificate Verify + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.4"> + 4.4.4. Finished + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.5"> + 4.5. End of Early Data + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.6.1"> + 4.6.1. New Session Ticket Message + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.6.2"> + 4.6.2. Post-Handshake Authentication + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.6.3"> + 4.6.3. Key and Initialization Vector Update + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.1"> + 5.1. Record Layer + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT be interleaved with other record types</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT span key changes</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT send zero-length fragments</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Alert messages MUST NOT be fragmented</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.2"> + 5.2. Record Payload Protection + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.3"> + 5.3. Per-Record Nonce + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.4"> + 5.4. Record Padding + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MAY choose to pad</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT send Handshake and Alert records that have a zero-length TLSInnerPlaintext.content</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">The padding sent is automatically verified</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.5"> + 5.5. Limits on Key Usage + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-6.1"> + 6.1. Closure Alerts + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">close_notify</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">user_cancelled</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-6.2"> + 6.2. Error Alerts + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.1"> + 7.1. Key Schedule + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.2"> + 7.2. Updating Traffic Secrets + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.3"> + 7.3. Traffic Key Calculation + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.5"> + 7.5. Exporters + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8"> + 8. 0-RTT and Anti-Replay + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8.1"> + 8.1. Single-Use Tickets + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8.2"> + 8.2. Client Hello Recording + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8.3"> + 8.3. Freshness Checks + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-9.1"> + 9.1. Mandatory-to-Implement Cipher Suites + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST implement the TLS_AES_128_GCM_SHA256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">SHOULD implement the TLS_AES_256_GCM_SHA384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">SHOULD implement the TLS_CHACHA20_POLY1305_SHA256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Digital signatures</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support rsa_pkcs1_sha256 (for certificates)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support rsa_pss_rsae_sha256 (for CertificateVerify and certificates)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support ecdsa_secp256r1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Key Exchange</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support key exchange with secp256r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">SHOULD support key exchange with X25519</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-9.2"> + 9.2. Mandatory-to-Implement Extensions + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Supported Versions</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Cookie</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Signature Algorithms</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Signature Algorithms Certificate</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Negotiated Groups</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Key Share</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Server Name Indication</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>MUST send and use these extensions</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"supported_versions" is REQUIRED for ClientHello, ServerHello and HelloRetryRequest</cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"signature_algorithms" is REQUIRED for certificate authentication</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"supported_groups" is REQUIRED for ClientHello messages using (EC)DHE key exchange</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"key_share" is REQUIRED for (EC)DHE key exchange</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"pre_shared_key" is REQUIRED for PSK key agreement</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"psk_key_exchange_modes" is REQUIRED for PSK key agreement</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>TLS 1.3 ClientHello</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">If not containing a "pre_shared_key" extension, it MUST contain both a "signature_algorithms" extension and a "supported_groups" extension.</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">If containing a "supported_groups" extension, it MUST also contain a "key_share" extension, and vice versa. An empty KeyShare.client_shares vector is permitted.</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>TLS 1.3 ServerHello</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support the use of the "server_name" extension</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-9.3"> + 9.3. Protocol Invariants + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>MUST correctly handle extensible fields</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">A client sending a ClientHello MUST support all parameters advertised in it.</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">A middlebox which terminates a TLS connection MUST behave as a compliant TLS server</cell> + <cell align="left" valign="middle"><em>NA</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">A middlebox which forwards ClientHello parameters it does not understand MUST NOT process any messages beyond that ClientHello.</cell> + <cell align="left" valign="middle"><em>NA</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-B.4"> + B.4. Cipher Suites + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_128_GCM_SHA256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_256_GCM_SHA384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_CHACHA20_POLY1305_SHA256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_128_CCM_SHA256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_128_CCM_8_SHA256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.1"> + C.1. Random Number Generation and Seeding + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.2"> + C.2. Certificates and Authentication + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.3"> + C.3. Implementation Pitfalls + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.4"> + C.4. Client Tracking Prevention + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.5"> + C.5. Unauthenticated Operation + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.1"> + D.1. Negotiating with an Older Server + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.2"> + D.2. Negotiating with an Older Client + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.3"> + D.3. 0-RTT Backward Compatibility + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.4"> + D.4. Middlebox Compatibility Mode + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.5"> + D.5. Security Restrictions Related to Backward Compatibility + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <tcaption>Standards Compliance</tcaption> + </table> + + </section> + +</chapter> diff --git a/lib/ssl/doc/src/usersguide.xml b/lib/ssl/doc/src/usersguide.xml index 23ccf668c3..b22b2456e4 100644 --- a/lib/ssl/doc/src/usersguide.xml +++ b/lib/ssl/doc/src/usersguide.xml @@ -38,6 +38,7 @@ <xi:include href="ssl_protocol.xml"/> <xi:include href="using_ssl.xml"/> <xi:include href="ssl_distribution.xml"/> + <xi:include href="standards_compliance.xml"/> </part> diff --git a/lib/ssl/src/Makefile b/lib/ssl/src/Makefile index 8d1341f594..8dc76f2638 100644 --- a/lib/ssl/src/Makefile +++ b/lib/ssl/src/Makefile @@ -39,60 +39,80 @@ RELSYSDIR = $(RELEASE_PATH)/lib/ssl-$(VSN) # ---------------------------------------------------- BEHAVIOUR_MODULES= \ - ssl_session_cache_api \ - ssl_crl_cache_api + ssl_crl_cache_api \ + ssl_session_cache_api + MODULES= \ - ssl \ - ssl_alert \ - ssl_app \ - ssl_sup \ - ssl_admin_sup\ - tls_connection_sup \ - ssl_connection_sup \ - ssl_listen_tracker_sup\ + dtls_connection \ dtls_connection_sup \ - dtls_packet_demux \ + dtls_handshake \ dtls_listener_sup \ - ssl_dist_sup\ - ssl_dist_admin_sup\ - ssl_dist_connection_sup\ + dtls_packet_demux \ + dtls_record \ + dtls_socket \ + dtls_v1 \ inet_tls_dist \ inet6_tls_dist \ - ssl_certificate\ - ssl_pkix_db\ + ssl \ + ssl_admin_sup \ + ssl_alert \ + ssl_app \ + ssl_certificate \ ssl_cipher \ ssl_cipher_format \ - ssl_srp_primes \ - tls_connection \ - dtls_connection \ - tls_sender\ ssl_config \ ssl_connection \ - tls_handshake \ - dtls_handshake\ - ssl_handshake\ - ssl_manager \ - ssl_session \ - ssl_session_cache \ - ssl_pem_cache \ - ssl_crl\ + ssl_connection_sup \ + ssl_crl \ ssl_crl_cache \ ssl_crl_hash_dir \ - tls_socket \ - dtls_socket \ - tls_record \ - dtls_record \ + ssl_dh_groups \ + ssl_dist_admin_sup \ + ssl_dist_connection_sup \ + ssl_dist_sup \ + ssl_handshake \ + ssl_listen_tracker_sup \ + ssl_logger \ + ssl_manager \ + ssl_pem_cache \ + ssl_pkix_db \ ssl_record \ + ssl_session \ + ssl_session_cache \ + ssl_srp_primes \ + ssl_sup \ ssl_v3 \ - tls_v1 \ - dtls_v1 + tls_connection \ + tls_connection_sup \ + tls_connection_1_3 \ + tls_handshake \ + tls_handshake_1_3 \ + tls_record \ + tls_record_1_3 \ + tls_sender \ + tls_socket \ + tls_v1 + INTERNAL_HRL_FILES = \ - ssl_alert.hrl ssl_cipher.hrl \ - tls_connection.hrl dtls_connection.hrl ssl_connection.hrl \ - ssl_handshake.hrl tls_handshake.hrl dtls_handshake.hrl ssl_api.hrl ssl_internal.hrl \ - ssl_record.hrl tls_record.hrl dtls_record.hrl ssl_srp.hrl + dtls_connection.hrl \ + dtls_handshake.hrl \ + dtls_record.hrl \ + ssl_alert.hrl \ + ssl_api.hrl \ + ssl_cipher.hrl \ + ssl_connection.hrl \ + ssl_handshake.hrl \ + ssl_internal.hrl \ + ssl_record.hrl \ + ssl_srp.hrl \ + tls_connection.hrl \ + tls_handshake.hrl \ + tls_handshake_1_3.hrl \ + tls_record.hrl \ + tls_record_1_3.hrl + ERL_FILES= \ $(MODULES:%=%.erl) \ @@ -111,6 +131,10 @@ APP_TARGET= $(EBIN)/$(APP_FILE) APPUP_SRC= $(APPUP_FILE).src APPUP_TARGET= $(EBIN)/$(APPUP_FILE) +DEPDIR=$(ERL_TOP)/lib/ssl/src/deps +DEP_FILE=$(DEPDIR)/ssl.d +$(shell mkdir -p $(dir $(DEP_FILE)) >/dev/null) + # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- @@ -118,7 +142,7 @@ EXTRA_ERLC_FLAGS = +warn_unused_vars ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/kernel/src \ -pz $(EBIN) \ -pz $(ERL_TOP)/lib/public_key/ebin \ - $(EXTRA_ERLC_FLAGS) -DVSN=\"$(VSN)\" + $(EXTRA_ERLC_FLAGS) # ---------------------------------------------------- @@ -127,11 +151,22 @@ ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/kernel/src \ $(TARGET_FILES): $(BEHAVIOUR_TARGET_FILES) -debug opt: $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) +$(DEP_FILE): $(ERL_FILES) + $(gen_verbose)erlc -M $(ERL_FILES) \ + | sed "s@$(ERL_TOP)@../../..@g" \ + | sed "s/\.$(EMULATOR)/\.$$\(EMULATOR\)/" \ + | sed 's@^dtls_@$$(EBIN)/dtls_@' \ + | sed 's@^inet_@$$(EBIN)/inet_@' \ + | sed 's@^ssl_@$$(EBIN)/ssl_@' \ + | sed 's@^tls_@$$(EBIN)/tls_@' \ + > $(DEP_FILE) + +debug opt: $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) $(DEP_FILE) clean: rm -f $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) $(BEHAVIOUR_TARGET_FILES) rm -f errs core *~ + rm -rf $(DEPDIR) $(APP_TARGET): $(APP_SRC) ../vsn.mk $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ @@ -141,7 +176,6 @@ $(APPUP_TARGET): $(APPUP_SRC) ../vsn.mk docs: - # ---------------------------------------------------- # Release Target # ---------------------------------------------------- @@ -159,22 +193,4 @@ release_docs_spec: # ---------------------------------------------------- # Dependencies # ---------------------------------------------------- -$(EBIN)/inet_tls_dist.$(EMULATOR): ../../kernel/include/net_address.hrl ../../kernel/include/dist.hrl ../../kernel/include/dist_util.hrl -$(EBIN)/tls.$(EMULATOR): ssl_internal.hrl ssl_record.hrl ssl_cipher.hrl ssl_handshake.hrl ../../public_key/include/public_key.hrl -$(EBIN)/ssl_alert.$(EMULATOR): ssl_alert.hrl ssl_record.hrl -$(EBIN)/ssl_certificate.$(EMULATOR): ssl_internal.hrl ssl_alert.hrl ssl_handshake.hrl ../../public_key/include/public_key.hrl -$(EBIN)/ssl_certificate_db.$(EMULATOR): ssl_internal.hrl ../../public_key/include/public_key.hrl ../../kernel/include/file.hrl -$(EBIN)/ssl_cipher.$(EMULATOR): ssl_internal.hrl ssl_record.hrl ssl_cipher.hrl ssl_handshake.hrl ssl_alert.hrl ../../public_key/include/public_key.hrl -$(EBIN)/tls_connection.$(EMULATOR): ssl_internal.hrl tls_connection.hrl tls_record.hrl ssl_cipher.hrl tls_handshake.hrl ssl_alert.hrl ../../public_key/include/public_key.hrl -$(EBIN)/dtls_connection.$(EMULATOR): ssl_internal.hrl dtls_connection.hrl dtls_record.hrl ssl_cipher.hrl dtls_handshake.hrl ssl_alert.hrl ../../public_key/include/public_key.hrl -$(EBIN)/tls_handshake.$(EMULATOR): ssl_internal.hrl tls_record.hrl ssl_cipher.hrl tls_handshake.hrl ssl_alert.hrl ../../public_key/include/public_key.hrl -$(EBIN)/tls_handshake.$(EMULATOR): ssl_internal.hrl ssl_connection.hrl ssl_record.hrl ssl_cipher.hrl ssl_handshake.hrl ssl_alert.hrl ../../public_key/include/public_key.hrl -$(EBIN)/ssl_manager.$(EMULATOR): ssl_internal.hrl ssl_handshake.hrl ../../kernel/include/file.hrl -$(EBIN)/ssl_record.$(EMULATOR): ssl_internal.hrl ssl_record.hrl ssl_cipher.hrl ssl_handshake.hrl ssl_alert.hrl -$(EBIN)/ssl_session.$(EMULATOR): ssl_internal.hrl ssl_handshake.hrl -$(EBIN)/ssl_session_cache.$(EMULATOR): ssl_internal.hrl ssl_handshake.hrl -$(EBIN)/ssl_session_cache_api.$(EMULATOR): ssl_internal.hrl ssl_handshake.hrl -$(EBIN)/ssl_ssl3.$(EMULATOR): ssl_internal.hrl ssl_record.hrl ssl_cipher.hrl -$(EBIN)/ssl_tls1.$(EMULATOR): ssl_internal.hrl ssl_record.hrl ssl_cipher.hrl -$(EBIN)/ssl_cache.$(EMULATOR): ssl_cache.erl ssl_internal.hrl ../../public_key/include/public_key.hrl - +-include $(DEP_FILE) diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index 2c6b71c97a..e070006900 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -32,6 +32,7 @@ -include("ssl_internal.hrl"). -include("ssl_srp.hrl"). -include_lib("public_key/include/public_key.hrl"). +-include_lib("kernel/include/logger.hrl"). %% Internal application API @@ -66,7 +67,7 @@ %% Setup %%==================================================================== start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_, Tracker} = Opts, - User, {CbModule, _,_, _} = CbInfo, + User, {CbModule, _, _, _, _} = CbInfo, Timeout) -> try {ok, Pid} = dtls_connection_sup:start_child([Role, Host, Port, Socket, @@ -146,13 +147,16 @@ next_record(#state{static_env = #static_env{role = server, socket = {Listener, {Client, _}}}} = State) -> dtls_packet_demux:active_once(Listener, Client, self()), {no_record, State}; -next_record(#state{static_env = #static_env{role = client, +next_record(#state{protocol_specific = #{active_n_toggle := true, + active_n := N} = ProtocolSpec, + static_env = #static_env{role = client, socket = {_Server, Socket} = DTLSSocket, close_tag = CloseTag, transport_cb = Transport}} = State) -> - case dtls_socket:setopts(Transport, Socket, [{active,once}]) of + case dtls_socket:setopts(Transport, Socket, [{active,N}]) of ok -> - {no_record, State}; + {no_record, State#state{protocol_specific = + ProtocolSpec#{active_n_toggle => false}}}; _ -> self() ! {CloseTag, DTLSSocket}, {no_record, State} @@ -192,7 +196,8 @@ next_event(StateName, no_record, %% TODO maybe buffer later epoch next_event(StateName, no_record, State, Actions); {#alert{} = Alert, State} -> - {next_state, StateName, State, [{next_event, internal, Alert} | Actions]} + Version = State#state.connection_env#connection_env.negotiated_version, + handle_own_alert(Alert, Version, StateName, State) end; next_event(connection = StateName, Record, #state{connection_states = #{current_read := #{epoch := CurrentEpoch}}} = State0, Actions) -> @@ -232,7 +237,8 @@ next_event(StateName, Record, %% TODO maybe buffer later epoch next_event(StateName, no_record, State0, Actions); #alert{} = Alert -> - {next_state, StateName, State0, [{next_event, internal, Alert} | Actions]} + Version = State0#state.connection_env#connection_env.negotiated_version, + handle_own_alert(Alert, Version, StateName, State0) end. %%% DTLS record protocol level application data messages @@ -249,10 +255,11 @@ handle_protocol_record(#ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, Stat handle_protocol_record(#ssl_tls{type = ?HANDSHAKE, fragment = Data}, StateName, - #state{protocol_buffers = Buffers0, - connection_env = #connection_env{negotiated_version = Version}} = State) -> + #state{protocol_buffers = Buffers0, + connection_env = #connection_env{negotiated_version = Version}, + ssl_options = Options} = State) -> try - case dtls_handshake:get_dtls_handshake(Version, Data, Buffers0) of + case dtls_handshake:get_dtls_handshake(Version, Data, Buffers0, Options) of {[], Buffers} -> next_event(StateName, no_record, State#state{protocol_buffers = Buffers}); {Packets, Buffers} -> @@ -287,9 +294,10 @@ handle_protocol_record(#ssl_tls{type = _Unknown}, StateName, State) -> %% Handshake handling %%==================================================================== -renegotiate(#state{static_env = #static_env{role = client}} = State, Actions) -> +renegotiate(#state{static_env = #static_env{role = client}} = State0, Actions) -> %% Handle same way as if server requested %% the renegotiation + State = reinit_handshake_data(State0), {next_state, connection, State, [{next_event, internal, #hello_request{}} | Actions]}; @@ -307,9 +315,12 @@ queue_handshake(Handshake0, #state{handshake_env = #handshake_env{tls_handshake_ connection_env = #connection_env{negotiated_version = Version}, flight_buffer = #{handshakes := HsBuffer0, change_cipher_spec := undefined, - next_sequence := Seq} = Flight0} = State) -> + next_sequence := Seq} = Flight0, + ssl_options = SslOpts} = State) -> Handshake = dtls_handshake:encode_handshake(Handshake0, Version, Seq), Hist = update_handshake_history(Handshake0, Handshake, Hist0), + ssl_logger:debug(SslOpts#ssl_options.log_level, outbound, 'handshake', Handshake0), + State#state{flight_buffer = Flight0#{handshakes => [Handshake | HsBuffer0], next_sequence => Seq +1}, handshake_env = HsEnv#handshake_env{tls_handshake_history = Hist}}; @@ -317,9 +328,12 @@ queue_handshake(Handshake0, #state{handshake_env = #handshake_env{tls_handshake_ queue_handshake(Handshake0, #state{handshake_env = #handshake_env{tls_handshake_history = Hist0} = HsEnv, connection_env = #connection_env{negotiated_version = Version}, flight_buffer = #{handshakes_after_change_cipher_spec := Buffer0, - next_sequence := Seq} = Flight0} = State) -> + next_sequence := Seq} = Flight0, + ssl_options = SslOpts} = State) -> Handshake = dtls_handshake:encode_handshake(Handshake0, Version, Seq), Hist = update_handshake_history(Handshake0, Handshake, Hist0), + ssl_logger:debug(SslOpts#ssl_options.log_level, outbound, 'handshake', Handshake0), + State#state{flight_buffer = Flight0#{handshakes_after_change_cipher_spec => [Handshake | Buffer0], next_sequence => Seq +1}, handshake_env = HsEnv#handshake_env{tls_handshake_history = Hist}}. @@ -327,7 +341,7 @@ queue_handshake(Handshake0, #state{handshake_env = #handshake_env{tls_handshake_ queue_change_cipher(ChangeCipher, #state{flight_buffer = Flight, connection_states = ConnectionStates0} = State) -> ConnectionStates = - dtls_record:next_epoch(ConnectionStates0, write), + dtls_record:next_epoch(ConnectionStates0, write), State#state{flight_buffer = Flight#{change_cipher_spec => ChangeCipher}, connection_states = ConnectionStates}. @@ -350,8 +364,8 @@ reinit_handshake_data(#state{static_env = #static_env{data_tag = DataTag}, dtls_handshake_later_fragments = [] }}. -select_sni_extension(#client_hello{extensions = HelloExtensions}) -> - HelloExtensions#hello_extensions.sni; +select_sni_extension(#client_hello{extensions = #{sni := SNI}}) -> + SNI; select_sni_extension(_) -> undefined. @@ -367,11 +381,14 @@ encode_alert(#alert{} = Alert, Version, ConnectionStates) -> send_alert(Alert, #state{static_env = #static_env{socket = Socket, transport_cb = Transport}, + connection_env = #connection_env{negotiated_version = Version}, - connection_states = ConnectionStates0} = State0) -> + connection_states = ConnectionStates0, + ssl_options = SslOpts} = State0) -> {BinMsg, ConnectionStates} = encode_alert(Alert, Version, ConnectionStates0), send(Transport, Socket, BinMsg), + ssl_logger:debug(SslOpts#ssl_options.log_level, outbound, 'record', BinMsg), State0#state{connection_states = ConnectionStates}. send_alert_in_connection(Alert, State) -> @@ -438,8 +455,7 @@ init({call, From}, {start, Timeout}, session = Session0#session{session_id = Hello#client_hello.session_id}, start_or_recv_from = From}, - {Record, State} = next_record(State3), - next_event(hello, Record, State, [{{timeout, handshake}, Timeout, close} | Actions]); + next_event(hello, no_record, State3, [{{timeout, handshake}, Timeout, close} | Actions]); init({call, _} = Type, Event, #state{static_env = #static_env{role = server}, protocol_specific = PS} = State) -> Result = gen_handshake(?FUNCTION_NAME, Type, Event, @@ -497,9 +513,8 @@ hello(internal, #client_hello{cookie = <<>>, %% negotiated. VerifyRequest = dtls_handshake:hello_verify_request(Cookie, ?HELLO_VERIFY_REQUEST_VERSION), State1 = prepare_flight(State0#state{connection_env = CEnv#connection_env{negotiated_version = Version}}), - {State2, Actions} = send_handshake(VerifyRequest, State1), - {Record, State} = next_record(State2), - next_event(?FUNCTION_NAME, Record, + {State, Actions} = send_handshake(VerifyRequest, State1), + next_event(?FUNCTION_NAME, no_record, State#state{handshake_env = HsEnv#handshake_env{ tls_handshake_history = ssl_handshake:init_handshake_history()}}, @@ -537,14 +552,14 @@ hello(internal, #client_hello{extensions = Extensions} = Hello, start_or_recv_from = From} = State) -> {next_state, user_hello, State#state{start_or_recv_from = undefined, handshake_env = HsEnv#handshake_env{hello = Hello}}, - [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; + [{reply, From, {ok, Extensions}}]}; hello(internal, #server_hello{extensions = Extensions} = Hello, #state{ssl_options = #ssl_options{handshake = hello}, handshake_env = HsEnv, start_or_recv_from = From} = State) -> {next_state, user_hello, State#state{start_or_recv_from = undefined, handshake_env = HsEnv#handshake_env{hello = Hello}}, - [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; + [{reply, From, {ok, Extensions}}]}; hello(internal, #client_hello{cookie = Cookie} = Hello, #state{static_env = #static_env{role = server, transport_cb = Transport, @@ -701,12 +716,10 @@ connection(internal, #hello_request{}, #state{static_env = #static_env{host = Ho HelloVersion = dtls_record:hello_version(Version, SslOpts#ssl_options.versions), State1 = prepare_flight(State0), {State2, Actions} = send_handshake(Hello, State1#state{connection_env = CEnv#connection_env{negotiated_version = HelloVersion}}), - {Record, State} = - next_record( - State2#state{protocol_specific = PS#{flight_state => initial_flight_state(DataTag)}, - session = Session0#session{session_id - = Hello#client_hello.session_id}}), - next_event(hello, Record, State, Actions); + State = State2#state{protocol_specific = PS#{flight_state => initial_flight_state(DataTag)}, + session = Session0#session{session_id + = Hello#client_hello.session_id}}, + next_event(hello, no_record, State, Actions); connection(internal, #client_hello{} = Hello, #state{static_env = #static_env{role = server}, handshake_env = #handshake_env{allow_renegotiate = true} = HsEnv} = State) -> %% Mitigate Computational DoS attack @@ -762,7 +775,7 @@ format_status(Type, Data) -> %%% Internal functions %%-------------------------------------------------------------------- initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions, _}, User, - {CbModule, DataTag, CloseTag, ErrorTag}) -> + {CbModule, DataTag, CloseTag, ErrorTag, PassiveTag}) -> #ssl_options{beast_mitigation = BeastMitigation} = SSLOptions, ConnectionStates = dtls_record:init_connection_states(Role, BeastMitigation), @@ -772,7 +785,12 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions, _}, User, _ -> ssl_session_cache end, - + InternalActiveN = case application:get_env(ssl, internal_active_n) of + {ok, N} when is_integer(N) -> + N; + _ -> + ?INTERNAL_ACTIVE_N + end, Monitor = erlang:monitor(process, User), InitStatEnv = #static_env{ role = Role, @@ -781,6 +799,7 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions, _}, User, data_tag = DataTag, close_tag = CloseTag, error_tag = ErrorTag, + passive_tag = PassiveTag, host = Host, port = Port, socket = Socket, @@ -804,7 +823,9 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions, _}, User, user_data_buffer = {[],0,[]}, start_or_recv_from = undefined, flight_buffer = new_flight(), - protocol_specific = #{flight_state => initial_flight_state(DataTag)} + protocol_specific = #{active_n => InternalActiveN, + active_n_toggle => true, + flight_state => initial_flight_state(DataTag)} }. initial_flight_state(udp)-> @@ -814,10 +835,11 @@ initial_flight_state(_) -> next_dtls_record(Data, StateName, #state{protocol_buffers = #protocol_buffers{ dtls_record_buffer = Buf0, - dtls_cipher_texts = CT0} = Buffers} = State0) -> + dtls_cipher_texts = CT0} = Buffers, + ssl_options = SslOpts} = State0) -> case dtls_record:get_dtls_records(Data, acceptable_record_versions(StateName, State0), - Buf0) of + Buf0, SslOpts) of {Records, Buf1} -> CT1 = CT0 ++ Records, next_record(State0#state{protocol_buffers = @@ -828,7 +850,7 @@ next_dtls_record(Data, StateName, #state{protocol_buffers = #protocol_buffers{ end. acceptable_record_versions(hello, _) -> - [dtls_record:protocol_version(Vsn) || Vsn <- ?ALL_DATAGRAM_SUPPORTED_VERSIONS]; + [dtls_record:protocol_version(Vsn) || Vsn <- ?ALL_AVAILABLE_DATAGRAM_VERSIONS]; acceptable_record_versions(_, #state{connection_env = #connection_env{negotiated_version = Version}}) -> [Version]. @@ -900,12 +922,21 @@ handle_info({Protocol, _, _, _, Data}, StateName, ssl_connection:handle_normal_shutdown(Alert, StateName, State0), {stop, {shutdown, own_alert}, State0} end; + +handle_info({PassiveTag, Socket}, StateName, + #state{static_env = #static_env{socket = {_, Socket}, + passive_tag = PassiveTag}, + protocol_specific = PS} = State) -> + next_event(StateName, no_record, + State#state{protocol_specific = PS#{active_n_toggle => true}}); + handle_info({CloseTag, Socket}, StateName, #state{static_env = #static_env{socket = Socket, close_tag = CloseTag}, connection_env = #connection_env{negotiated_version = Version}, socket_options = #socket_options{active = Active}, - protocol_buffers = #protocol_buffers{dtls_cipher_texts = CTs}} = State) -> + protocol_buffers = #protocol_buffers{dtls_cipher_texts = CTs}, + protocol_specific = PS} = State) -> %% Note that as of DTLS 1.2 (TLS 1.1), %% failure to properly close a connection no longer requires that a %% session not be resumed. This is a change from DTLS 1.0 to conform @@ -928,7 +959,8 @@ handle_info({CloseTag, Socket}, StateName, %% Fixes non-delivery of final DTLS record in {active, once}. %% Basically allows the application the opportunity to set {active, once} again %% and then receive the final message. - next_event(StateName, no_record, State) + next_event(StateName, no_record, State#state{ + protocol_specific = PS#{active_n_toggle => true}}) end; handle_info(new_cookie_secret, StateName, @@ -963,7 +995,7 @@ handle_own_alert(Alert, Version, StateName, #state{static_env = #static_env{data ssl_options = Options} = State0) -> case ignore_alert(Alert, State0) of {true, State} -> - log_ignore_alert(Options#ssl_options.log_alert, StateName, Alert, Role), + log_ignore_alert(Options#ssl_options.log_level, StateName, Alert, Role), {next_state, StateName, State}; {false, State} -> ssl_connection:handle_own_alert(Alert, Version, StateName, State) @@ -1063,21 +1095,24 @@ start_retransmision_timer(Timeout, #state{protocol_specific = PS} = State) -> {State#state{protocol_specific = PS#{flight_state => {retransmit, new_timeout(Timeout)}}}, [{state_timeout, Timeout, flight_retransmission_timeout}]}. -new_timeout(N) when N =< 30 -> +new_timeout(N) when N =< 30000 -> N * 2; new_timeout(_) -> - 60. + 60000. send_handshake_flight(#state{static_env = #static_env{socket = Socket, transport_cb = Transport}, connection_env = #connection_env{negotiated_version = Version}, flight_buffer = #{handshakes := Flight, change_cipher_spec := undefined}, - connection_states = ConnectionStates0} = State0, Epoch) -> + connection_states = ConnectionStates0, + ssl_options = #ssl_options{log_level = LogLevel}} = State0, + Epoch) -> %% TODO remove hardcoded Max size {Encoded, ConnectionStates} = encode_handshake_flight(lists:reverse(Flight), Version, 1400, Epoch, ConnectionStates0), - send(Transport, Socket, Encoded), + send(Transport, Socket, Encoded), + ssl_logger:debug(LogLevel, outbound, 'record', Encoded), {State0#state{connection_states = ConnectionStates}, []}; send_handshake_flight(#state{static_env = #static_env{socket = Socket, @@ -1086,12 +1121,16 @@ send_handshake_flight(#state{static_env = #static_env{socket = Socket, flight_buffer = #{handshakes := [_|_] = Flight0, change_cipher_spec := ChangeCipher, handshakes_after_change_cipher_spec := []}, - connection_states = ConnectionStates0} = State0, Epoch) -> + connection_states = ConnectionStates0, + ssl_options = #ssl_options{log_level = LogLevel}} = State0, + Epoch) -> {HsBefore, ConnectionStates1} = encode_handshake_flight(lists:reverse(Flight0), Version, 1400, Epoch, ConnectionStates0), {EncChangeCipher, ConnectionStates} = encode_change_cipher(ChangeCipher, Version, Epoch, ConnectionStates1), send(Transport, Socket, [HsBefore, EncChangeCipher]), + ssl_logger:debug(LogLevel, outbound, 'record', [HsBefore]), + ssl_logger:debug(LogLevel, outbound, 'record', [EncChangeCipher]), {State0#state{connection_states = ConnectionStates}, []}; send_handshake_flight(#state{static_env = #static_env{socket = Socket, @@ -1100,7 +1139,9 @@ send_handshake_flight(#state{static_env = #static_env{socket = Socket, flight_buffer = #{handshakes := [_|_] = Flight0, change_cipher_spec := ChangeCipher, handshakes_after_change_cipher_spec := Flight1}, - connection_states = ConnectionStates0} = State0, Epoch) -> + connection_states = ConnectionStates0, + ssl_options = #ssl_options{log_level = LogLevel}} = State0, + Epoch) -> {HsBefore, ConnectionStates1} = encode_handshake_flight(lists:reverse(Flight0), Version, 1400, Epoch-1, ConnectionStates0), {EncChangeCipher, ConnectionStates2} = @@ -1108,6 +1149,9 @@ send_handshake_flight(#state{static_env = #static_env{socket = Socket, {HsAfter, ConnectionStates} = encode_handshake_flight(lists:reverse(Flight1), Version, 1400, Epoch, ConnectionStates2), send(Transport, Socket, [HsBefore, EncChangeCipher, HsAfter]), + ssl_logger:debug(LogLevel, outbound, 'record', [HsBefore]), + ssl_logger:debug(LogLevel, outbound, 'record', [EncChangeCipher]), + ssl_logger:debug(LogLevel, outbound, 'record', [HsAfter]), {State0#state{connection_states = ConnectionStates}, []}; send_handshake_flight(#state{static_env = #static_env{socket = Socket, @@ -1116,12 +1160,16 @@ send_handshake_flight(#state{static_env = #static_env{socket = Socket, flight_buffer = #{handshakes := [], change_cipher_spec := ChangeCipher, handshakes_after_change_cipher_spec := Flight1}, - connection_states = ConnectionStates0} = State0, Epoch) -> + connection_states = ConnectionStates0, + ssl_options = #ssl_options{log_level = LogLevel}} = State0, + Epoch) -> {EncChangeCipher, ConnectionStates1} = encode_change_cipher(ChangeCipher, Version, Epoch-1, ConnectionStates0), {HsAfter, ConnectionStates} = encode_handshake_flight(lists:reverse(Flight1), Version, 1400, Epoch, ConnectionStates1), send(Transport, Socket, [EncChangeCipher, HsAfter]), + ssl_logger:debug(LogLevel, outbound, 'record', [EncChangeCipher]), + ssl_logger:debug(LogLevel, outbound, 'record', [HsAfter]), {State0#state{connection_states = ConnectionStates}, []}. retransmit_epoch(_StateName, #state{connection_states = ConnectionStates}) -> @@ -1160,11 +1208,11 @@ is_ignore_alert(#alert{description = ?ILLEGAL_PARAMETER}) -> is_ignore_alert(_) -> false. -log_ignore_alert(true, StateName, Alert, Role) -> +log_ignore_alert(debug, StateName, Alert, Role) -> Txt = ssl_alert:alert_txt(Alert), - error_logger:format("DTLS over UDP ~p: In state ~p ignored to send ALERT ~s as DoS-attack mitigation \n", - [Role, StateName, Txt]); -log_ignore_alert(false, _, _,_) -> + ?LOG_ERROR("DTLS over UDP ~p: In state ~p ignored to send ALERT ~s as DoS-attack mitigation \n", + [Role, StateName, Txt]); +log_ignore_alert(_, _, _, _) -> ok. send_application_data(Data, From, _StateName, @@ -1173,7 +1221,8 @@ send_application_data(Data, From, _StateName, connection_env = #connection_env{negotiated_version = Version}, handshake_env = HsEnv, connection_states = ConnectionStates0, - ssl_options = #ssl_options{renegotiate_at = RenegotiateAt}} = State0) -> + ssl_options = #ssl_options{renegotiate_at = RenegotiateAt, + log_level = LogLevel}} = State0) -> case time_to_renegotiate(Data, ConnectionStates0, RenegotiateAt) of true -> @@ -1185,6 +1234,7 @@ send_application_data(Data, From, _StateName, State = State0#state{connection_states = ConnectionStates}, case send(Transport, Socket, Msgs) of ok -> + ssl_logger:debug(LogLevel, outbound, 'record', Msgs), ssl_connection:hibernate_after(connection, State, [{reply, From, ok}]); Result -> ssl_connection:hibernate_after(connection, State, [{reply, From, Result}]) diff --git a/lib/ssl/src/dtls_handshake.erl b/lib/ssl/src/dtls_handshake.erl index 6e9bf99e52..d8c0e30973 100644 --- a/lib/ssl/src/dtls_handshake.erl +++ b/lib/ssl/src/dtls_handshake.erl @@ -37,7 +37,7 @@ -export([fragment_handshake/2, encode_handshake/3]). %% Handshake decodeing --export([get_dtls_handshake/3]). +-export([get_dtls_handshake/4]). -type dtls_handshake() :: #client_hello{} | #hello_verify_request{} | ssl_handshake:ssl_handshake(). @@ -79,7 +79,7 @@ client_hello(Host, Port, Cookie, ConnectionStates, Extensions = ssl_handshake:client_hello_extensions(TLSVersion, CipherSuites, SslOpts, ConnectionStates, - Renegotiation), + Renegotiation, undefined), Id = ssl_session:client_id({Host, Port, SslOpts}, Cache, CacheCb, OwnCert), #client_hello{session_id = Id, @@ -151,15 +151,15 @@ encode_handshake(Handshake, Version, Seq) -> %%-------------------------------------------------------------------- %%-------------------------------------------------------------------- --spec get_dtls_handshake(ssl_record:ssl_version(), binary(), #protocol_buffers{}) -> +-spec get_dtls_handshake(ssl_record:ssl_version(), binary(), #protocol_buffers{}, #ssl_options{}) -> {[dtls_handshake()], #protocol_buffers{}}. %% %% Description: Given buffered and new data from dtls_record, collects %% and returns it as a list of handshake messages, also returns %% possible leftover data in the new "protocol_buffers". %%-------------------------------------------------------------------- -get_dtls_handshake(Version, Fragment, ProtocolBuffers) -> - handle_fragments(Version, Fragment, ProtocolBuffers, []). +get_dtls_handshake(Version, Fragment, ProtocolBuffers, Options) -> + handle_fragments(Version, Fragment, ProtocolBuffers, Options, []). %%-------------------------------------------------------------------- %%% Internal functions @@ -169,10 +169,7 @@ handle_client_hello(Version, cipher_suites = CipherSuites, compression_methods = Compressions, random = Random, - extensions = - #hello_extensions{elliptic_curves = Curves, - signature_algs = ClientHashSigns} - = HelloExt}, + extensions = HelloExt}, #ssl_options{versions = Versions, signature_algs = SupportedHashSigns, eccs = SupportedECCs, @@ -181,6 +178,8 @@ handle_client_hello(Version, Renegotiation) -> case dtls_record:is_acceptable_version(Version, Versions) of true -> + Curves = maps:get(elliptic_curves, HelloExt, undefined), + ClientHashSigns = maps:get(signature_algs, HelloExt, undefined), TLSVersion = dtls_v1:corresponding_tls_version(Version), AvailableHashSigns = ssl_handshake:available_signature_algs( ClientHashSigns, SupportedHashSigns, Cert,TLSVersion), @@ -194,8 +193,8 @@ handle_client_hello(Version, no_suite -> ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY); _ -> - #{key_exchange := KeyExAlg} = ssl_cipher_format:suite_definition(CipherSuite), - case ssl_handshake:select_hashsign(ClientHashSigns, Cert, KeyExAlg, + #{key_exchange := KeyExAlg} = ssl_cipher_format:suite_bin_to_map(CipherSuite), + case ssl_handshake:select_hashsign({ClientHashSigns, undefined}, Cert, KeyExAlg, SupportedHashSigns, TLSVersion) of #alert{} = Alert -> Alert; @@ -311,20 +310,21 @@ address_to_bin({A,B,C,D,E,F,G,H}, Port) -> %%-------------------------------------------------------------------- -handle_fragments(Version, FragmentData, Buffers0, Acc) -> +handle_fragments(Version, FragmentData, Buffers0, Options, Acc) -> Fragments = decode_handshake_fragments(FragmentData), - do_handle_fragments(Version, Fragments, Buffers0, Acc). + do_handle_fragments(Version, Fragments, Buffers0, Options, Acc). -do_handle_fragments(_, [], Buffers, Acc) -> +do_handle_fragments(_, [], Buffers, _Options, Acc) -> {lists:reverse(Acc), Buffers}; -do_handle_fragments(Version, [Fragment | Fragments], Buffers0, Acc) -> +do_handle_fragments(Version, [Fragment | Fragments], Buffers0, Options, Acc) -> case reassemble(Version, Fragment, Buffers0) of {more_data, Buffers} when Fragments == [] -> {lists:reverse(Acc), Buffers}; {more_data, Buffers} -> - do_handle_fragments(Version, Fragments, Buffers, Acc); - {HsPacket, Buffers} -> - do_handle_fragments(Version, Fragments, Buffers, [HsPacket | Acc]) + do_handle_fragments(Version, Fragments, Buffers, Options, Acc); + {{Handshake, _} = HsPacket, Buffers} -> + ssl_logger:debug(Options#ssl_options.log_level, inbound, 'handshake', Handshake), + do_handle_fragments(Version, Fragments, Buffers, Options, [HsPacket | Acc]) end. decode_handshake(Version, <<?BYTE(Type), Bin/binary>>) -> @@ -332,7 +332,7 @@ decode_handshake(Version, <<?BYTE(Type), Bin/binary>>) -> decode_handshake(_, ?HELLO_REQUEST, <<>>) -> #hello_request{}; -decode_handshake(_Version, ?CLIENT_HELLO, <<?UINT24(_), ?UINT16(_), +decode_handshake(Version, ?CLIENT_HELLO, <<?UINT24(_), ?UINT16(_), ?UINT24(_), ?UINT24(_), ?BYTE(Major), ?BYTE(Minor), Random:32/binary, ?BYTE(SID_length), Session_ID:SID_length/binary, @@ -340,8 +340,10 @@ decode_handshake(_Version, ?CLIENT_HELLO, <<?UINT24(_), ?UINT16(_), ?UINT16(Cs_length), CipherSuites:Cs_length/binary, ?BYTE(Cm_length), Comp_methods:Cm_length/binary, Extensions/binary>>) -> - - DecodedExtensions = ssl_handshake:decode_hello_extensions({client, Extensions}), + TLSVersion = dtls_v1:corresponding_tls_version(Version), + LegacyVersion = dtls_v1:corresponding_tls_version({Major, Minor}), + Exts = ssl_handshake:decode_vector(Extensions), + DecodedExtensions = ssl_handshake:decode_hello_extensions(Exts, TLSVersion, LegacyVersion, client), #client_hello{ client_version = {Major,Minor}, @@ -362,9 +364,9 @@ decode_handshake(_Version, ?HELLO_VERIFY_REQUEST, <<?UINT24(_), ?UINT16(_), decode_handshake(Version, Tag, <<?UINT24(_), ?UINT16(_), ?UINT24(_), ?UINT24(_), Msg/binary>>) -> %% DTLS specifics stripped - decode_tls_thandshake(Version, Tag, Msg). + decode_tls_handshake(Version, Tag, Msg). -decode_tls_thandshake(Version, Tag, Msg) -> +decode_tls_handshake(Version, Tag, Msg) -> TLSVersion = dtls_v1:corresponding_tls_version(Version), ssl_handshake:decode_handshake(TLSVersion, Tag, Msg). @@ -425,74 +427,135 @@ merge_fragment(Frag0, [Frag1 | Rest]) -> Frag -> merge_fragment(Frag, Rest) end. -%% Duplicate + + +%% Duplicate (fully contained fragment) +%% 2,5 _ _ P P P P P +%% 2,5 _ _ C C C C C merge_fragments(#handshake_fragment{ - fragment_offset = PreviousOffSet, + fragment_offset = PreviousOffSet, fragment_length = PreviousLen, fragment = PreviousData - } = Previous, + } = Previous, #handshake_fragment{ fragment_offset = PreviousOffSet, fragment_length = PreviousLen, fragment = PreviousData}) -> Previous; -%% Lager fragment save new data +%% Duplicate (fully contained fragment) +%% 2,5 _ _ P P P P P +%% 2,2 _ _ C C +%% 0,3 X X X +%% 5,3 _ _ _ _ _ X X X merge_fragments(#handshake_fragment{ - fragment_offset = PreviousOffSet, - fragment_length = PreviousLen, + fragment_offset = PreviousOffset, + fragment_length = PreviousLen + } = Previous, + #handshake_fragment{ + fragment_offset = CurrentOffset, + fragment_length = CurrentLen}) + when PreviousOffset =< CurrentOffset andalso + CurrentOffset =< PreviousOffset + PreviousLen andalso + CurrentOffset + CurrentLen =< PreviousOffset + PreviousLen -> + Previous; + +%% Fully overlapping fragments +%% 2,5 _ _ P P P P P +%% 0,8 C C C C C C C C +merge_fragments(#handshake_fragment{ + fragment_offset = PreviousOffset, + fragment_length = PreviousLen + }, + #handshake_fragment{ + fragment_offset = CurrentOffset, + fragment_length = CurrentLen} = Current) + when CurrentOffset =< PreviousOffset andalso + CurrentOffset + CurrentLen >= PreviousOffset + PreviousLen -> + Current; + +%% Overlapping fragments +%% 2,5 _ _ P P P P P +%% 0,3 C C C +merge_fragments(#handshake_fragment{ + fragment_offset = PreviousOffset, + fragment_length = PreviousLen, fragment = PreviousData - } = Previous, - #handshake_fragment{ - fragment_offset = PreviousOffSet, - fragment_length = CurrentLen, - fragment = CurrentData}) when CurrentLen > PreviousLen -> - NewLength = CurrentLen - PreviousLen, - <<_:PreviousLen/binary, NewData/binary>> = CurrentData, + } = Previous, + #handshake_fragment{ + fragment_offset = CurrentOffset, + fragment_length = CurrentLen, + fragment = CurrentData}) + when CurrentOffset < PreviousOffset andalso + CurrentOffset + CurrentLen < PreviousOffset + PreviousLen -> + NewDataLen = PreviousOffset - CurrentOffset, + <<NewData:NewDataLen/binary, _/binary>> = CurrentData, Previous#handshake_fragment{ - fragment_length = PreviousLen + NewLength, - fragment = <<PreviousData/binary, NewData/binary>> + fragment_length = PreviousLen + NewDataLen, + fragment = <<NewData/binary, PreviousData/binary>> }; -%% Smaller fragment +%% Overlapping fragments +%% 2,5 _ _ P P P P P +%% 5,3 _ _ _ _ _ C C C merge_fragments(#handshake_fragment{ - fragment_offset = PreviousOffSet, - fragment_length = PreviousLen - } = Previous, - #handshake_fragment{ - fragment_offset = PreviousOffSet, - fragment_length = CurrentLen}) when CurrentLen < PreviousLen -> - Previous; -%% Next fragment, might be overlapping + fragment_offset = PreviousOffset, + fragment_length = PreviousLen, + fragment = PreviousData + } = Previous, + #handshake_fragment{ + fragment_offset = CurrentOffset, + fragment_length = CurrentLen, + fragment = CurrentData}) + when CurrentOffset > PreviousOffset andalso + CurrentOffset < PreviousOffset + PreviousLen -> + NewDataLen = CurrentOffset + CurrentLen - (PreviousOffset + PreviousLen), + DropLen = CurrentLen - NewDataLen, + <<_:DropLen/binary, NewData/binary>> = CurrentData, + Previous#handshake_fragment{ + fragment_length = PreviousLen + NewDataLen, + fragment = <<PreviousData/binary, NewData/binary>> + }; + +%% Adjacent fragments +%% 2,5 _ _ P P P P P +%% 7,3 _ _ _ _ _ _ _ C C C merge_fragments(#handshake_fragment{ - fragment_offset = PreviousOffSet, - fragment_length = PreviousLen, + fragment_offset = PreviousOffset, + fragment_length = PreviousLen, fragment = PreviousData - } = Previous, - #handshake_fragment{ - fragment_offset = CurrentOffSet, - fragment_length = CurrentLen, - fragment = CurrentData}) - when PreviousOffSet + PreviousLen >= CurrentOffSet andalso - PreviousOffSet + PreviousLen < CurrentOffSet + CurrentLen -> - CurrentStart = PreviousOffSet + PreviousLen - CurrentOffSet, - <<_:CurrentStart/bytes, Data/binary>> = CurrentData, + } = Previous, + #handshake_fragment{ + fragment_offset = CurrentOffset, + fragment_length = CurrentLen, + fragment = CurrentData}) + when CurrentOffset =:= PreviousOffset + PreviousLen -> Previous#handshake_fragment{ - fragment_length = PreviousLen + CurrentLen - CurrentStart, - fragment = <<PreviousData/binary, Data/binary>>}; -%% already fully contained fragment + fragment_length = PreviousLen + CurrentLen, + fragment = <<PreviousData/binary, CurrentData/binary>> + }; + +%% Adjacent fragments +%% 2,5 _ _ P P P P P +%% 0,2 C C merge_fragments(#handshake_fragment{ - fragment_offset = PreviousOffSet, - fragment_length = PreviousLen - } = Previous, + fragment_offset = PreviousOffset, + fragment_length = PreviousLen, + fragment = PreviousData + } = Previous, #handshake_fragment{ - fragment_offset = CurrentOffSet, - fragment_length = CurrentLen}) - when PreviousOffSet + PreviousLen >= CurrentOffSet andalso - PreviousOffSet + PreviousLen >= CurrentOffSet + CurrentLen -> - Previous; + fragment_offset = CurrentOffset, + fragment_length = CurrentLen, + fragment = CurrentData}) + when PreviousOffset =:= CurrentOffset + CurrentLen -> + Previous#handshake_fragment{ + fragment_length = PreviousLen + CurrentLen, + fragment = <<CurrentData/binary, PreviousData/binary>> + }; %% No merge there is a gap +%% 3,5 _ _ _ P P P P +%% 0,2 C C merge_fragments(Previous, Current) -> [Previous, Current]. diff --git a/lib/ssl/src/dtls_handshake.hrl b/lib/ssl/src/dtls_handshake.hrl index 41da8e5c8c..de2be1daeb 100644 --- a/lib/ssl/src/dtls_handshake.hrl +++ b/lib/ssl/src/dtls_handshake.hrl @@ -26,23 +26,13 @@ -ifndef(dtls_handshake). -define(dtls_handshake, true). +-include("tls_handshake.hrl"). %% Common TLS and DTLS records and Constantes -include("ssl_handshake.hrl"). %% Common TLS and DTLS records and Constantes -include("ssl_api.hrl"). -define(HELLO_VERIFY_REQUEST, 3). -define(HELLO_VERIFY_REQUEST_VERSION, {254, 255}). --record(client_hello, { - client_version, - random, - session_id, % opaque SessionID<0..32> - cookie, % opaque<2..2^16-1> - cipher_suites, % cipher_suites<2..2^16-1> - compression_methods, % compression_methods<1..2^8-1>, - %% Extensions - extensions - }). - -record(hello_verify_request, { protocol_version, cookie @@ -57,4 +47,11 @@ fragment }). +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% RFC 7764 Datagram Transport Layer Security (DTLS) Extension to Establish Keys +%% for the Secure Real-time Transport Protocol (SRTP) +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Not supported +-define(USE_SRTP, 14). + -endif. % -ifdef(dtls_handshake). diff --git a/lib/ssl/src/dtls_packet_demux.erl b/lib/ssl/src/dtls_packet_demux.erl index 092366b7c0..c6431b55a9 100644 --- a/lib/ssl/src/dtls_packet_demux.erl +++ b/lib/ssl/src/dtls_packet_demux.erl @@ -24,6 +24,7 @@ -behaviour(gen_server). -include("ssl_internal.hrl"). +-include_lib("kernel/include/logger.hrl"). %% API -export([start_link/5, active_once/3, accept/2, sockname/1, close/1, @@ -34,7 +35,8 @@ terminate/2, code_change/3]). -record(state, - {port, + {active_n, + port, listener, transport, dtls_options, @@ -75,10 +77,18 @@ set_sock_opts(PacketSocket, Opts) -> %%% gen_server callbacks %%%=================================================================== -init([Port, {TransportModule, _,_,_} = TransportInfo, EmOpts, InetOptions, DTLSOptions]) -> +init([Port, {TransportModule, _,_,_,_} = TransportInfo, EmOpts, InetOptions, DTLSOptions]) -> try {ok, Socket} = TransportModule:open(Port, InetOptions), - {ok, #state{port = Port, + InternalActiveN = case application:get_env(ssl, internal_active_n) of + {ok, N} when is_integer(N) -> + N; + _ -> + ?INTERNAL_ACTIVE_N + end, + + {ok, #state{active_n = InternalActiveN, + port = Port, first = true, transport = TransportInfo, dtls_options = DTLSOptions, @@ -91,10 +101,11 @@ init([Port, {TransportModule, _,_,_} = TransportInfo, EmOpts, InetOptions, DTLSO handle_call({accept, _}, _, #state{close = true} = State) -> {reply, {error, closed}, State}; -handle_call({accept, Accepter}, From, #state{first = true, +handle_call({accept, Accepter}, From, #state{active_n = N, + first = true, accepters = Accepters, listener = Socket} = State0) -> - next_datagram(Socket), + next_datagram(Socket, N), State = State0#state{first = false, accepters = queue:in({Accepter, From}, Accepters)}, {noreply, State}; @@ -136,21 +147,26 @@ handle_cast({active_once, Client, Pid}, State0) -> State = handle_active_once(Client, Pid, State0), {noreply, State}. -handle_info({Transport, Socket, IP, InPortNo, _} = Msg, #state{listener = Socket, transport = {_,Transport,_,_}} = State0) -> +handle_info({Transport, Socket, IP, InPortNo, _} = Msg, #state{listener = Socket, transport = {_,Transport,_,_,_}} = State0) -> State = handle_datagram({IP, InPortNo}, Msg, State0), - next_datagram(Socket), {noreply, State}; +handle_info({PassiveTag, Socket}, + #state{active_n = N, + listener = Socket, + transport = {_,_,_, udp_error, PassiveTag}}) -> + next_datagram(Socket, N); + %% UDP socket does not have a connection and should not receive an econnreset %% This does however happens on some windows versions. Just ignoring it %% appears to make things work as expected! -handle_info({udp_error, Socket, econnreset = Error}, #state{listener = Socket, transport = {_,_,_, udp_error}} = State) -> +handle_info({udp_error, Socket, econnreset = Error}, #state{listener = Socket, transport = {_,_,_, udp_error,_}} = State) -> Report = io_lib:format("Ignore SSL UDP Listener: Socket error: ~p ~n", [Error]), - error_logger:info_report(Report), + ?LOG_NOTICE(Report), {noreply, State}; -handle_info({ErrorTag, Socket, Error}, #state{listener = Socket, transport = {_,_,_, ErrorTag}} = State) -> +handle_info({ErrorTag, Socket, Error}, #state{listener = Socket, transport = {_,_,_, ErrorTag,_}} = State) -> Report = io_lib:format("SSL Packet muliplxer shutdown: Socket error: ~p ~n", [Error]), - error_logger:info_report(Report), + ?LOG_NOTICE(Report), {noreply, State#state{close=true}}; handle_info({'DOWN', _, process, Pid, _}, #state{clients = Clients, @@ -202,16 +218,16 @@ dispatch(Client, Msg, #state{dtls_msq_queues = MsgQueues} = State) -> Pid ! Msg, State#state{dtls_msq_queues = kv_update(Client, Queue, MsgQueues)}; - {{value, _}, Queue} -> + {{value, _UDP}, _Queue} -> State#state{dtls_msq_queues = - kv_update(Client, queue:in(Msg, Queue), MsgQueues)}; + kv_update(Client, queue:in(Msg, Queue0), MsgQueues)}; {empty, Queue} -> State#state{dtls_msq_queues = kv_update(Client, queue:in(Msg, Queue), MsgQueues)} end end. -next_datagram(Socket) -> - inet:setopts(Socket, [{active, once}]). +next_datagram(Socket, N) -> + inet:setopts(Socket, [{active, N}]). handle_active_once(Client, Pid, #state{dtls_msq_queues = MsgQueues} = State0) -> Queue0 = kv_get(Client, MsgQueues), diff --git a/lib/ssl/src/dtls_record.erl b/lib/ssl/src/dtls_record.erl index 2fe875da31..a4846f42c5 100644 --- a/lib/ssl/src/dtls_record.erl +++ b/lib/ssl/src/dtls_record.erl @@ -30,7 +30,7 @@ -include("ssl_cipher.hrl"). %% Handling of incoming data --export([get_dtls_records/3, init_connection_states/2, empty_connection_state/1]). +-export([get_dtls_records/4, init_connection_states/2, empty_connection_state/1]). -export([save_current_connection_state/2, next_epoch/2, get_connection_state_by_epoch/3, replay_detect/2, init_connection_state_seq/2, current_connection_state_epoch/2]). @@ -162,24 +162,25 @@ current_connection_state_epoch(#{current_write := #{epoch := Epoch}}, Epoch. %%-------------------------------------------------------------------- --spec get_dtls_records(binary(), [ssl_record:ssl_version()], binary()) -> {[binary()], binary()} | #alert{}. +-spec get_dtls_records(binary(), [ssl_record:ssl_version()], binary(), + #ssl_options{}) -> {[binary()], binary()} | #alert{}. %% %% Description: Given old buffer and new data from UDP/SCTP, packs up a records %% and returns it as a list of tls_compressed binaries also returns leftover %% data %%-------------------------------------------------------------------- -get_dtls_records(Data, Versions, Buffer) -> +get_dtls_records(Data, Versions, Buffer, SslOpts) -> BinData = list_to_binary([Buffer, Data]), case erlang:byte_size(BinData) of N when N >= 3 -> case assert_version(BinData, Versions) of true -> - get_dtls_records_aux(BinData, []); + get_dtls_records_aux(BinData, [], SslOpts); false -> ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC) end; _ -> - get_dtls_records_aux(BinData, []) + get_dtls_records_aux(BinData, [], SslOpts) end. %%==================================================================== @@ -409,42 +410,47 @@ assert_version(<<?BYTE(_), ?BYTE(MajVer), ?BYTE(MinVer), _/binary>>, Versions) - get_dtls_records_aux(<<?BYTE(?APPLICATION_DATA),?BYTE(MajVer),?BYTE(MinVer), ?UINT16(Epoch), ?UINT48(SequenceNumber), - ?UINT16(Length), Data:Length/binary, Rest/binary>>, - Acc) -> + ?UINT16(Length), Data:Length/binary, Rest/binary>> = RawDTLSRecord, + Acc, SslOpts) -> + ssl_logger:debug(SslOpts#ssl_options.log_level, inbound, 'record', [RawDTLSRecord]), get_dtls_records_aux(Rest, [#ssl_tls{type = ?APPLICATION_DATA, version = {MajVer, MinVer}, epoch = Epoch, sequence_number = SequenceNumber, - fragment = Data} | Acc]); + fragment = Data} | Acc], SslOpts); get_dtls_records_aux(<<?BYTE(?HANDSHAKE),?BYTE(MajVer),?BYTE(MinVer), ?UINT16(Epoch), ?UINT48(SequenceNumber), ?UINT16(Length), - Data:Length/binary, Rest/binary>>, Acc) when MajVer >= 128 -> + Data:Length/binary, Rest/binary>> = RawDTLSRecord, + Acc, SslOpts) when MajVer >= 128 -> + ssl_logger:debug(SslOpts#ssl_options.log_level, inbound, 'record', [RawDTLSRecord]), get_dtls_records_aux(Rest, [#ssl_tls{type = ?HANDSHAKE, version = {MajVer, MinVer}, epoch = Epoch, sequence_number = SequenceNumber, - fragment = Data} | Acc]); + fragment = Data} | Acc], SslOpts); get_dtls_records_aux(<<?BYTE(?ALERT),?BYTE(MajVer),?BYTE(MinVer), ?UINT16(Epoch), ?UINT48(SequenceNumber), ?UINT16(Length), Data:Length/binary, - Rest/binary>>, Acc) -> + Rest/binary>> = RawDTLSRecord, Acc, SslOpts) -> + ssl_logger:debug(SslOpts#ssl_options.log_level, inbound, 'record', [RawDTLSRecord]), get_dtls_records_aux(Rest, [#ssl_tls{type = ?ALERT, version = {MajVer, MinVer}, epoch = Epoch, sequence_number = SequenceNumber, - fragment = Data} | Acc]); + fragment = Data} | Acc], SslOpts); get_dtls_records_aux(<<?BYTE(?CHANGE_CIPHER_SPEC),?BYTE(MajVer),?BYTE(MinVer), ?UINT16(Epoch), ?UINT48(SequenceNumber), - ?UINT16(Length), Data:Length/binary, Rest/binary>>, - Acc) -> + ?UINT16(Length), Data:Length/binary, Rest/binary>> = RawDTLSRecord, + Acc, SslOpts) -> + ssl_logger:debug(SslOpts#ssl_options.log_level, inbound, 'record', [RawDTLSRecord]), get_dtls_records_aux(Rest, [#ssl_tls{type = ?CHANGE_CIPHER_SPEC, version = {MajVer, MinVer}, epoch = Epoch, sequence_number = SequenceNumber, - fragment = Data} | Acc]); + fragment = Data} | Acc], SslOpts); get_dtls_records_aux(<<?BYTE(_), ?BYTE(_MajVer), ?BYTE(_MinVer), ?UINT16(Length), _/binary>>, - _Acc) when Length > ?MAX_CIPHER_TEXT_LENGTH -> + _Acc, _) when Length > ?MAX_CIPHER_TEXT_LENGTH -> ?ALERT_REC(?FATAL, ?RECORD_OVERFLOW); -get_dtls_records_aux(Data, Acc) -> +get_dtls_records_aux(Data, Acc, _) -> case size(Data) =< ?MAX_CIPHER_TEXT_LENGTH + ?INITIAL_BYTES of true -> {lists:reverse(Acc), Data}; diff --git a/lib/ssl/src/dtls_socket.erl b/lib/ssl/src/dtls_socket.erl index 4d07372e31..b305d08f70 100644 --- a/lib/ssl/src/dtls_socket.erl +++ b/lib/ssl/src/dtls_socket.erl @@ -45,7 +45,7 @@ listen(Port, #config{transport_info = TransportInfo, Err end. -accept(dtls, #config{transport_info = {Transport,_,_,_}, +accept(dtls, #config{transport_info = {Transport,_,_,_,_}, connection_cb = ConnectionCb, dtls_handler = {Listner, _}}, _Timeout) -> case dtls_packet_demux:accept(Listner, self()) of @@ -55,7 +55,7 @@ accept(dtls, #config{transport_info = {Transport,_,_,_}, {error, Reason} end. -connect(Address, Port, #config{transport_info = {Transport, _, _, _} = CbInfo, +connect(Address, Port, #config{transport_info = {Transport, _, _, _, _} = CbInfo, connection_cb = ConnectionCb, ssl = SslOpts, emulated = EmOpts, @@ -174,7 +174,7 @@ default_inet_values() -> [{active, true}, {mode, list}, {packet, 0}, {packet_size, 0}]. default_cb_info() -> - {gen_udp, udp, udp_closed, udp_error}. + {gen_udp, udp, udp_closed, udp_error, udp_passive}. get_emulated_opts(EmOpts, EmOptNames) -> lists:map(fun(Name) -> {value, Value} = lists:keysearch(Name, 1, EmOpts), diff --git a/lib/ssl/src/dtls_v1.erl b/lib/ssl/src/dtls_v1.erl index b365961a6a..fc9dce02ce 100644 --- a/lib/ssl/src/dtls_v1.erl +++ b/lib/ssl/src/dtls_v1.erl @@ -31,18 +31,18 @@ suites(Minor) -> lists:filter(fun(Cipher) -> - is_acceptable_cipher(ssl_cipher_format:suite_definition(Cipher)) + is_acceptable_cipher(ssl_cipher_format:suite_bin_to_map(Cipher)) end, tls_v1:suites(corresponding_minor_tls_version(Minor))). all_suites(Version) -> lists:filter(fun(Cipher) -> - is_acceptable_cipher(ssl_cipher_format:suite_definition(Cipher)) + is_acceptable_cipher(ssl_cipher_format:suite_bin_to_map(Cipher)) end, ssl_cipher:all_suites(corresponding_tls_version(Version))). anonymous_suites(Version) -> lists:filter(fun(Cipher) -> - is_acceptable_cipher(ssl_cipher_format:suite_definition(Cipher)) + is_acceptable_cipher(ssl_cipher_format:suite_bin_to_map(Cipher)) end, ssl_cipher:anonymous_suites(corresponding_tls_version(Version))). diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl index 5cab35fd4b..8d9b92361b 100644 --- a/lib/ssl/src/inet_tls_dist.erl +++ b/lib/ssl/src/inet_tls_dist.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2011-2018. All Rights Reserved. +%% Copyright Ericsson AB 2011-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -41,6 +41,7 @@ -include_lib("public_key/include/public_key.hrl"). -include("ssl_api.hrl"). +-include_lib("kernel/include/logger.hrl"). %% ------------------------------------------------------------------------- @@ -131,8 +132,8 @@ f_recv(SslSocket, Length, Timeout) -> f_setopts_pre_nodeup(_SslSocket) -> ok. -f_setopts_post_nodeup(_SslSocket) -> - ok. +f_setopts_post_nodeup(SslSocket) -> + ssl:setopts(SslSocket, [nodelay()]). f_getll(DistCtrl) -> {ok, DistCtrl}. @@ -198,7 +199,7 @@ listen(Name) -> gen_listen(Driver, Name) -> case inet_tcp_dist:gen_listen(Driver, Name) of {ok, {Socket, Address, Creation}} -> - inet:setopts(Socket, [{packet, 4}]), + inet:setopts(Socket, [{packet, 4}, {nodelay, true}]), {ok, {Socket, Address#net_address{protocol=tls}, Creation}}; Other -> Other @@ -225,7 +226,7 @@ accept_loop(Driver, Listen, Kernel) -> true -> accept_loop(Driver, Listen, Kernel, Socket); {false,IP} -> - error_logger:error_msg( + ?LOG_ERROR( "** Connection attempt from " "disallowed IP ~w ** ~n", [IP]), ?shutdown2(no_node, trace({disallowed, IP})) @@ -260,7 +261,7 @@ accept_loop(Driver, Listen, Kernel, Socket) -> {error, {options, _}} = Error -> %% Bad options: that's probably our fault. %% Let's log that. - error_logger:error_msg( + ?LOG_ERROR( "Cannot accept TLS distribution connection: ~s~n", [ssl:format_error(Error)]), gen_tcp:close(Socket), @@ -436,7 +437,7 @@ allowed_nodes(SslSocket, Allowed) -> PeerCert, allowed_hosts(Allowed), PeerIP) of [] -> - error_logger:error_msg( + ?LOG_ERROR( "** Connection attempt from " "disallowed node(s) ~p ** ~n", [PeerIP]), ?shutdown2( @@ -531,7 +532,7 @@ do_setup_connect(Driver, Kernel, Node, Address, Ip, TcpPort, Version, Type, MyNo case ssl:connect( Address, TcpPort, [binary, {active, false}, {packet, 4}, - Driver:family(), nodelay()] ++ Opts, + Driver:family(), {nodelay, true}] ++ Opts, net_kernel:connecttime()) of {ok, #sslsocket{pid = [_, DistCtrl| _]} = SslSocket} -> _ = monitor_pid(DistCtrl), @@ -695,12 +696,12 @@ split_node(Driver, Node, LongOrShortNames) -> {node, Name, Host} -> check_node(Driver, Node, Name, Host, LongOrShortNames); {host, _} -> - error_logger:error_msg( + ?LOG_ERROR( "** Nodename ~p illegal, no '@' character **~n", [Node]), ?shutdown2(Node, trace({illegal_node_n@me, Node})); _ -> - error_logger:error_msg( + ?LOG_ERROR( "** Nodename ~p illegal **~n", [Node]), ?shutdown2(Node, trace({illegal_node_name, Node})) end. @@ -712,7 +713,7 @@ check_node(Driver, Node, Name, Host, LongOrShortNames) -> {ok, _} -> {Name, Host}; _ -> - error_logger:error_msg( + ?LOG_ERROR( "** System running to use " "fully qualified hostnames **~n" "** Hostname ~s is illegal **~n", @@ -720,7 +721,7 @@ check_node(Driver, Node, Name, Host, LongOrShortNames) -> ?shutdown2(Node, trace({not_longnames, Host})) end; [_,_|_] when LongOrShortNames =:= shortnames -> - error_logger:error_msg( + ?LOG_ERROR( "** System NOT running to use " "fully qualified hostnames **~n" "** Hostname ~s is illegal **~n", @@ -850,13 +851,13 @@ monitor_pid(Pid) -> %% MRef = erlang:monitor(process, Pid), %% receive %% {'DOWN', MRef, _, _, normal} -> - %% error_logger:error_report( - %% [dist_proc_died, + %% ?LOG_ERROR( + %% [{slogan, dist_proc_died}, %% {reason, normal}, %% {pid, Pid}]); %% {'DOWN', MRef, _, _, Reason} -> - %% error_logger:info_report( - %% [dist_proc_died, + %% ?LOG_NOTICE( + %% [{slogan, dist_proc_died}, %% {reason, Reason}, %% {pid, Pid}]) %% end diff --git a/lib/ssl/src/ssl.app.src b/lib/ssl/src/ssl.app.src index 936df12e70..e7a4d73ec4 100644 --- a/lib/ssl/src/ssl.app.src +++ b/lib/ssl/src/ssl.app.src @@ -4,13 +4,17 @@ {modules, [ %% TLS/SSL tls_connection, + tls_connection_1_3, tls_handshake, + tls_handshake_1_3, tls_record, + tls_record_1_3, tls_socket, tls_v1, ssl_v3, tls_connection_sup, tls_sender, + ssl_dh_groups, %% DTLS dtls_connection, dtls_handshake, @@ -51,6 +55,8 @@ ssl_crl_cache, ssl_crl_cache_api, ssl_crl_hash_dir, + %% Logging + ssl_logger, %% App structure ssl_app, ssl_sup, diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 7c1d0a3829..e3bb4df1ac 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -51,7 +51,7 @@ %% SSL/TLS protocol handling -export([cipher_suites/0, cipher_suites/1, cipher_suites/2, filter_cipher_suites/2, prepend_cipher_suites/2, append_cipher_suites/2, - eccs/0, eccs/1, versions/0, + eccs/0, eccs/1, versions/0, groups/0, groups/1, format_error/1, renegotiate/1, prf/5, negotiated_protocol/1, connection_information/1, connection_information/2]). %% Misc @@ -74,6 +74,7 @@ ciphers/0, cipher/0, hash/0, + key/0, kex_algo/0, prf_random/0, cipher_filters/0, @@ -101,16 +102,21 @@ -type host() :: hostname() | ip_address(). % exported -type hostname() :: string(). -type ip_address() :: inet:ip_address(). --type session_id() :: binary(). --type protocol_version() :: tls_version() | dtls_version(). --type tls_version() :: tlsv1 | 'tlsv1.1' | 'tlsv1.2' | 'tlsv1.3' | legacy_version(). --type dtls_version() :: 'dtlsv1' | 'dtlsv1.2'. --type legacy_version() :: sslv3. +-type session_id() :: binary(). % exported +-type protocol_version() :: tls_version() | dtls_version(). % exported +-type tls_version() :: 'tlsv1.2' | 'tlsv1.3' | tls_legacy_version(). +-type dtls_version() :: 'dtlsv1.2' | dtls_legacy_version(). +-type tls_legacy_version() :: tlsv1 | 'tlsv1.1' | sslv3. +-type dtls_legacy_version() :: 'dtlsv1'. -type verify_type() :: verify_none | verify_peer. -type cipher() :: aes_128_cbc | aes_256_cbc | aes_128_gcm | aes_256_gcm | + aes_128_ccm | + aes_256_ccm | + aes_128_ccm_8 | + aes_256_ccm_8 | chacha20_poly1305 | legacy_cipher(). % exported -type legacy_cipher() :: rc4_128 | @@ -128,14 +134,29 @@ -type legacy_hash() :: md5. --type sign_algo() :: rsa | dsa | ecdsa. +-type sign_algo() :: rsa | dsa | ecdsa. % exported + +-type sign_scheme() :: rsa_pkcs1_sha256 + | rsa_pkcs1_sha384 + | rsa_pkcs1_sha512 + | ecdsa_secp256r1_sha256 + | ecdsa_secp384r1_sha384 + | ecdsa_secp521r1_sha512 + | rsa_pss_rsae_sha256 + | rsa_pss_rsae_sha384 + | rsa_pss_rsae_sha512 + | rsa_pss_pss_sha256 + | rsa_pss_pss_sha384 + | rsa_pss_pss_sha512 + | rsa_pkcs1_sha1 + | ecdsa_sha1. -type kex_algo() :: rsa | dhe_rsa | dhe_dss | ecdhe_ecdsa | ecdh_ecdsa | ecdh_rsa | srp_rsa| srp_dss | psk | dhe_psk | rsa_psk | - dh_anon | ecdh_anon | srp_anon. - + dh_anon | ecdh_anon | srp_anon | + any. %% TLS 1.3 , exported -type erl_cipher_suite() :: #{key_exchange := kex_algo(), cipher := cipher(), mac := hash() | aead, @@ -175,6 +196,9 @@ secp160r1 | secp160r2. +-type group() :: secp256r1 | secp384r1 | secp521r1 | ffdhe2048 | + ffdhe3072 | ffdhe4096 | ffdhe6144 | ffdhe8192. + -type srp_param_type() :: srp_1024 | srp_1536 | srp_2048 | @@ -224,6 +248,8 @@ {keyfile, key_pem()} | {password, key_password()} | {ciphers, cipher_suites()} | + {eccs, [named_curve()]} | + {signature_algs_cert, signature_schemes()} | {secure_renegotiate, secure_renegotiation()} | {depth, allowed_cert_chain_length()} | {verify_fun, custom_verify()} | @@ -233,6 +259,7 @@ {partial_chain, root_fun()} | {versions, protocol_versions()} | {user_lookup_fun, custom_user_lookup()} | + {log_level, logging_level()} | {log_alert, log_alert()} | {hibernate_after, hibernate_after()} | {padding_check, padding_check()} | @@ -259,20 +286,23 @@ -type algo_filter() :: fun((kex_algo()|cipher()|hash()|aead|default_prf) -> true | false). -type secure_renegotiation() :: boolean(). -type allowed_cert_chain_length() :: integer(). --type custom_verify() :: {Verifyfun :: fun(), InitialUserState :: term()}. --type crl_check() :: boolean() | peer | best_effort. --type crl_cache_opts() :: [term()]. --type handshake_size() :: integer(). --type hibernate_after() :: timeout(). --type root_fun() :: fun(). --type protocol_versions() :: [protocol_version()]. --type signature_algs() :: [{hash(), sign_algo()}]. --type custom_user_lookup() :: {Lookupfun :: fun(), UserState :: term()}. --type padding_check() :: boolean(). --type beast_mitigation() :: one_n_minus_one | zero_n | disabled. --type srp_identity() :: {Username :: string(), Password :: string()}. --type psk_identity() :: string(). --type log_alert() :: boolean(). + +-type custom_verify() :: {Verifyfun :: fun(), InitialUserState :: any()}. +-type crl_check() :: boolean() | peer | best_effort. +-type crl_cache_opts() :: [any()]. +-type handshake_size() :: integer(). +-type hibernate_after() :: timeout(). +-type root_fun() :: fun(). +-type protocol_versions() :: [protocol_version()]. +-type signature_algs() :: [{hash(), sign_algo()}]. +-type signature_schemes() :: [sign_scheme()]. +-type custom_user_lookup() :: {Lookupfun :: fun(), UserState :: any()}. +-type padding_check() :: boolean(). +-type beast_mitigation() :: one_n_minus_one | zero_n | disabled. +-type srp_identity() :: {Username :: string(), Password :: string()}. +-type psk_identity() :: string(). +-type log_alert() :: boolean(). +-type logging_level() :: logger:level(). %% ------------------------------------------------------------------------------------------------------- @@ -311,6 +341,7 @@ -type ssl_imp() :: new | old. %% ------------------------------------------------------------------------------------------------------- + -type server_option() :: {cacerts, server_cacerts()} | {cacertfile, server_cafile()} | {dh, dh_der()} | @@ -392,6 +423,7 @@ stop() -> %% %% Description: Connect to an ssl server. %%-------------------------------------------------------------------- + -spec connect(TCPSocket, TLSOptions) -> {ok, sslsocket()} | {error, reason()} | @@ -415,6 +447,7 @@ connect(Socket, SslOptions) when is_port(Socket) -> Host :: host(), Port :: inet:port_number(), TLSOptions :: [tls_client_option()]. + connect(Socket, SslOptions0, Timeout) when is_port(Socket), (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> CbInfo = handle_option(cb_info, SslOptions0, default_cb_info(tls)), @@ -431,6 +464,7 @@ connect(Socket, SslOptions0, Timeout) when is_port(Socket), connect(Host, Port, Options) -> connect(Host, Port, Options, infinity). + -spec connect(Host, Port, TLSOptions, Timeout) -> {ok, sslsocket()} | {ok, sslsocket(),Ext :: protocol_extensions()} | @@ -553,7 +587,6 @@ ssl_accept(Socket, SslOptions, Timeout) -> %%-------------------------------------------------------------------- %% Performs the SSL/TLS/DTLS server-side handshake. - -spec handshake(HsSocket) -> {ok, SslSocket} | {ok, SslSocket, Ext} | {error, Reason} when HsSocket :: sslsocket(), SslSocket :: sslsocket(), @@ -580,6 +613,12 @@ handshake(#sslsocket{} = Socket, Timeout) when (is_integer(Timeout) andalso Tim (Timeout == infinity) -> ssl_connection:handshake(Socket, Timeout); +%% If Socket is a ordinary socket(): upgrades a gen_tcp, or equivalent, socket to +%% an SSL socket, that is, performs the SSL/TLS server-side handshake and returns +%% the SSL socket. +%% +%% If Socket is an sslsocket(): provides extra SSL/TLS/DTLS options to those +%% specified in ssl:listen/2 and then performs the SSL/TLS/DTLS handshake. handshake(ListenSocket, SslOptions) when is_port(ListenSocket) -> handshake(ListenSocket, SslOptions, infinity). @@ -877,14 +916,14 @@ cipher_suites() -> %% Description: Returns all supported cipher suites. %%-------------------------------------------------------------------- cipher_suites(erlang) -> - [ssl_cipher_format:erl_suite_definition(Suite) || Suite <- available_suites(default)]; + [ssl_cipher_format:suite_legacy(Suite) || Suite <- available_suites(default)]; cipher_suites(openssl) -> - [ssl_cipher_format:openssl_suite_name(Suite) || + [ssl_cipher_format:suite_map_to_openssl_str(ssl_cipher_format:suite_bin_to_map(Suite)) || Suite <- available_suites(default)]; cipher_suites(all) -> - [ssl_cipher_format:erl_suite_definition(Suite) || Suite <- available_suites(all)]. + [ssl_cipher_format:suite_legacy(Suite) || Suite <- available_suites(all)]. %%-------------------------------------------------------------------- -spec cipher_suites(Supported, Version) -> ciphers() when @@ -903,7 +942,7 @@ cipher_suites(Base, Version) when Version == 'dtlsv1.2'; Version == 'dtlsv1'-> cipher_suites(Base, dtls_record:protocol_version(Version)); cipher_suites(Base, Version) -> - [ssl_cipher_format:suite_definition(Suite) || Suite <- supported_suites(Base, Version)]. + [ssl_cipher_format:suite_bin_to_map(Suite) || Suite <- supported_suites(Base, Version)]. %%-------------------------------------------------------------------- -spec filter_cipher_suites(Suites, Filters) -> Ciphers when @@ -997,6 +1036,20 @@ eccs_filter_supported(Curves) -> Curves). %%-------------------------------------------------------------------- +-spec groups() -> [group()]. +%% Description: returns all supported groups (TLS 1.3 and later) +%%-------------------------------------------------------------------- +groups() -> + tls_v1:groups(4). + +%%-------------------------------------------------------------------- +-spec groups(default) -> [group()]. +%% Description: returns the default groups (TLS 1.3 and later) +%%-------------------------------------------------------------------- +groups(default) -> + tls_v1:default_groups(4). + +%%-------------------------------------------------------------------- -spec getopts(SslSocket, OptionNames) -> {ok, [gen_tcp:option()]} | {error, reason()} when SslSocket :: sslsocket(), @@ -1006,7 +1059,7 @@ eccs_filter_supported(Curves) -> %%-------------------------------------------------------------------- getopts(#sslsocket{pid = [Pid|_]}, OptionTags) when is_pid(Pid), is_list(OptionTags) -> ssl_connection:get_opts(Pid, OptionTags); -getopts(#sslsocket{pid = {dtls, #config{transport_info = {Transport,_,_,_}}}} = ListenSocket, OptionTags) when is_list(OptionTags) -> +getopts(#sslsocket{pid = {dtls, #config{transport_info = {Transport,_,_,_,_}}}} = ListenSocket, OptionTags) when is_list(OptionTags) -> try dtls_socket:getopts(Transport, ListenSocket, OptionTags) of {ok, _} = Result -> Result; @@ -1065,7 +1118,7 @@ setopts(#sslsocket{pid = [Pid|_]}, Options0) when is_pid(Pid), is_list(Options0) _:_ -> {error, {options, {not_a_proplist, Options0}}} end; -setopts(#sslsocket{pid = {dtls, #config{transport_info = {Transport,_,_,_}}}} = ListenSocket, Options) when is_list(Options) -> +setopts(#sslsocket{pid = {dtls, #config{transport_info = {Transport,_,_,_,_}}}} = ListenSocket, Options) when is_list(Options) -> try dtls_socket:setopts(Transport, ListenSocket, Options) of ok -> ok; @@ -1108,7 +1161,7 @@ getstat(Socket) -> %% %% Description: Get one or more statistic options for a socket. %%-------------------------------------------------------------------- -getstat(#sslsocket{pid = {Listen, #config{transport_info = {Transport, _, _, _}}}}, Options) when is_port(Listen), is_list(Options) -> +getstat(#sslsocket{pid = {Listen, #config{transport_info = {Transport, _, _, _, _}}}}, Options) when is_port(Listen), is_list(Options) -> tls_socket:getstat(Transport, Listen, Options); getstat(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _, _}}, Options) when is_pid(Pid), is_list(Options) -> @@ -1257,7 +1310,6 @@ tls_version({3, _} = Version) -> tls_version({254, _} = Version) -> dtls_v1:corresponding_tls_version(Version). - %%-------------------------------------------------------------------- -spec suite_to_str(CipherSuite) -> string() when CipherSuite :: erl_cipher_suite(); @@ -1271,8 +1323,7 @@ tls_version({254, _} = Version) -> %% Description: Return the string representation of a cipher suite. %%-------------------------------------------------------------------- suite_to_str(Cipher) -> - ssl_cipher_format:suite_to_str(Cipher). - + ssl_cipher_format:suite_map_to_str(Cipher). %%%-------------------------------------------------------------- %%% Internal functions @@ -1333,9 +1384,10 @@ handle_options(Opts0, #ssl_options{protocol = Protocol, cacerts = CaCerts0, [] -> new_ssl_options(SslOpts1, NewVerifyOpts, RecordCB); Value -> - Versions = [RecordCB:protocol_version(Vsn) || Vsn <- Value], + Versions0 = [RecordCB:protocol_version(Vsn) || Vsn <- Value], + Versions1 = lists:sort(fun RecordCB:is_higher/2, Versions0), new_ssl_options(proplists:delete(versions, SslOpts1), - NewVerifyOpts#ssl_options{versions = Versions}, record_cb(Protocol)) + NewVerifyOpts#ssl_options{versions = Versions1}, record_cb(Protocol)) end; %% Handle all options in listen and connect @@ -1351,12 +1403,14 @@ handle_options(Opts0, Role, Host) -> CertFile = handle_option(certfile, Opts, <<>>), - Versions = case handle_option(versions, Opts, []) of - [] -> - RecordCb:supported_protocol_versions(); - Vsns -> - [RecordCb:protocol_version(Vsn) || Vsn <- Vsns] - end, + [HighestVersion|_] = Versions = + case handle_option(versions, Opts, []) of + [] -> + RecordCb:supported_protocol_versions(); + Vsns -> + Versions0 = [RecordCb:protocol_version(Vsn) || Vsn <- Vsns], + lists:sort(fun RecordCb:is_higher/2, Versions0) + end, Protocol = handle_option(protocol, Opts, tls), @@ -1367,7 +1421,7 @@ handle_options(Opts0, Role, Host) -> ok end, - SSLOptions = #ssl_options{ + SSLOptions0 = #ssl_options{ versions = Versions, verify = validate_option(verify, Verify), verify_fun = VerifyFun, @@ -1388,13 +1442,29 @@ handle_options(Opts0, Role, Host) -> psk_identity = handle_option(psk_identity, Opts, undefined), srp_identity = handle_option(srp_identity, Opts, undefined), ciphers = handle_cipher_option(proplists:get_value(ciphers, Opts, []), - RecordCb:highest_protocol_version(Versions)), + HighestVersion), eccs = handle_eccs_option(proplists:get_value(eccs, Opts, eccs()), - RecordCb:highest_protocol_version(Versions)), - signature_algs = handle_hashsigns_option(proplists:get_value(signature_algs, Opts, - default_option_role(server, - tls_v1:default_signature_algs(Versions), Role)), - tls_version(RecordCb:highest_protocol_version(Versions))), + HighestVersion), + supported_groups = handle_supported_groups_option( + proplists:get_value(supported_groups, Opts, groups(default)), + HighestVersion), + signature_algs = + handle_hashsigns_option( + proplists:get_value( + signature_algs, + Opts, + default_option_role_sign_algs(server, + tls_v1:default_signature_algs(HighestVersion), + Role, + HighestVersion)), + tls_version(HighestVersion)), + signature_algs_cert = + handle_signature_algorithms_option( + proplists:get_value( + signature_algs_cert, + Opts, + undefined), %% Do not send by default + tls_version(HighestVersion)), reuse_sessions = handle_reuse_sessions_option(reuse_sessions, Opts, Role), reuse_session = handle_reuse_session_option(reuse_session, Opts, Role), secure_renegotiate = handle_option(secure_renegotiate, Opts, true), @@ -1413,7 +1483,6 @@ handle_options(Opts0, Role, Host) -> next_protocol_selector = make_next_protocol_selector( handle_option(client_preferred_next_protocols, Opts, undefined)), - log_alert = handle_option(log_alert, Opts, true), server_name_indication = handle_option(server_name_indication, Opts, default_option_role(client, server_name_indication_default(Host), Role)), @@ -1439,6 +1508,10 @@ handle_options(Opts0, Role, Host) -> handshake = handle_option(handshake, Opts, full), customize_hostname_check = handle_option(customize_hostname_check, Opts, []) }, + LogLevel = handle_option(log_alert, Opts, true), + SSLOptions = SSLOptions0#ssl_options{ + log_level = handle_option(log_level, Opts, LogLevel) + }, CbInfo = handle_option(cb_info, Opts, default_cb_info(Protocol)), SslOptions = [protocol, versions, verify, verify_fun, partial_chain, @@ -1450,10 +1523,12 @@ handle_options(Opts0, Role, Host) -> cb_info, renegotiate_at, secure_renegotiate, hibernate_after, erl_dist, alpn_advertised_protocols, sni_hosts, sni_fun, alpn_preferred_protocols, next_protocols_advertised, - client_preferred_next_protocols, log_alert, + client_preferred_next_protocols, log_alert, log_level, server_name_indication, honor_cipher_order, padding_check, crl_check, crl_cache, - fallback, signature_algs, eccs, honor_ecc_order, - beast_mitigation, max_handshake_size, handshake, customize_hostname_check], + fallback, signature_algs, signature_algs_cert, eccs, honor_ecc_order, + beast_mitigation, max_handshake_size, handshake, customize_hostname_check, + supported_groups], + SockOpts = lists:foldl(fun(Key, PropList) -> proplists:delete(Key, PropList) end, Opts, SslOptions), @@ -1638,7 +1713,20 @@ validate_option(client_preferred_next_protocols, {Precedence, PreferredProtocols Value; validate_option(client_preferred_next_protocols, undefined) -> undefined; -validate_option(log_alert, Value) when is_boolean(Value) -> +validate_option(log_alert, true) -> + notice; +validate_option(log_alert, false) -> + warning; +validate_option(log_level, Value) when + is_atom(Value) andalso + (Value =:= emergency orelse + Value =:= alert orelse + Value =:= critical orelse + Value =:= error orelse + Value =:= warning orelse + Value =:= notice orelse + Value =:= info orelse + Value =:= debug) -> Value; validate_option(next_protocols_advertised, Value) when is_list(Value) -> validate_binary_list(next_protocols_advertised, Value), @@ -1730,19 +1818,42 @@ handle_cb_info({V1, V2, V3, V4}, {_,_,_,_,_}) -> handle_cb_info(CbInfo, _) -> CbInfo. +handle_hashsigns_option(Value, Version) when is_list(Value) + andalso Version >= {3, 4} -> + case tls_v1:signature_schemes(Version, Value) of + [] -> + throw({error, {options, + no_supported_signature_schemes, + {signature_algs, Value}}}); + _ -> + Value + end; handle_hashsigns_option(Value, Version) when is_list(Value) - andalso Version >= {3, 3} -> + andalso Version =:= {3, 3} -> case tls_v1:signature_algs(Version, Value) of [] -> throw({error, {options, no_supported_algorithms, {signature_algs, Value}}}); _ -> Value end; -handle_hashsigns_option(_, Version) when Version >= {3, 3} -> +handle_hashsigns_option(_, Version) when Version =:= {3, 3} -> handle_hashsigns_option(tls_v1:default_signature_algs(Version), Version); handle_hashsigns_option(_, _Version) -> undefined. +handle_signature_algorithms_option(Value, Version) when is_list(Value) + andalso Version >= {3, 4} -> + case tls_v1:signature_schemes(Version, Value) of + [] -> + throw({error, {options, + no_supported_signature_schemes, + {signature_algs_cert, Value}}}); + _ -> + Value + end; +handle_signature_algorithms_option(_, _Version) -> + undefined. + handle_reuse_sessions_option(Key, Opts, client) -> Value = proplists:get_value(Key, Opts, true), validate_option(Key, Value), @@ -1786,7 +1897,8 @@ validate_binary_list(Opt, List) -> end, List). validate_versions([], Versions) -> Versions; -validate_versions([Version | Rest], Versions) when Version == 'tlsv1.2'; +validate_versions([Version | Rest], Versions) when Version == 'tlsv1.3'; + Version == 'tlsv1.2'; Version == 'tlsv1.1'; Version == tlsv1; Version == sslv3 -> @@ -1799,10 +1911,11 @@ validate_versions([Ver| _], Versions) -> tls_validate_versions([], Versions) -> Versions; -tls_validate_versions([Version | Rest], Versions) when Version == 'tlsv1.2'; - Version == 'tlsv1.1'; - Version == tlsv1; - Version == sslv3 -> +tls_validate_versions([Version | Rest], Versions) when Version == 'tlsv1.3'; + Version == 'tlsv1.2'; + Version == 'tlsv1.1'; + Version == tlsv1; + Version == sslv3 -> tls_validate_versions(Rest, Versions); tls_validate_versions([Ver| _], Versions) -> throw({error, {options, {Ver, {versions, Versions}}}}). @@ -1850,10 +1963,10 @@ binary_cipher_suites(Version, []) -> %% not require explicit configuration default_binary_suites(Version); binary_cipher_suites(Version, [Map|_] = Ciphers0) when is_map(Map) -> - Ciphers = [ssl_cipher_format:suite(C) || C <- Ciphers0], + Ciphers = [ssl_cipher_format:suite_map_to_bin(C) || C <- Ciphers0], binary_cipher_suites(Version, Ciphers); binary_cipher_suites(Version, [Tuple|_] = Ciphers0) when is_tuple(Tuple) -> - Ciphers = [ssl_cipher_format:suite(tuple_to_map(C)) || C <- Ciphers0], + Ciphers = [ssl_cipher_format:suite_map_to_bin(tuple_to_map(C)) || C <- Ciphers0], binary_cipher_suites(Version, Ciphers); binary_cipher_suites(Version, [Cipher0 | _] = Ciphers0) when is_binary(Cipher0) -> All = ssl_cipher:all_suites(Version) ++ @@ -1868,11 +1981,11 @@ binary_cipher_suites(Version, [Cipher0 | _] = Ciphers0) when is_binary(Cipher0) end; binary_cipher_suites(Version, [Head | _] = Ciphers0) when is_list(Head) -> %% Format: ["RC4-SHA","RC4-MD5"] - Ciphers = [ssl_cipher_format:openssl_suite(C) || C <- Ciphers0], + Ciphers = [ssl_cipher_format:suite_openssl_str_to_map(C) || C <- Ciphers0], binary_cipher_suites(Version, Ciphers); binary_cipher_suites(Version, Ciphers0) -> %% Format: "RC4-SHA:RC4-MD5" - Ciphers = [ssl_cipher_format:openssl_suite(C) || C <- string:lexemes(Ciphers0, ":")], + Ciphers = [ssl_cipher_format:suite_openssl_str_to_map(C) || C <- string:lexemes(Ciphers0, ":")], binary_cipher_suites(Version, Ciphers). default_binary_suites(Version) -> @@ -1908,6 +2021,16 @@ handle_eccs_option(Value, Version) when is_list(Value) -> error:_ -> throw({error, {options, {eccs, Value}}}) end. +handle_supported_groups_option(Value, Version) when is_list(Value) -> + {_Major, Minor} = tls_version(Version), + try tls_v1:groups(Minor, Value) of + Groups -> #supported_groups{supported_groups = Groups} + catch + exit:_ -> throw({error, {options, {supported_groups, Value}}}); + error:_ -> throw({error, {options, {supported_groups, Value}}}) + end. + + unexpected_format(Error) -> lists:flatten(io_lib:format("Unexpected error: ~p", [Error])). @@ -2053,8 +2176,10 @@ new_ssl_options([{next_protocols_advertised, Value} | Rest], #ssl_options{} = Op new_ssl_options([{client_preferred_next_protocols, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{next_protocol_selector = make_next_protocol_selector(validate_option(client_preferred_next_protocols, Value))}, RecordCB); -new_ssl_options([{log_alert, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> - new_ssl_options(Rest, Opts#ssl_options{log_alert = validate_option(log_alert, Value)}, RecordCB); +new_ssl_options([{log_alert, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> + new_ssl_options(Rest, Opts#ssl_options{log_level = validate_option(log_alert, Value)}, RecordCB); +new_ssl_options([{log_level, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> + new_ssl_options(Rest, Opts#ssl_options{log_level = validate_option(log_level, Value)}, RecordCB); new_ssl_options([{server_name_indication, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{server_name_indication = validate_option(server_name_indication, Value)}, RecordCB); new_ssl_options([{honor_cipher_order, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> @@ -2067,12 +2192,26 @@ new_ssl_options([{eccs, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> handle_eccs_option(Value, RecordCB:highest_protocol_version()) }, RecordCB); +new_ssl_options([{supported_groups, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> + new_ssl_options(Rest, + Opts#ssl_options{supported_groups = + handle_supported_groups_option(Value, RecordCB:highest_protocol_version()) + }, + RecordCB); new_ssl_options([{signature_algs, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{signature_algs = handle_hashsigns_option(Value, tls_version(RecordCB:highest_protocol_version()))}, RecordCB); +new_ssl_options([{signature_algs_cert, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> + new_ssl_options( + Rest, + Opts#ssl_options{signature_algs_cert = + handle_signature_algorithms_option( + Value, + tls_version(RecordCB:highest_protocol_version()))}, + RecordCB); new_ssl_options([{protocol, dtls = Value} | Rest], #ssl_options{} = Opts, dtls_record = RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{protocol = Value}, RecordCB); new_ssl_options([{protocol, tls = Value} | Rest], #ssl_options{} = Opts, tls_record = RecordCB) -> @@ -2134,15 +2273,24 @@ handle_verify_options(Opts, CaCerts) -> throw({error, {options, {verify, Value}}}) end. +%% Added to handle default values for signature_algs in TLS 1.3 +default_option_role_sign_algs(_, Value, _, Version) when Version >= {3,4} -> + Value; +default_option_role_sign_algs(Role, Value, Role, _) -> + Value; +default_option_role_sign_algs(_, _, _, _) -> + undefined. + default_option_role(Role, Value, Role) -> Value; default_option_role(_,_,_) -> undefined. + default_cb_info(tls) -> {gen_tcp, tcp, tcp_closed, tcp_error, tcp_passive}; default_cb_info(dtls) -> - {gen_udp, udp, udp_closed, udp_error}. + {gen_udp, udp, udp_closed, udp_error, udp_passive}. include_security_info([]) -> false; diff --git a/lib/ssl/src/ssl_alert.erl b/lib/ssl/src/ssl_alert.erl index 2a20d13cd5..06b1b005a5 100644 --- a/lib/ssl/src/ssl_alert.erl +++ b/lib/ssl/src/ssl_alert.erl @@ -161,10 +161,14 @@ description_txt(?INSUFFICIENT_SECURITY) -> "Insufficient Security"; description_txt(?INTERNAL_ERROR) -> "Internal Error"; +description_txt(?INAPPROPRIATE_FALLBACK) -> + "Inappropriate Fallback"; description_txt(?USER_CANCELED) -> "User Canceled"; description_txt(?NO_RENEGOTIATION) -> "No Renegotiation"; +description_txt(?MISSING_EXTENSION) -> + "Missing extension"; description_txt(?UNSUPPORTED_EXTENSION) -> "Unsupported Extension"; description_txt(?CERTIFICATE_UNOBTAINABLE) -> @@ -177,8 +181,8 @@ description_txt(?BAD_CERTIFICATE_HASH_VALUE) -> "Bad Certificate Hash Value"; description_txt(?UNKNOWN_PSK_IDENTITY) -> "Unknown Psk Identity"; -description_txt(?INAPPROPRIATE_FALLBACK) -> - "Inappropriate Fallback"; +description_txt(?CERTIFICATE_REQUIRED) -> + "Certificate required"; description_txt(?NO_APPLICATION_PROTOCOL) -> "No application protocol"; description_txt(Enum) -> @@ -228,10 +232,14 @@ description_atom(?INSUFFICIENT_SECURITY) -> insufficient_security; description_atom(?INTERNAL_ERROR) -> internal_error; +description_atom(?INAPPROPRIATE_FALLBACK) -> + inappropriate_fallback; description_atom(?USER_CANCELED) -> user_canceled; description_atom(?NO_RENEGOTIATION) -> no_renegotiation; +description_atom(?MISSING_EXTENSION) -> + missing_extension; description_atom(?UNSUPPORTED_EXTENSION) -> unsupported_extension; description_atom(?CERTIFICATE_UNOBTAINABLE) -> @@ -244,9 +252,9 @@ description_atom(?BAD_CERTIFICATE_HASH_VALUE) -> bad_certificate_hash_value; description_atom(?UNKNOWN_PSK_IDENTITY) -> unknown_psk_identity; -description_atom(?INAPPROPRIATE_FALLBACK) -> - inappropriate_fallback; +description_atom(?CERTIFICATE_REQUIRED) -> + certificate_required; description_atom(?NO_APPLICATION_PROTOCOL) -> no_application_protocol; description_atom(_) -> - 'unsupported/unkonwn_alert'. + 'unsupported/unknown_alert'. diff --git a/lib/ssl/src/ssl_alert.hrl b/lib/ssl/src/ssl_alert.hrl index b23123905e..9b2322da17 100644 --- a/lib/ssl/src/ssl_alert.hrl +++ b/lib/ssl/src/ssl_alert.hrl @@ -29,6 +29,9 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Alert protocol - RFC 2246 section 7.2 +%%% updated by RFC 8486 with +%%% missing_extension(109), +%%% certificate_required(116), %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% AlertLevel @@ -100,12 +103,14 @@ -define(INAPPROPRIATE_FALLBACK, 86). -define(USER_CANCELED, 90). -define(NO_RENEGOTIATION, 100). +-define(MISSING_EXTENSION, 109). -define(UNSUPPORTED_EXTENSION, 110). -define(CERTIFICATE_UNOBTAINABLE, 111). -define(UNRECOGNISED_NAME, 112). -define(BAD_CERTIFICATE_STATUS_RESPONSE, 113). -define(BAD_CERTIFICATE_HASH_VALUE, 114). -define(UNKNOWN_PSK_IDENTITY, 115). +-define(CERTIFICATE_REQUIRED, 116). -define(NO_APPLICATION_PROTOCOL, 120). -define(ALERT_REC(Level,Desc), #alert{level=Level,description=Desc,where={?FILE, ?LINE}}). diff --git a/lib/ssl/src/ssl_app.erl b/lib/ssl/src/ssl_app.erl index 62e8765d4a..9e6d676bef 100644 --- a/lib/ssl/src/ssl_app.erl +++ b/lib/ssl/src/ssl_app.erl @@ -29,9 +29,26 @@ -export([start/2, stop/1]). start(_Type, _StartArgs) -> + start_logger(), ssl_sup:start_link(). stop(_State) -> + stop_logger(), ok. +%% +%% Description: Start SSL logger +start_logger() -> + Config = #{level => debug, + filter_default => stop, + formatter => {ssl_logger, #{}}}, + Filter = {fun logger_filters:domain/2,{log,sub,[otp,ssl]}}, + logger:add_handler(ssl_handler, logger_std_h, Config), + logger:add_handler_filter(ssl_handler, filter_non_ssl, Filter), + logger:set_application_level(ssl, debug). +%% +%% Description: Stop SSL logger +stop_logger() -> + logger:unset_application_level(ssl), + logger:remove_handler(ssl_handler). diff --git a/lib/ssl/src/ssl_certificate.erl b/lib/ssl/src/ssl_certificate.erl index 549e557beb..9997f5e0c8 100644 --- a/lib/ssl/src/ssl_certificate.erl +++ b/lib/ssl/src/ssl_certificate.erl @@ -71,7 +71,7 @@ trusted_cert_and_path(CertChain, CertDbHandle, CertDbRef, PartialChainHandler) - case SignedAndIssuerID of {error, issuer_not_found} -> - %% The root CA was not sent and can not be found. + %% The root CA was not sent and cannot be found. handle_incomplete_chain(Path, PartialChainHandler); {self, _} when length(Path) == 1 -> {selfsigned_peer, Path}; diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index fce48d1678..21db887bb5 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -31,10 +31,11 @@ -include("ssl_cipher.hrl"). -include("ssl_handshake.hrl"). -include("ssl_alert.hrl"). +-include("tls_handshake_1_3.hrl"). -include_lib("public_key/include/public_key.hrl"). --export([security_parameters/2, security_parameters/3, - cipher_init/3, nonce_seed/2, decipher/6, cipher/5, aead_encrypt/5, aead_decrypt/6, +-export([security_parameters/2, security_parameters/3, security_parameters_1_3/2, + cipher_init/3, nonce_seed/2, decipher/6, cipher/5, aead_encrypt/6, aead_decrypt/6, suites/1, all_suites/1, crypto_support_filters/0, chacha_suites/1, anonymous_suites/1, psk_suites/1, psk_suites_anon/1, srp_suites/0, srp_suites_anon/0, @@ -42,7 +43,12 @@ filter/3, filter_suites/1, filter_suites/2, hash_algorithm/1, sign_algorithm/1, is_acceptable_hash/2, is_fallback/1, random_bytes/1, calc_mac_hash/4, calc_mac_hash/6, - is_stream_ciphersuite/1]). + is_stream_ciphersuite/1, signature_scheme/1, + scheme_to_components/1, hash_size/1, effective_key_bits/1, + key_material/1, signature_algorithm_to_scheme/1]). + +%% RFC 8446 TLS 1.3 +-export([generate_client_shares/1, generate_server_share/1, add_zero_padding/2]). -compile(inline). @@ -70,7 +76,7 @@ security_parameters(?TLS_NULL_WITH_NULL_NULL = CipherSuite, SecParams) -> %%------------------------------------------------------------------- security_parameters(Version, CipherSuite, SecParams) -> #{cipher := Cipher, mac := Hash, - prf := PrfHashAlg} = ssl_cipher_format:suite_definition(CipherSuite), + prf := PrfHashAlg} = ssl_cipher_format:suite_bin_to_map(CipherSuite), SecParams#security_parameters{ cipher_suite = CipherSuite, bulk_cipher_algorithm = bulk_cipher_algorithm(Cipher), @@ -83,6 +89,15 @@ security_parameters(Version, CipherSuite, SecParams) -> prf_algorithm = prf_algorithm(PrfHashAlg, Version), hash_size = hash_size(Hash)}. +security_parameters_1_3(SecParams, CipherSuite) -> + #{cipher := Cipher, prf := PrfHashAlg} = + ssl_cipher_format:suite_bin_to_map(CipherSuite), + SecParams#security_parameters{ + cipher_suite = CipherSuite, + bulk_cipher_algorithm = bulk_cipher_algorithm(Cipher), + prf_algorithm = PrfHashAlg, %% HKDF hash algorithm + cipher_type = ?AEAD}. + %%-------------------------------------------------------------------- -spec cipher_init(cipher_enum(), binary(), binary()) -> #cipher_state{}. %% @@ -91,9 +106,13 @@ security_parameters(Version, CipherSuite, SecParams) -> cipher_init(?RC4, IV, Key) -> State = crypto:stream_init(rc4, Key), #cipher_state{iv = IV, key = Key, state = State}; -cipher_init(?AES_GCM, IV, Key) -> +cipher_init(Type, IV, Key) when Type == ?AES_GCM; + Type == ?AES_CCM -> <<Nonce:64>> = random_bytes(8), #cipher_state{iv = IV, key = Key, nonce = Nonce, tag_len = 16}; +cipher_init(?AES_CCM_8, IV, Key) -> + <<Nonce:64>> = random_bytes(8), + #cipher_state{iv = IV, key = Key, nonce = Nonce, tag_len = 8}; cipher_init(?CHACHA20_POLY1305, IV, Key) -> #cipher_state{iv = IV, key = Key, tag_len = 16}; cipher_init(_BCA, IV, Key) -> @@ -133,14 +152,18 @@ cipher(?AES_CBC, CipherState, Mac, Fragment, Version) -> crypto:block_encrypt(aes_cbc256, Key, IV, T) end, block_size(aes_128_cbc), CipherState, Mac, Fragment, Version). -aead_encrypt(Type, Key, Nonce, Fragment, AdditionalData) -> - crypto:block_encrypt(aead_type(Type), Key, Nonce, {AdditionalData, Fragment}). +aead_encrypt(Type, Key, Nonce, Fragment, AdditionalData, TagLen) -> + crypto:block_encrypt(aead_type(Type), Key, Nonce, {AdditionalData, Fragment, TagLen}). aead_decrypt(Type, Key, Nonce, CipherText, CipherTag, AdditionalData) -> crypto:block_decrypt(aead_type(Type), Key, Nonce, {AdditionalData, CipherText, CipherTag}). aead_type(?AES_GCM) -> aes_gcm; +aead_type(?AES_CCM) -> + aes_ccm; +aead_type(?AES_CCM_8) -> + aes_ccm; aead_type(?CHACHA20_POLY1305) -> chacha20_poly1305. @@ -158,7 +181,7 @@ block_cipher(Fun, BlockSz, #cipher_state{key=Key, iv=IV} = CS0, block_cipher(Fun, BlockSz, #cipher_state{key=Key, iv=IV, state = IV_Cache0} = CS0, Mac, Fragment, {3, N}) - when N == 2; N == 3 -> + when N == 2; N == 3; N == 4 -> IV_Size = byte_size(IV), <<NextIV:IV_Size/binary, IV_Cache/binary>> = case IV_Cache0 of @@ -294,8 +317,9 @@ anonymous_suites({3, N}) -> srp_suites_anon() ++ anonymous_suites(N); anonymous_suites({254, _} = Version) -> dtls_v1:anonymous_suites(Version); -anonymous_suites(N) - when N >= 3 -> +anonymous_suites(4) -> + []; %% Raw public key negotiation may be used instead +anonymous_suites( 3 = N) -> psk_suites_anon(N) ++ [?TLS_DH_anon_WITH_AES_128_GCM_SHA256, ?TLS_DH_anon_WITH_AES_256_GCM_SHA384, @@ -328,8 +352,9 @@ anonymous_suites(N) when N == 0; %%-------------------------------------------------------------------- psk_suites({3, N}) -> psk_suites(N); -psk_suites(N) - when N >= 3 -> +psk_suites(4) -> + []; %% TODO Add new PSK, PSK_(EC)DHE suites +psk_suites(3) -> [ ?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384, ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384, @@ -350,20 +375,32 @@ psk_suites(_) -> %%-------------------------------------------------------------------- psk_suites_anon({3, N}) -> psk_suites_anon(N); -psk_suites_anon(N) - when N >= 3 -> +psk_suites_anon(3) -> [ ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384, ?TLS_PSK_WITH_AES_256_GCM_SHA384, ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384, ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384, ?TLS_PSK_WITH_AES_256_CBC_SHA384, + ?TLS_DHE_PSK_WITH_AES_256_CCM, + ?TLS_PSK_DHE_WITH_AES_256_CCM_8, + ?TLS_PSK_WITH_AES_256_CCM, + ?TLS_PSK_WITH_AES_256_CCM_8, ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256, ?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256, ?TLS_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256, ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256, ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256, - ?TLS_PSK_WITH_AES_128_CBC_SHA256 + ?TLS_PSK_WITH_AES_128_CBC_SHA256, + ?TLS_DHE_PSK_WITH_AES_128_CCM, + ?TLS_PSK_DHE_WITH_AES_128_CCM_8, + ?TLS_PSK_WITH_AES_128_CCM, + ?TLS_PSK_WITH_AES_128_CCM_8, + ?TLS_ECDHE_PSK_WITH_RC4_128_SHA ] ++ psk_suites_anon(0); psk_suites_anon(_) -> [?TLS_DHE_PSK_WITH_AES_256_CBC_SHA, @@ -418,11 +455,12 @@ rc4_suites({3, Minor}) -> rc4_suites(0) -> [?TLS_RSA_WITH_RC4_128_SHA, ?TLS_RSA_WITH_RC4_128_MD5]; -rc4_suites(N) when N =< 3 -> +rc4_suites(N) when N =< 4 -> [?TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, ?TLS_ECDHE_RSA_WITH_RC4_128_SHA, ?TLS_ECDH_ECDSA_WITH_RC4_128_SHA, ?TLS_ECDH_RSA_WITH_RC4_128_SHA]. + %%-------------------------------------------------------------------- -spec des_suites(Version::ssl_record:ssl_version()) -> [ssl_cipher_format:cipher_suite()]. %% @@ -457,7 +495,7 @@ rsa_suites(0) -> ?TLS_RSA_WITH_AES_128_CBC_SHA, ?TLS_RSA_WITH_3DES_EDE_CBC_SHA ]; -rsa_suites(N) when N =< 3 -> +rsa_suites(N) when N =< 4 -> [ ?TLS_RSA_WITH_AES_256_GCM_SHA384, ?TLS_RSA_WITH_AES_256_CBC_SHA256, @@ -511,7 +549,7 @@ filter_suite(#{key_exchange := KeyExchange, all_filters(Hash, HashFilters) andalso all_filters(Prf, PrfFilters); filter_suite(Suite, Filters) -> - filter_suite(ssl_cipher_format:suite_definition(Suite), Filters). + filter_suite(ssl_cipher_format:suite_bin_to_map(Suite), Filters). %%-------------------------------------------------------------------- -spec filter_suites([ssl:erl_cipher_suite()] | [ssl_cipher_format:cipher_suite()]) -> @@ -556,7 +594,8 @@ crypto_support_filters() -> end]}. is_acceptable_keyexchange(KeyExchange, _Algos) when KeyExchange == psk; - KeyExchange == null -> + KeyExchange == null; + KeyExchange == any -> true; is_acceptable_keyexchange(KeyExchange, Algos) when KeyExchange == dh_anon; KeyExchange == dhe_psk -> @@ -568,7 +607,7 @@ is_acceptable_keyexchange(dhe_rsa, Algos) -> proplists:get_bool(dh, Algos) andalso proplists:get_bool(rsa, Algos); is_acceptable_keyexchange(KeyExchange, Algos) when KeyExchange == ecdh_anon; - KeyExchange == ecdhe_psk -> + KeyExchange == ecdhe_psk -> proplists:get_bool(ecdh, Algos); is_acceptable_keyexchange(KeyExchange, Algos) when KeyExchange == ecdh_ecdsa; KeyExchange == ecdhe_ecdsa -> @@ -608,6 +647,12 @@ is_acceptable_cipher(Cipher, Algos) when Cipher == aes_128_gcm; Cipher == aes_256_gcm -> proplists:get_bool(aes_gcm, Algos); +is_acceptable_cipher(Cipher, Algos) + when Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8 -> + proplists:get_bool(aes_ccm, Algos); is_acceptable_cipher(Cipher, Algos) -> proplists:get_bool(Cipher, Algos). @@ -651,6 +696,28 @@ is_stream_ciphersuite(#{cipher := rc4_128}) -> true; is_stream_ciphersuite(_) -> false. + +-spec hash_size(atom()) -> integer(). +hash_size(null) -> + 0; +%% The AEAD MAC hash size is not used in the context +%% of calculating the master secret. See RFC 5246 Section 6.2.3.3. +hash_size(aead) -> + 0; +hash_size(md5) -> + 16; +hash_size(sha) -> + 20; +%% Uncomment when adding cipher suite that needs it +%hash_size(sha224) -> +% 28; +hash_size(sha256) -> + 32; +hash_size(sha384) -> + 48; +hash_size(sha512) -> + 64. + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- @@ -660,7 +727,7 @@ mac_hash({_,_}, ?NULL, _MacSecret, _SeqNo, _Type, mac_hash({3, 0}, MacAlg, MacSecret, SeqNo, Type, Length, Fragment) -> ssl_v3:mac_hash(MacAlg, MacSecret, SeqNo, Type, Length, Fragment); mac_hash({3, N} = Version, MacAlg, MacSecret, SeqNo, Type, Length, Fragment) - when N =:= 1; N =:= 2; N =:= 3 -> + when N =:= 1; N =:= 2; N =:= 3; N =:= 4 -> tls_v1:mac_hash(MacAlg, MacSecret, SeqNo, Type, Version, Length, Fragment). @@ -678,6 +745,12 @@ bulk_cipher_algorithm(Cipher) when Cipher == aes_128_cbc; bulk_cipher_algorithm(Cipher) when Cipher == aes_128_gcm; Cipher == aes_256_gcm -> ?AES_GCM; +bulk_cipher_algorithm(Cipher) when Cipher == aes_128_ccm; + Cipher == aes_256_ccm -> + ?AES_CCM; +bulk_cipher_algorithm(Cipher) when Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8 -> + ?AES_CCM_8; bulk_cipher_algorithm(chacha20_poly1305) -> ?CHACHA20_POLY1305. @@ -692,6 +765,10 @@ type(Cipher) when Cipher == des_cbc; ?BLOCK; type(Cipher) when Cipher == aes_128_gcm; Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> ?AEAD. @@ -709,8 +786,16 @@ key_material(aes_256_cbc) -> 32; key_material(aes_128_gcm) -> 16; +key_material(aes_128_ccm) -> + 16; +key_material(aes_128_ccm_8) -> + 16; key_material(aes_256_gcm) -> 32; +key_material(aes_256_ccm_8) -> + 32; +key_material(aes_256_ccm) -> + 32; key_material(chacha20_poly1305) -> 32. @@ -726,6 +811,10 @@ expanded_key_material(Cipher) when Cipher == aes_128_cbc; Cipher == aes_256_cbc; Cipher == aes_128_gcm; Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> unknown. @@ -735,22 +824,31 @@ effective_key_bits(des_cbc) -> 56; effective_key_bits(Cipher) when Cipher == rc4_128; Cipher == aes_128_cbc; - Cipher == aes_128_gcm -> + Cipher == aes_128_gcm; + Cipher == aes_128_ccm; + Cipher == aes_128_ccm_8 -> 128; effective_key_bits('3des_ede_cbc') -> 168; effective_key_bits(Cipher) when Cipher == aes_256_cbc; Cipher == aes_256_gcm; + Cipher == aes_256_ccm; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> 256. iv_size(Cipher) when Cipher == null; - Cipher == rc4_128; - Cipher == chacha20_poly1305-> + Cipher == rc4_128 -> 0; iv_size(Cipher) when Cipher == aes_128_gcm; - Cipher == aes_256_gcm -> + Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8 -> 4; +iv_size(chacha20_poly1305) -> + 12; iv_size(Cipher) -> block_size(Cipher). @@ -761,6 +859,10 @@ block_size(Cipher) when Cipher == aes_128_cbc; Cipher == aes_256_cbc; Cipher == aes_128_gcm; Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> 16. @@ -804,26 +906,75 @@ sign_algorithm(?ECDSA) -> ecdsa; sign_algorithm(Other) when is_integer(Other) andalso ((Other >= 4) and (Other =< 223)) -> unassigned; sign_algorithm(Other) when is_integer(Other) andalso ((Other >= 224) and (Other =< 255)) -> Other. -hash_size(null) -> - 0; -%% The AEAD MAC hash size is not used in the context -%% of calculating the master secret. See RFC 5246 Section 6.2.3.3. -hash_size(aead) -> - 0; -hash_size(md5) -> - 16; -hash_size(sha) -> - 20; -%% Uncomment when adding cipher suite that needs it -%hash_size(sha224) -> -% 28; -hash_size(sha256) -> - 32; -hash_size(sha384) -> - 48. -%% Uncomment when adding cipher suite that needs it -%hash_size(sha512) -> -% 64. + +signature_scheme(rsa_pkcs1_sha256) -> ?RSA_PKCS1_SHA256; +signature_scheme(rsa_pkcs1_sha384) -> ?RSA_PKCS1_SHA384; +signature_scheme(rsa_pkcs1_sha512) -> ?RSA_PKCS1_SHA512; +signature_scheme(ecdsa_secp256r1_sha256) -> ?ECDSA_SECP256R1_SHA256; +signature_scheme(ecdsa_secp384r1_sha384) -> ?ECDSA_SECP384R1_SHA384; +signature_scheme(ecdsa_secp521r1_sha512) -> ?ECDSA_SECP521R1_SHA512; +signature_scheme(rsa_pss_rsae_sha256) -> ?RSA_PSS_RSAE_SHA256; +signature_scheme(rsa_pss_rsae_sha384) -> ?RSA_PSS_RSAE_SHA384; +signature_scheme(rsa_pss_rsae_sha512) -> ?RSA_PSS_RSAE_SHA512; +signature_scheme(ed25519) -> ?ED25519; +signature_scheme(ed448) -> ?ED448; +signature_scheme(rsa_pss_pss_sha256) -> ?RSA_PSS_PSS_SHA256; +signature_scheme(rsa_pss_pss_sha384) -> ?RSA_PSS_PSS_SHA384; +signature_scheme(rsa_pss_pss_sha512) -> ?RSA_PSS_PSS_SHA512; +signature_scheme(rsa_pkcs1_sha1) -> ?RSA_PKCS1_SHA1; +signature_scheme(ecdsa_sha1) -> ?ECDSA_SHA1; +signature_scheme(?RSA_PKCS1_SHA256) -> rsa_pkcs1_sha256; +signature_scheme(?RSA_PKCS1_SHA384) -> rsa_pkcs1_sha384; +signature_scheme(?RSA_PKCS1_SHA512) -> rsa_pkcs1_sha512; +signature_scheme(?ECDSA_SECP256R1_SHA256) -> ecdsa_secp256r1_sha256; +signature_scheme(?ECDSA_SECP384R1_SHA384) -> ecdsa_secp384r1_sha384; +signature_scheme(?ECDSA_SECP521R1_SHA512) -> ecdsa_secp521r1_sha512; +signature_scheme(?RSA_PSS_RSAE_SHA256) -> rsa_pss_rsae_sha256; +signature_scheme(?RSA_PSS_RSAE_SHA384) -> rsa_pss_rsae_sha384; +signature_scheme(?RSA_PSS_RSAE_SHA512) -> rsa_pss_rsae_sha512; +signature_scheme(?ED25519) -> ed25519; +signature_scheme(?ED448) -> ed448; +signature_scheme(?RSA_PSS_PSS_SHA256) -> rsa_pss_pss_sha256; +signature_scheme(?RSA_PSS_PSS_SHA384) -> rsa_pss_pss_sha384; +signature_scheme(?RSA_PSS_PSS_SHA512) -> rsa_pss_pss_sha512; +signature_scheme(?RSA_PKCS1_SHA1) -> rsa_pkcs1_sha1; +signature_scheme(?ECDSA_SHA1) -> ecdsa_sha1; +%% Handling legacy signature algorithms for logging purposes. These algorithms +%% cannot be used in TLS 1.3 handshakes. +signature_scheme(SignAlgo) when is_integer(SignAlgo) -> + <<?BYTE(Hash),?BYTE(Sign)>> = <<?UINT16(SignAlgo)>>, + {ssl_cipher:hash_algorithm(Hash), ssl_cipher:sign_algorithm(Sign)}; +signature_scheme(_) -> unassigned. +%% TODO: reserved code points? + +scheme_to_components(rsa_pkcs1_sha256) -> {sha256, rsa_pkcs1, undefined}; +scheme_to_components(rsa_pkcs1_sha384) -> {sha384, rsa_pkcs1, undefined}; +scheme_to_components(rsa_pkcs1_sha512) -> {sha512, rsa_pkcs1, undefined}; +scheme_to_components(ecdsa_secp256r1_sha256) -> {sha256, ecdsa, secp256r1}; +scheme_to_components(ecdsa_secp384r1_sha384) -> {sha384, ecdsa, secp384r1}; +scheme_to_components(ecdsa_secp521r1_sha512) -> {sha512, ecdsa, secp521r1}; +scheme_to_components(rsa_pss_rsae_sha256) -> {sha256, rsa_pss_rsae, undefined}; +scheme_to_components(rsa_pss_rsae_sha384) -> {sha384, rsa_pss_rsae, undefined}; +scheme_to_components(rsa_pss_rsae_sha512) -> {sha512, rsa_pss_rsae, undefined}; +scheme_to_components(ed25519) -> {undefined, undefined, undefined}; +scheme_to_components(ed448) -> {undefined, undefined, undefined}; +scheme_to_components(rsa_pss_pss_sha256) -> {sha256, rsa_pss_pss, undefined}; +scheme_to_components(rsa_pss_pss_sha384) -> {sha384, rsa_pss_pss, undefined}; +scheme_to_components(rsa_pss_pss_sha512) -> {sha512, rsa_pss_pss, undefined}; +scheme_to_components(rsa_pkcs1_sha1) -> {sha1, rsa_pkcs1, undefined}; +scheme_to_components(ecdsa_sha1) -> {sha1, ecdsa, undefined}. + + +%% TODO: Add support for EC and RSA-SSA signatures +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha1WithRSAEncryption}) -> + rsa_pkcs1_sha1; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha256WithRSAEncryption}) -> + rsa_pkcs1_sha256; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha384WithRSAEncryption}) -> + rsa_pkcs1_sha384; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha512WithRSAEncryption}) -> + rsa_pkcs1_sha512. + %% RFC 5246: 6.2.3.2. CBC Block Cipher %% @@ -861,7 +1012,7 @@ generic_block_cipher_from_bin({3, N}, T, IV, HashSize) next_iv = IV}; generic_block_cipher_from_bin({3, N}, T, IV, HashSize) - when N == 2; N == 3 -> + when N == 2; N == 3; N == 4 -> Sz1 = byte_size(T) - 1, <<_:Sz1/binary, ?BYTE(PadLength)>> = T, IVLength = byte_size(IV), @@ -1159,3 +1310,55 @@ filter_keyuse_suites(Use, KeyUse, CipherSuits, Suites) -> false -> CipherSuits -- Suites end. + +generate_server_share(Group) -> + Key = generate_key_exchange(Group), + #key_share_server_hello{ + server_share = #key_share_entry{ + group = Group, + key_exchange = Key + }}. + +generate_client_shares([]) -> + #key_share_client_hello{client_shares = []}; +generate_client_shares(Groups) -> + generate_client_shares(Groups, []). +%% +generate_client_shares([], Acc) -> + #key_share_client_hello{client_shares = lists:reverse(Acc)}; +generate_client_shares([Group|Groups], Acc) -> + Key = generate_key_exchange(Group), + KeyShareEntry = #key_share_entry{ + group = Group, + key_exchange = Key + }, + generate_client_shares(Groups, [KeyShareEntry|Acc]). + + +generate_key_exchange(secp256r1) -> + public_key:generate_key({namedCurve, secp256r1}); +generate_key_exchange(secp384r1) -> + public_key:generate_key({namedCurve, secp384r1}); +generate_key_exchange(secp521r1) -> + public_key:generate_key({namedCurve, secp521r1}); +generate_key_exchange(x25519) -> + crypto:generate_key(ecdh, x25519); +generate_key_exchange(x448) -> + crypto:generate_key(ecdh, x448); +generate_key_exchange(FFDHE) -> + public_key:generate_key(ssl_dh_groups:dh_params(FFDHE)). + + +%% TODO: Move this functionality to crypto! +%% 7.4.1. Finite Field Diffie-Hellman +%% +%% For finite field groups, a conventional Diffie-Hellman [DH76] +%% computation is performed. The negotiated key (Z) is converted to a +%% byte string by encoding in big-endian form and left-padded with zeros +%% up to the size of the prime. This byte string is used as the shared +%% secret in the key schedule as specified above. +add_zero_padding(Bin, PrimeSize) + when byte_size (Bin) =:= PrimeSize -> + Bin; +add_zero_padding(Bin, PrimeSize) -> + add_zero_padding(<<0, Bin/binary>>, PrimeSize). diff --git a/lib/ssl/src/ssl_cipher.hrl b/lib/ssl/src/ssl_cipher.hrl index 2371e8bd32..0fa5f66c49 100644 --- a/lib/ssl/src/ssl_cipher.hrl +++ b/lib/ssl/src/ssl_cipher.hrl @@ -47,6 +47,7 @@ -record(cipher_state, { iv, key, + finished_key, state, nonce, tag_len @@ -600,15 +601,98 @@ %% TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 = {0xC0,0x32}; -define(TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384, <<?BYTE(16#C0), ?BYTE(16#32)>>). -%%% Chacha20/Poly1305 Suites draft-agl-tls-chacha20poly1305-04 -%% TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = {0xcc, 0x13} --define(TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#13)>>). +%%% ChaCha20-Poly1305 Cipher Suites for Transport Layer Security (TLS) RFC7905 -%% TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = {0xcc, 0x14} --define(TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#14)>>). +%% TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = {0xCC, 0xA8} +-define(TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#A8)>>). -%% TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = {0xcc, 0x15} --define(TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#15)>>). +%% TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = {0xCC, 0xA9} +-define(TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#A9)>>). + +%% TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = {0xCC, 0xAA} +-define(TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#AA)>>). + +%% TLS_PSK_WITH_CHACHA20_POLY1305_SHA256 = {0xCC, 0xAB} +-define(TLS_PSK_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#AB)>>). + +%% TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256 = {0xCC, 0xAC} +-define(TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#AC)>>). + +%% TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256 = {0xCC, 0xAD} +-define(TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#AD)>>). + +%% TLS_RSA_PSK_WITH_CHACHA20_POLY1305_SHA256 = {0xCC, 0xAE} +-define(TLS_RSA_PSK_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#AE)>>). + + + +%% RFC 6655 - TLS-1.2 cipher suites + +%% TLS_RSA_WITH_AES_128_CCM = {0xC0,0x9C} +-define(TLS_RSA_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#9C)>>). + +%% TLS_RSA_WITH_AES_256_CCM = {0xC0,0x9D} +-define(TLS_RSA_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#9D)>>). + +%% TLS_DHE_RSA_WITH_AES_256_CCM = {0xC0,0x9E} +-define(TLS_DHE_RSA_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#9E)>>). + +%% TLS_DHE_RSA_WITH_AES_128_CCM = {0xC0,0x9F} +-define(TLS_DHE_RSA_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#9F)>>). + +%% TLS_RSA_WITH_AES_256_CCM_8 = {0xC0,0x9A0} +-define(TLS_RSA_WITH_AES_256_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A0)>>). + +%% TLS_RSA_WITH_AES_128_CCM_8 = {0xC0,0xA1} +-define(TLS_RSA_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A1)>>). + +%% TLS_DHE_RSA_WITH_AES_128_CCM_8 = {0xC0,0xA2} +-define(TLS_DHE_RSA_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A2)>>). + +%% TLS_DHE_RSA_WITH_AES_256_CCM_8 = {0xC0,0xA3} +-define(TLS_DHE_RSA_WITH_AES_256_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A3)>>). + +%% TLS_PSK_WITH_AES_128_CCM = {0xC0,0xA4} +-define(TLS_PSK_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#A4)>>). + +%% TLS_PSK_WITH_AES_256_CCM = {0xC0,0xA5) +-define(TLS_PSK_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#A5)>>). + +%% TLS_DHE_PSK_WITH_AES_128_CCM = {0xC0,0xA6} +-define(TLS_DHE_PSK_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#A6)>>). + +%% TLS_DHE_PSK_WITH_AES_256_CCM = {0xC0,0xA7} +-define(TLS_DHE_PSK_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#A7)>>). + +%% TLS_PSK_WITH_AES_128_CCM_8 = {0xC0,0xA8} +-define(TLS_PSK_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A8)>>). + +%% TLS_PSK_WITH_AES_256_CCM_8 = {0xC0,0xA9) +-define(TLS_PSK_WITH_AES_256_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A9)>>). + +%% TLS_PSK_DHE_WITH_AES_128_CCM_8 = {0xC0,0xAA} +-define(TLS_PSK_DHE_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#AA)>>). + +%% TLS_PSK_DHE_WITH_AES_256_CCM_8 = << ?BYTE(0xC0,0xAB} +-define(TLS_PSK_DHE_WITH_AES_256_CCM_8, <<?BYTE(16#C0),?BYTE(16#AB)>>). + + +%%% TLS 1.3 cipher suites RFC8446 + +%% TLS_AES_128_GCM_SHA256 = {0x13,0x01} +-define(TLS_AES_128_GCM_SHA256, <<?BYTE(16#13), ?BYTE(16#01)>>). + +%% TLS_AES_256_GCM_SHA384 = {0x13,0x02} +-define(TLS_AES_256_GCM_SHA384, <<?BYTE(16#13),?BYTE(16#02)>>). + +%% TLS_CHACHA20_POLY1305_SHA256 = {0x13,0x03} +-define(TLS_CHACHA20_POLY1305_SHA256, <<?BYTE(16#13),?BYTE(16#03)>>). + +%% %% TLS_AES_128_CCM_SHA256 = {0x13,0x04} +-define(TLS_AES_128_CCM_SHA256, <<?BYTE(16#13), ?BYTE(16#04)>>). + +%% %% TLS_AES_128_CCM_8_SHA256 = {0x13,0x05} +-define(TLS_AES_128_CCM_8_SHA256, <<?BYTE(16#13),?BYTE(16#05)>>). -endif. % -ifdef(ssl_cipher). diff --git a/lib/ssl/src/ssl_cipher_format.erl b/lib/ssl/src/ssl_cipher_format.erl index 1d28e1e3b4..577156a4b5 100644 --- a/lib/ssl/src/ssl_cipher_format.erl +++ b/lib/ssl/src/ssl_cipher_format.erl @@ -48,42 +48,134 @@ -type openssl_cipher_suite() :: string(). --export([suite_to_str/1, suite_definition/1, suite/1, erl_suite_definition/1, - openssl_suite/1, openssl_suite_name/1]). +-export([suite_map_to_bin/1, %% Binary format + suite_bin_to_map/1, %% Erlang API format + suite_map_to_str/1, %% RFC string + suite_str_to_map/1, + suite_map_to_openssl_str/1, %% OpenSSL name + suite_openssl_str_to_map/1, + suite_legacy/1 %% Erlang legacy format + ]). %%-------------------------------------------------------------------- --spec suite_to_str(internal_erl_cipher_suite()) -> string(). +-spec suite_map_to_str(internal_erl_cipher_suite()) -> string(). %% %% Description: Return the string representation of a cipher suite. %%-------------------------------------------------------------------- -suite_to_str(#{key_exchange := null, +suite_map_to_str(#{key_exchange := null, cipher := null, mac := null, prf := null}) -> "TLS_EMPTY_RENEGOTIATION_INFO_SCSV"; -suite_to_str(#{key_exchange := Kex, +suite_map_to_str(#{key_exchange := any, + cipher := Cipher, + mac := aead, + prf := PRF}) -> + "TLS_" ++ string:to_upper(atom_to_list(Cipher)) ++ + "_" ++ string:to_upper(atom_to_list(PRF)); +suite_map_to_str(#{key_exchange := Kex, cipher := Cipher, mac := aead, prf := PRF}) -> "TLS_" ++ string:to_upper(atom_to_list(Kex)) ++ "_WITH_" ++ string:to_upper(atom_to_list(Cipher)) ++ "_" ++ string:to_upper(atom_to_list(PRF)); -suite_to_str(#{key_exchange := Kex, +suite_map_to_str(#{key_exchange := Kex, cipher := Cipher, mac := Mac}) -> "TLS_" ++ string:to_upper(atom_to_list(Kex)) ++ "_WITH_" ++ string:to_upper(atom_to_list(Cipher)) ++ "_" ++ string:to_upper(atom_to_list(Mac)). +suite_str_to_map("TLS_EMPTY_RENEGOTIATION_INFO_SCSV") -> + #{key_exchange => null, + cipher => null, + mac => null, + prf => null}; +suite_str_to_map(SuiteStr)-> + Str0 = string:trim(SuiteStr, leading, "TLS_"), + case string:split(Str0, "_WITH_") of + [Rest] -> + tls_1_3_suite_str_to_map(Rest); + [Kex| Rest] -> + pre_tls_1_3_suite_str_to_map(Kex, Rest) + end. + +suite_map_to_openssl_str(#{key_exchange := any, + mac := aead} = Suite) -> + %% TLS 1.3 OpenSSL finally use RFC names + suite_map_to_str(Suite); +suite_map_to_openssl_str(#{key_exchange := null} = Suite) -> + %% TLS_EMPTY_RENEGOTIATION_INFO_SCSV + suite_map_to_str(Suite); +suite_map_to_openssl_str(#{key_exchange := Kex, + cipher := chacha20_poly1305 = Cipher, + mac := aead}) -> + openssl_suite_start(string:to_upper(atom_to_list(Kex))) + ++ openssl_cipher_name(Kex, string:to_upper(atom_to_list(Cipher))); +suite_map_to_openssl_str(#{key_exchange := Kex, + cipher := Cipher, + mac := aead, + prf := PRF}) -> + openssl_suite_start(string:to_upper(atom_to_list(Kex))) + ++ openssl_cipher_name(Kex, string:to_upper(atom_to_list(Cipher))) ++ + "-" ++ string:to_upper(atom_to_list(PRF)); +suite_map_to_openssl_str(#{key_exchange := Kex, + cipher := Cipher, + mac := Mac}) -> + openssl_suite_start(string:to_upper(atom_to_list(Kex))) + ++ openssl_cipher_name(Kex, string:to_upper(atom_to_list(Cipher))) ++ + "-" ++ string:to_upper(atom_to_list(Mac)). + + +suite_openssl_str_to_map("TLS_" ++ _ = SuiteStr) -> + suite_str_to_map(SuiteStr); +suite_openssl_str_to_map("DHE-RSA-" ++ Rest) -> + suite_openssl_str_to_map("DHE-RSA", Rest); +suite_openssl_str_to_map("DHE-DSS-" ++ Rest) -> + suite_openssl_str_to_map("DHE-DSS", Rest); +suite_openssl_str_to_map("EDH-RSA-" ++ Rest) -> + suite_openssl_str_to_map("DHE-RSA", Rest); +suite_openssl_str_to_map("EDH-DSS-" ++ Rest) -> + suite_openssl_str_to_map("DHE-DSS", Rest); +suite_openssl_str_to_map("DES" ++ _ = Rest) -> + suite_openssl_str_to_map("RSA", Rest); +suite_openssl_str_to_map("AES" ++ _ = Rest) -> + suite_openssl_str_to_map("RSA", Rest); +suite_openssl_str_to_map("RC4" ++ _ = Rest) -> + suite_openssl_str_to_map("RSA", Rest); +suite_openssl_str_to_map("ECDH-RSA-" ++ Rest) -> + suite_openssl_str_to_map("ECDH-RSA", Rest); +suite_openssl_str_to_map("ECDH-ECDSA-" ++ Rest) -> + suite_openssl_str_to_map("ECDH-ECDSA", Rest); +suite_openssl_str_to_map("ECDHE-RSA-" ++ Rest) -> + suite_openssl_str_to_map("ECDHE-RSA", Rest); +suite_openssl_str_to_map("ECDHE-ECDSA-" ++ Rest) -> + suite_openssl_str_to_map("ECDHE-ECDSA", Rest); +suite_openssl_str_to_map("RSA-PSK-" ++ Rest) -> + suite_openssl_str_to_map("RSA-PSK", Rest); +suite_openssl_str_to_map("RSA-" ++ Rest) -> + suite_openssl_str_to_map("RSA", Rest); +suite_openssl_str_to_map("DHE-PSK-" ++ Rest) -> + suite_openssl_str_to_map("DHE-PSK", Rest); +suite_openssl_str_to_map("ECDHE-PSK-" ++ Rest) -> + suite_openssl_str_to_map("ECDHE-PSK", Rest); +suite_openssl_str_to_map("PSK-" ++ Rest) -> + suite_openssl_str_to_map("PSK", Rest); +suite_openssl_str_to_map("SRP-RSA-" ++ Rest) -> + suite_openssl_str_to_map("SRP-RSA", Rest); +suite_openssl_str_to_map("SRP-" ++ Rest) -> + suite_openssl_str_to_map("SRP", Rest). + %%-------------------------------------------------------------------- --spec suite_definition(cipher_suite()) -> internal_erl_cipher_suite(). +-spec suite_bin_to_map(cipher_suite()) -> internal_erl_cipher_suite(). %% %% Description: Return erlang cipher suite definition. %% Note: Currently not supported suites are commented away. %% They should be supported or removed in the future. %%------------------------------------------------------------------- %% TLS v1.1 suites -suite_definition(?TLS_NULL_WITH_NULL_NULL) -> +suite_bin_to_map(?TLS_NULL_WITH_NULL_NULL) -> #{key_exchange => null, cipher => null, mac => null, @@ -91,111 +183,111 @@ suite_definition(?TLS_NULL_WITH_NULL_NULL) -> %% RFC 5746 - Not a real cipher suite used to signal empty "renegotiation_info" extension %% to avoid handshake failure from old servers that do not ignore %% hello extension data as they should. -suite_definition(?TLS_EMPTY_RENEGOTIATION_INFO_SCSV) -> +suite_bin_to_map(?TLS_EMPTY_RENEGOTIATION_INFO_SCSV) -> #{key_exchange => null, cipher => null, mac => null, prf => null}; -suite_definition(?TLS_RSA_WITH_RC4_128_MD5) -> +suite_bin_to_map(?TLS_RSA_WITH_RC4_128_MD5) -> #{key_exchange => rsa, cipher => rc4_128, mac => md5, prf => default_prf}; -suite_definition(?TLS_RSA_WITH_RC4_128_SHA) -> +suite_bin_to_map(?TLS_RSA_WITH_RC4_128_SHA) -> #{key_exchange => rsa, cipher => rc4_128, mac => sha, prf => default_prf}; -suite_definition(?TLS_RSA_WITH_DES_CBC_SHA) -> +suite_bin_to_map(?TLS_RSA_WITH_DES_CBC_SHA) -> #{key_exchange => rsa, cipher => des_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_RSA_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_RSA_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => rsa, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_DSS_WITH_DES_CBC_SHA) -> +suite_bin_to_map(?TLS_DHE_DSS_WITH_DES_CBC_SHA) -> #{key_exchange => dhe_dss, cipher => des_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => dhe_dss, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_RSA_WITH_DES_CBC_SHA) -> +suite_bin_to_map(?TLS_DHE_RSA_WITH_DES_CBC_SHA) -> #{key_exchange => dhe_rsa, cipher => des_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => dhe_rsa, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; %%% TSL V1.1 AES suites -suite_definition(?TLS_RSA_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_RSA_WITH_AES_128_CBC_SHA) -> #{key_exchange => rsa, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_DSS_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_DHE_DSS_WITH_AES_128_CBC_SHA) -> #{key_exchange => dhe_dss, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_RSA_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_DHE_RSA_WITH_AES_128_CBC_SHA) -> #{key_exchange => dhe_rsa, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_RSA_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_RSA_WITH_AES_256_CBC_SHA) -> #{key_exchange => rsa, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_DSS_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_DHE_DSS_WITH_AES_256_CBC_SHA) -> #{key_exchange => dhe_dss, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_RSA_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_DHE_RSA_WITH_AES_256_CBC_SHA) -> #{key_exchange => dhe_rsa, cipher => aes_256_cbc, mac => sha, prf => default_prf}; %% TLS v1.2 suites -%% suite_definition(?TLS_RSA_WITH_NULL_SHA) -> +%% suite_bin_to_map(?TLS_RSA_WITH_NULL_SHA) -> %% {rsa, null, sha, default_prf}; -suite_definition(?TLS_RSA_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_RSA_WITH_AES_128_CBC_SHA256) -> #{key_exchange => rsa, cipher => aes_128_cbc, mac => sha256, prf => default_prf}; -suite_definition(?TLS_RSA_WITH_AES_256_CBC_SHA256) -> +suite_bin_to_map(?TLS_RSA_WITH_AES_256_CBC_SHA256) -> #{key_exchange => rsa, cipher => aes_256_cbc, mac => sha256, prf => default_prf}; -suite_definition(?TLS_DHE_DSS_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_DHE_DSS_WITH_AES_128_CBC_SHA256) -> #{key_exchange => dhe_dss, cipher => aes_128_cbc, mac => sha256, prf => default_prf}; -suite_definition(?TLS_DHE_RSA_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_DHE_RSA_WITH_AES_128_CBC_SHA256) -> #{key_exchange => dhe_rsa, cipher => aes_128_cbc, mac => sha256, prf => default_prf}; -suite_definition(?TLS_DHE_DSS_WITH_AES_256_CBC_SHA256) -> +suite_bin_to_map(?TLS_DHE_DSS_WITH_AES_256_CBC_SHA256) -> #{key_exchange => dhe_dss, cipher => aes_256_cbc, mac => sha256, prf => default_prf}; -suite_definition(?TLS_DHE_RSA_WITH_AES_256_CBC_SHA256) -> +suite_bin_to_map(?TLS_DHE_RSA_WITH_AES_256_CBC_SHA256) -> #{key_exchange => dhe_rsa, cipher => aes_256_cbc, mac => sha256, @@ -207,611 +299,683 @@ suite_definition(?TLS_DHE_RSA_WITH_AES_256_CBC_SHA256) -> %% TLS_DH_RSA_WITH_AES_256_CBC_SHA256 DH_RSA AES_256_CBC SHA256 %%% DH-ANON deprecated by TLS spec and not available %%% by default, but good for testing purposes. -suite_definition(?TLS_DH_anon_WITH_RC4_128_MD5) -> +suite_bin_to_map(?TLS_DH_anon_WITH_RC4_128_MD5) -> #{key_exchange => dh_anon, cipher => rc4_128, mac => md5, prf => default_prf}; -suite_definition(?TLS_DH_anon_WITH_DES_CBC_SHA) -> +suite_bin_to_map(?TLS_DH_anon_WITH_DES_CBC_SHA) -> #{key_exchange => dh_anon, cipher => des_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_DH_anon_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_DH_anon_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => dh_anon, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_DH_anon_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_DH_anon_WITH_AES_128_CBC_SHA) -> #{key_exchange => dh_anon, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_DH_anon_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_DH_anon_WITH_AES_256_CBC_SHA) -> #{key_exchange => dh_anon, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_DH_anon_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_DH_anon_WITH_AES_128_CBC_SHA256) -> #{key_exchange => dh_anon, cipher => aes_128_cbc, mac => sha256, prf => default_prf}; -suite_definition(?TLS_DH_anon_WITH_AES_256_CBC_SHA256) -> +suite_bin_to_map(?TLS_DH_anon_WITH_AES_256_CBC_SHA256) -> #{key_exchange => dh_anon, cipher => aes_256_cbc, mac => sha256, prf => default_prf}; %%% PSK Cipher Suites RFC 4279 -suite_definition(?TLS_PSK_WITH_RC4_128_SHA) -> +suite_bin_to_map(?TLS_PSK_WITH_RC4_128_SHA) -> #{key_exchange => psk, cipher => rc4_128, mac => sha, prf => default_prf}; -suite_definition(?TLS_PSK_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_PSK_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => psk, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_PSK_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_PSK_WITH_AES_128_CBC_SHA) -> #{key_exchange => psk, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_PSK_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_PSK_WITH_AES_256_CBC_SHA) -> #{key_exchange => psk, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_PSK_WITH_RC4_128_SHA) -> +suite_bin_to_map(?TLS_DHE_PSK_WITH_RC4_128_SHA) -> #{key_exchange => dhe_psk, cipher => rc4_128, mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => dhe_psk, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_PSK_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_128_CBC_SHA) -> #{key_exchange => dhe_psk, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_PSK_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_256_CBC_SHA) -> #{key_exchange => dhe_psk, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_RSA_PSK_WITH_RC4_128_SHA) -> +suite_bin_to_map(?TLS_RSA_PSK_WITH_RC4_128_SHA) -> #{key_exchange => rsa_psk, cipher => rc4_128, mac => sha, prf => default_prf}; -suite_definition(?TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => rsa_psk, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_RSA_PSK_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_RSA_PSK_WITH_AES_128_CBC_SHA) -> #{key_exchange => rsa_psk, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_RSA_PSK_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_RSA_PSK_WITH_AES_256_CBC_SHA) -> #{key_exchange => rsa_psk, cipher => aes_256_cbc, mac => sha, prf => default_prf}; %%% PSK NULL Cipher Suites RFC 4785 -suite_definition(?TLS_PSK_WITH_NULL_SHA) -> +suite_bin_to_map(?TLS_PSK_WITH_NULL_SHA) -> #{key_exchange => psk, cipher => null, mac => sha, prf => default_prf}; -suite_definition(?TLS_DHE_PSK_WITH_NULL_SHA) -> +suite_bin_to_map(?TLS_DHE_PSK_WITH_NULL_SHA) -> #{key_exchange => dhe_psk, cipher => null, mac => sha, prf => default_prf}; -suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA) -> +suite_bin_to_map(?TLS_RSA_PSK_WITH_NULL_SHA) -> #{key_exchange => rsa_psk, cipher => null, mac => sha, prf => default_prf}; %%% TLS 1.2 PSK Cipher Suites RFC 5487 -suite_definition(?TLS_PSK_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_PSK_WITH_AES_128_GCM_SHA256) -> #{key_exchange => psk, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_PSK_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_PSK_WITH_AES_256_GCM_SHA384) -> #{key_exchange => psk, cipher => aes_256_gcm, mac => aead, prf => sha384}; -suite_definition(?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dhe_psk, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dhe_psk, cipher => aes_256_gcm, mac => aead, prf => sha384}; -suite_definition(?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256) -> #{key_exchange => rsa_psk, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384) -> #{key_exchange => rsa_psk, cipher => aes_256_gcm, mac => aead, prf => sha384}; -suite_definition(?TLS_PSK_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_PSK_WITH_AES_128_CBC_SHA256) -> #{key_exchange => psk, cipher => aes_128_cbc, mac => sha256, prf => default_prf}; -suite_definition(?TLS_PSK_WITH_AES_256_CBC_SHA384) -> +suite_bin_to_map(?TLS_PSK_WITH_AES_256_CBC_SHA384) -> #{key_exchange => psk, cipher => aes_256_cbc, mac => sha384, prf => default_prf}; -suite_definition(?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256) -> #{key_exchange => dhe_psk, cipher => aes_128_cbc, mac => sha256, prf => default_prf}; -suite_definition(?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384) -> +suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384) -> #{key_exchange => dhe_psk, cipher => aes_256_cbc, mac => sha384, prf => default_prf}; -suite_definition(?TLS_RSA_PSK_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_RSA_PSK_WITH_AES_128_CBC_SHA256) -> #{key_exchange => rsa_psk, cipher => aes_128_cbc, mac => sha256, prf => default_prf}; -suite_definition(?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384) -> +suite_bin_to_map(?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384) -> #{key_exchange => rsa_psk, cipher => aes_256_cbc, mac => sha384, prf => default_prf}; -suite_definition(?TLS_PSK_WITH_NULL_SHA256) -> +suite_bin_to_map(?TLS_PSK_WITH_NULL_SHA256) -> #{key_exchange => psk, cipher => null, mac => sha256, prf => default_prf}; -suite_definition(?TLS_PSK_WITH_NULL_SHA384) -> +suite_bin_to_map(?TLS_PSK_WITH_NULL_SHA384) -> #{key_exchange => psk, cipher => null, mac => sha384, prf => default_prf}; -suite_definition(?TLS_DHE_PSK_WITH_NULL_SHA256) -> +suite_bin_to_map(?TLS_DHE_PSK_WITH_NULL_SHA256) -> #{key_exchange => dhe_psk, cipher => null, mac => sha256, prf => default_prf}; -suite_definition(?TLS_DHE_PSK_WITH_NULL_SHA384) -> +suite_bin_to_map(?TLS_DHE_PSK_WITH_NULL_SHA384) -> #{key_exchange => dhe_psk, cipher => null, mac => sha384, prf => default_prf}; -suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA256) -> +suite_bin_to_map(?TLS_RSA_PSK_WITH_NULL_SHA256) -> #{key_exchange => rsa_psk, cipher => null, mac => sha256, prf => default_prf}; -suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA384) -> +suite_bin_to_map(?TLS_RSA_PSK_WITH_NULL_SHA384) -> #{key_exchange => rsa_psk, cipher => null, mac => sha384, prf => default_prf}; %%% ECDHE PSK Cipher Suites RFC 5489 -suite_definition(?TLS_ECDHE_PSK_WITH_RC4_128_SHA) -> +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_RC4_128_SHA) -> #{key_exchange => ecdhe_psk, cipher => rc4_128, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => ecdhe_psk, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA) -> #{key_exchange => ecdhe_psk, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA) -> #{key_exchange => ecdhe_psk, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256) -> #{key_exchange => ecdhe_psk, cipher => aes_128_cbc, mac => sha256, prf => default_prf}; -suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384) -> +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384) -> #{key_exchange => ecdhe_psk, cipher => aes_256_cbc, mac => sha384, prf => default_prf}; -suite_definition(?TLS_ECDHE_PSK_WITH_NULL_SHA256) -> +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_NULL_SHA256) -> #{key_exchange => ecdhe_psk, cipher => null, mac => sha256, prf => default_prf}; -suite_definition(?TLS_ECDHE_PSK_WITH_NULL_SHA384) -> +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_NULL_SHA384) -> #{key_exchange => ecdhe_psk, cipher => null, mac => sha384, prf => default_prf}; %%% ECDHE_PSK with AES-GCM and AES-CCM Cipher Suites, draft-ietf-tls-ecdhe-psk-aead-05 -suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdhe_psk, cipher => aes_128_gcm, mac => null, prf => sha256}; -suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdhe_psk, cipher => aes_256_gcm, mac => null, prf => sha384}; -%% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256) -> -%% #{key_exchange => ecdhe_psk, -%% cipher => aes_128_ccm, -%% mac => null, -%% prf =>sha256}; -%% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256) -> -%% #{key_exchange => ecdhe_psk, -%% cipher => aes_256_ccm, -%% mac => null, -%% prf => sha256}; +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256) -> + #{key_exchange => ecdhe_psk, + cipher => aes_128_ccm, + mac => null, + prf =>sha256}; +suite_bin_to_map(?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256) -> + #{key_exchange => ecdhe_psk, + cipher => aes_128_ccm_8, + mac => null, + prf =>sha256}; %%% SRP Cipher Suites RFC 5054 -suite_definition(?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => srp_anon, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => srp_rsa, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => srp_dss, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_SRP_SHA_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_SRP_SHA_WITH_AES_128_CBC_SHA) -> #{key_exchange => srp_anon, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA) -> #{key_exchange => srp_rsa, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA) -> #{key_exchange => srp_dss, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_SRP_SHA_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_SRP_SHA_WITH_AES_256_CBC_SHA) -> #{key_exchange => srp_anon, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA) -> #{key_exchange => srp_rsa, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA) -> #{key_exchange => srp_dss, cipher => aes_256_cbc, mac => sha, prf => default_prf}; %% RFC 4492 EC TLS suites -suite_definition(?TLS_ECDH_ECDSA_WITH_NULL_SHA) -> +suite_bin_to_map(?TLS_ECDH_ECDSA_WITH_NULL_SHA) -> #{key_exchange => ecdh_ecdsa, cipher => null, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_ECDSA_WITH_RC4_128_SHA) -> +suite_bin_to_map(?TLS_ECDH_ECDSA_WITH_RC4_128_SHA) -> #{key_exchange => ecdh_ecdsa, cipher => rc4_128, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => ecdh_ecdsa, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA) -> #{key_exchange => ecdh_ecdsa, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA) -> #{key_exchange => ecdh_ecdsa, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_ECDSA_WITH_NULL_SHA) -> +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_NULL_SHA) -> #{key_exchange => ecdhe_ecdsa, cipher => null, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_ECDSA_WITH_RC4_128_SHA) -> +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_RC4_128_SHA) -> #{key_exchange => ecdhe_ecdsa, cipher => rc4_128, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => ecdhe_ecdsa, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA) -> #{key_exchange => ecdhe_ecdsa, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA) -> #{key_exchange => ecdhe_ecdsa, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_RSA_WITH_NULL_SHA) -> +suite_bin_to_map(?TLS_ECDH_RSA_WITH_NULL_SHA) -> #{key_exchange => ecdh_rsa, cipher => null, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_RSA_WITH_RC4_128_SHA) -> +suite_bin_to_map(?TLS_ECDH_RSA_WITH_RC4_128_SHA) -> #{key_exchange => ecdh_rsa, cipher => rc4_128, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => ecdh_rsa, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA) -> #{key_exchange => ecdh_rsa, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA) -> #{key_exchange => ecdh_rsa, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_RSA_WITH_NULL_SHA) -> +suite_bin_to_map(?TLS_ECDHE_RSA_WITH_NULL_SHA) -> #{key_exchange => ecdhe_rsa, cipher => null, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_RSA_WITH_RC4_128_SHA) -> +suite_bin_to_map(?TLS_ECDHE_RSA_WITH_RC4_128_SHA) -> #{key_exchange => ecdhe_rsa, cipher => rc4_128, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => ecdhe_rsa, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA) -> #{key_exchange => ecdhe_rsa, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA) -> #{key_exchange => ecdhe_rsa, cipher => aes_256_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_anon_WITH_NULL_SHA) -> +suite_bin_to_map(?TLS_ECDH_anon_WITH_NULL_SHA) -> #{key_exchange => ecdh_anon, cipher => null, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_anon_WITH_RC4_128_SHA) -> +suite_bin_to_map(?TLS_ECDH_anon_WITH_RC4_128_SHA) -> #{key_exchange => ecdh_anon, cipher => rc4_128, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => ecdh_anon, cipher => '3des_ede_cbc', mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_anon_WITH_AES_128_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDH_anon_WITH_AES_128_CBC_SHA) -> #{key_exchange => ecdh_anon, cipher => aes_128_cbc, mac => sha, prf => default_prf}; -suite_definition(?TLS_ECDH_anon_WITH_AES_256_CBC_SHA) -> +suite_bin_to_map(?TLS_ECDH_anon_WITH_AES_256_CBC_SHA) -> #{key_exchange => ecdh_anon, cipher => aes_256_cbc, mac => sha, prf => default_prf}; %% RFC 5289 EC TLS suites -suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256) -> #{key_exchange => ecdhe_ecdsa, cipher => aes_128_cbc, mac => sha256, prf => sha256}; -suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384) -> +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384) -> #{key_exchange => ecdhe_ecdsa, cipher => aes_256_cbc, mac => sha384, prf => sha384}; -suite_definition(?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256) -> #{key_exchange => ecdh_ecdsa, cipher => aes_128_cbc, mac => sha256, prf => sha256}; -suite_definition(?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384) -> +suite_bin_to_map(?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384) -> #{key_exchange => ecdh_ecdsa, cipher => aes_256_cbc, mac => sha384, prf => sha384}; -suite_definition(?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256) -> #{key_exchange => ecdhe_rsa, cipher => aes_128_cbc, mac => sha256, prf => sha256}; -suite_definition(?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) -> +suite_bin_to_map(?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) -> #{key_exchange => ecdhe_rsa, cipher => aes_256_cbc, mac => sha384, prf => sha384}; -suite_definition(?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256) -> +suite_bin_to_map(?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256) -> #{key_exchange => ecdh_rsa, cipher => aes_128_cbc, mac => sha256, prf => sha256}; -suite_definition(?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384) -> +suite_bin_to_map(?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384) -> #{key_exchange => ecdh_rsa, cipher => aes_256_cbc, mac => sha384, prf => sha384}; %% RFC 5288 AES-GCM Cipher Suites -suite_definition(?TLS_RSA_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => rsa, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_RSA_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => rsa, cipher => aes_256_gcm, mac => aead, prf => sha384}; -suite_definition(?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dhe_rsa, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dhe_rsa, cipher => aes_256_gcm, mac => aead, prf => sha384}; -suite_definition(?TLS_DH_RSA_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_DH_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dh_rsa, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_DH_RSA_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_DH_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dh_rsa, cipher => aes_256_gcm, mac => aead, prf => sha384}; -suite_definition(?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dhe_dss, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dhe_dss, cipher => aes_256_gcm, mac => aead, prf => sha384}; -suite_definition(?TLS_DH_DSS_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_DH_DSS_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dh_dss, cipher => aes_128_gcm, mac => null, prf => sha256}; -suite_definition(?TLS_DH_DSS_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_DH_DSS_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dh_dss, cipher => aes_256_gcm, mac => aead, prf => sha384}; -suite_definition(?TLS_DH_anon_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_DH_anon_WITH_AES_128_GCM_SHA256) -> #{key_exchange => dh_anon, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_DH_anon_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_DH_anon_WITH_AES_256_GCM_SHA384) -> #{key_exchange => dh_anon, cipher => aes_256_gcm, mac => aead, prf => sha384}; %% RFC 5289 ECC AES-GCM Cipher Suites -suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdhe_ecdsa, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdhe_ecdsa, cipher => aes_256_gcm, mac => aead, prf => sha384}; -suite_definition(?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdh_ecdsa, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdh_ecdsa, cipher => aes_256_gcm, mac => aead, prf => sha384}; -suite_definition(?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdhe_rsa, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdhe_rsa, cipher => aes_256_gcm, mac => aead, prf => sha384}; -suite_definition(?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256) -> +suite_bin_to_map(?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256) -> #{key_exchange => ecdh_rsa, cipher => aes_128_gcm, mac => aead, prf => sha256}; -suite_definition(?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384) -> +suite_bin_to_map(?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384) -> #{key_exchange => ecdh_rsa, cipher => aes_256_gcm, mac => aead, prf => sha384}; -%% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites -suite_definition(?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> +suite_bin_to_map(?TLS_PSK_WITH_AES_128_CCM) -> + #{key_exchange => psk, + cipher => aes_128_ccm, + mac => aead, + prf => sha256}; +suite_bin_to_map(?TLS_PSK_WITH_AES_256_CCM) -> + #{key_exchange => psk, + cipher => aes_256_ccm, + mac => aead, + prf => sha256}; +suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_128_CCM) -> + #{key_exchange => dhe_psk, + cipher => aes_128_ccm, + mac => aead, + prf => sha256}; +suite_bin_to_map(?TLS_DHE_PSK_WITH_AES_256_CCM) -> + #{key_exchange => dhe_psk, + cipher => aes_256_ccm, + mac => aead, + prf => sha256}; +suite_bin_to_map(?TLS_PSK_WITH_AES_128_CCM_8) -> + #{key_exchange => psk, + cipher => aes_128_ccm_8, + mac => aead, + prf => sha256}; +suite_bin_to_map(?TLS_PSK_WITH_AES_256_CCM_8) -> + #{key_exchange => psk, + cipher => aes_256_ccm_8, + mac => aead, + prf => sha256}; +suite_bin_to_map(?TLS_PSK_DHE_WITH_AES_128_CCM_8) -> + #{key_exchange => dhe_psk, + cipher => aes_128_ccm_8, + mac => aead, + prf => sha256}; +suite_bin_to_map(?TLS_PSK_DHE_WITH_AES_256_CCM_8) -> + #{key_exchange => dhe_psk, + cipher => aes_256_ccm_8, + mac => aead, + prf => sha256}; +suite_bin_to_map(#{key_exchange := psk_dhe, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_DHE_WITH_AES_256_CCM_8; + +% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites +suite_bin_to_map(?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> #{key_exchange => ecdhe_rsa, cipher => chacha20_poly1305, mac => aead, prf => sha256}; -suite_definition(?TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256) -> +suite_bin_to_map(?TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256) -> #{key_exchange => ecdhe_ecdsa, cipher => chacha20_poly1305, mac => aead, prf => sha256}; -suite_definition(?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> +suite_bin_to_map(?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> #{key_exchange => dhe_rsa, cipher => chacha20_poly1305, mac => aead, - prf => sha256}. + prf => sha256}; +%% TLS 1.3 Cipher Suites RFC8446 +suite_bin_to_map(?TLS_AES_128_GCM_SHA256) -> + #{key_exchange => any, + cipher => aes_128_gcm, + mac => aead, + prf => sha256}; +suite_bin_to_map(?TLS_AES_256_GCM_SHA384) -> + #{key_exchange => any, + cipher => aes_256_gcm, + mac => aead, + prf => sha384}; +suite_bin_to_map(?TLS_CHACHA20_POLY1305_SHA256) -> + #{key_exchange => any, + cipher => chacha20_poly1305, + mac => aead, + prf => sha256}; +suite_bin_to_map(?TLS_AES_128_CCM_SHA256) -> + #{key_exchange => any, + cipher => aes_128_ccm, + mac => aead, + prf => sha256}. +%% suite_bin_to_map(?TLS_AES_128_CCM_8_SHA256) -> +%% #{key_exchange => any, +%% cipher => aes_128_ccm_8, +%% mac => aead, +%% prf => sha256}. %%-------------------------------------------------------------------- --spec erl_suite_definition(cipher_suite() | internal_erl_cipher_suite()) -> old_erl_cipher_suite(). +-spec suite_legacy(cipher_suite() | internal_erl_cipher_suite()) -> old_erl_cipher_suite(). %% %% Description: Return erlang cipher suite definition. Filters last value %% for now (compatibility reasons). %%-------------------------------------------------------------------- -erl_suite_definition(Bin) when is_binary(Bin) -> - erl_suite_definition(suite_definition(Bin)); -erl_suite_definition(#{key_exchange := KeyExchange, cipher := Cipher, +suite_legacy(Bin) when is_binary(Bin) -> + suite_legacy(suite_bin_to_map(Bin)); +suite_legacy(#{key_exchange := KeyExchange, cipher := Cipher, mac := Hash, prf := Prf}) -> case Prf of default_prf -> @@ -821,943 +985,896 @@ erl_suite_definition(#{key_exchange := KeyExchange, cipher := Cipher, end. %%-------------------------------------------------------------------- --spec suite(internal_erl_cipher_suite()) -> cipher_suite(). +-spec suite_map_to_bin(internal_erl_cipher_suite()) -> cipher_suite(). %% %% Description: Return TLS cipher suite definition. %%-------------------------------------------------------------------- %% TLS v1.1 suites -suite(#{key_exchange := rsa, +suite_map_to_bin(#{key_exchange := rsa, cipher := rc4_128, mac := md5}) -> ?TLS_RSA_WITH_RC4_128_MD5; -suite(#{key_exchange := rsa, +suite_map_to_bin(#{key_exchange := rsa, cipher := rc4_128, mac := sha}) -> ?TLS_RSA_WITH_RC4_128_SHA; -suite(#{key_exchange := rsa, +suite_map_to_bin(#{key_exchange := rsa, cipher := des_cbc, mac := sha}) -> ?TLS_RSA_WITH_DES_CBC_SHA; -suite(#{key_exchange := rsa, +suite_map_to_bin(#{key_exchange := rsa, cipher :='3des_ede_cbc', mac := sha}) -> ?TLS_RSA_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := dhe_dss, +suite_map_to_bin(#{key_exchange := dhe_dss, cipher:= des_cbc, mac := sha}) -> ?TLS_DHE_DSS_WITH_DES_CBC_SHA; -suite(#{key_exchange := dhe_dss, +suite_map_to_bin(#{key_exchange := dhe_dss, cipher:= '3des_ede_cbc', mac := sha}) -> ?TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := dhe_rsa, +suite_map_to_bin(#{key_exchange := dhe_rsa, cipher:= des_cbc, mac := sha}) -> ?TLS_DHE_RSA_WITH_DES_CBC_SHA; -suite(#{key_exchange := dhe_rsa, +suite_map_to_bin(#{key_exchange := dhe_rsa, cipher:= '3des_ede_cbc', mac := sha}) -> ?TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := dh_anon, +suite_map_to_bin(#{key_exchange := dh_anon, cipher:= rc4_128, mac := md5}) -> ?TLS_DH_anon_WITH_RC4_128_MD5; -suite(#{key_exchange := dh_anon, +suite_map_to_bin(#{key_exchange := dh_anon, cipher:= des_cbc, mac := sha}) -> ?TLS_DH_anon_WITH_DES_CBC_SHA; -suite(#{key_exchange := dh_anon, +suite_map_to_bin(#{key_exchange := dh_anon, cipher:= '3des_ede_cbc', mac := sha}) -> ?TLS_DH_anon_WITH_3DES_EDE_CBC_SHA; %%% TSL V1.1 AES suites -suite(#{key_exchange := rsa, +suite_map_to_bin(#{key_exchange := rsa, cipher := aes_128_cbc, mac := sha}) -> ?TLS_RSA_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := dhe_dss, +suite_map_to_bin(#{key_exchange := dhe_dss, cipher := aes_128_cbc, mac := sha}) -> ?TLS_DHE_DSS_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := dhe_rsa, +suite_map_to_bin(#{key_exchange := dhe_rsa, cipher := aes_128_cbc, mac := sha}) -> ?TLS_DHE_RSA_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := dh_anon, +suite_map_to_bin(#{key_exchange := dh_anon, cipher := aes_128_cbc, mac := sha}) -> ?TLS_DH_anon_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := rsa, +suite_map_to_bin(#{key_exchange := rsa, cipher := aes_256_cbc, mac := sha}) -> ?TLS_RSA_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := dhe_dss, +suite_map_to_bin(#{key_exchange := dhe_dss, cipher := aes_256_cbc, mac := sha}) -> ?TLS_DHE_DSS_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := dhe_rsa, +suite_map_to_bin(#{key_exchange := dhe_rsa, cipher := aes_256_cbc, mac := sha}) -> ?TLS_DHE_RSA_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := dh_anon, +suite_map_to_bin(#{key_exchange := dh_anon, cipher := aes_256_cbc, mac := sha}) -> ?TLS_DH_anon_WITH_AES_256_CBC_SHA; %% TLS v1.2 suites -suite(#{key_exchange := rsa, +suite_map_to_bin(#{key_exchange := rsa, cipher := aes_128_cbc, mac := sha256}) -> ?TLS_RSA_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := rsa, +suite_map_to_bin(#{key_exchange := rsa, cipher := aes_256_cbc, mac := sha256}) -> ?TLS_RSA_WITH_AES_256_CBC_SHA256; -suite(#{key_exchange := dhe_dss, +suite_map_to_bin(#{key_exchange := dhe_dss, cipher := aes_128_cbc, mac := sha256}) -> ?TLS_DHE_DSS_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := dhe_rsa, +suite_map_to_bin(#{key_exchange := dhe_rsa, cipher := aes_128_cbc, mac := sha256}) -> ?TLS_DHE_RSA_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := dhe_dss, +suite_map_to_bin(#{key_exchange := dhe_dss, cipher := aes_256_cbc, mac := sha256}) -> ?TLS_DHE_DSS_WITH_AES_256_CBC_SHA256; -suite(#{key_exchange := dhe_rsa, +suite_map_to_bin(#{key_exchange := dhe_rsa, cipher := aes_256_cbc, mac := sha256}) -> ?TLS_DHE_RSA_WITH_AES_256_CBC_SHA256; -suite(#{key_exchange := dh_anon, +suite_map_to_bin(#{key_exchange := dh_anon, cipher := aes_128_cbc, mac := sha256}) -> ?TLS_DH_anon_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := dh_anon, +suite_map_to_bin(#{key_exchange := dh_anon, cipher := aes_256_cbc, mac := sha256}) -> ?TLS_DH_anon_WITH_AES_256_CBC_SHA256; %%% PSK Cipher Suites RFC 4279 -suite(#{key_exchange := psk, +suite_map_to_bin(#{key_exchange := psk, cipher := rc4_128, mac := sha}) -> ?TLS_PSK_WITH_RC4_128_SHA; -suite(#{key_exchange := psk, +suite_map_to_bin(#{key_exchange := psk, cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_PSK_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := psk, +suite_map_to_bin(#{key_exchange := psk, cipher := aes_128_cbc, mac := sha}) -> ?TLS_PSK_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := psk, +suite_map_to_bin(#{key_exchange := psk, cipher := aes_256_cbc, mac := sha}) -> ?TLS_PSK_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := dhe_psk, +suite_map_to_bin(#{key_exchange := dhe_psk, cipher := rc4_128, mac := sha}) -> ?TLS_DHE_PSK_WITH_RC4_128_SHA; -suite(#{key_exchange := dhe_psk, +suite_map_to_bin(#{key_exchange := dhe_psk, cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := dhe_psk, +suite_map_to_bin(#{key_exchange := dhe_psk, cipher := aes_128_cbc, mac := sha}) -> ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := dhe_psk, +suite_map_to_bin(#{key_exchange := dhe_psk, cipher := aes_256_cbc, mac := sha}) -> ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := rsa_psk, +suite_map_to_bin(#{key_exchange := rsa_psk, cipher := rc4_128, mac := sha}) -> ?TLS_RSA_PSK_WITH_RC4_128_SHA; -suite(#{key_exchange := rsa_psk, +suite_map_to_bin(#{key_exchange := rsa_psk, cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := rsa_psk, +suite_map_to_bin(#{key_exchange := rsa_psk, cipher := aes_128_cbc, mac := sha}) -> ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := rsa_psk, +suite_map_to_bin(#{key_exchange := rsa_psk, cipher := aes_256_cbc, mac := sha}) -> ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA; %%% PSK NULL Cipher Suites RFC 4785 -suite(#{key_exchange := psk, +suite_map_to_bin(#{key_exchange := psk, cipher := null, mac := sha}) -> ?TLS_PSK_WITH_NULL_SHA; -suite(#{key_exchange := dhe_psk, +suite_map_to_bin(#{key_exchange := dhe_psk, cipher := null, mac := sha}) -> ?TLS_DHE_PSK_WITH_NULL_SHA; -suite(#{key_exchange := rsa_psk, +suite_map_to_bin(#{key_exchange := rsa_psk, cipher := null, mac := sha}) -> ?TLS_RSA_PSK_WITH_NULL_SHA; %%% TLS 1.2 PSK Cipher Suites RFC 5487 -suite(#{key_exchange := psk, +suite_map_to_bin(#{key_exchange := psk, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_PSK_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := psk, +suite_map_to_bin(#{key_exchange := psk, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_PSK_WITH_AES_256_GCM_SHA384; -suite(#{key_exchange := dhe_psk, +suite_map_to_bin(#{key_exchange := dhe_psk, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := dhe_psk, +suite_map_to_bin(#{key_exchange := dhe_psk, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384; -suite(#{key_exchange := rsa_psk, +suite_map_to_bin(#{key_exchange := rsa_psk, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := rsa_psk, +suite_map_to_bin(#{key_exchange := rsa_psk, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384; -suite(#{key_exchange := psk, +suite_map_to_bin(#{key_exchange := psk, cipher := aes_128_cbc, mac := sha256}) -> ?TLS_PSK_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := psk, +suite_map_to_bin(#{key_exchange := psk, cipher := aes_256_cbc, mac := sha384}) -> ?TLS_PSK_WITH_AES_256_CBC_SHA384; -suite(#{key_exchange := dhe_psk, +suite_map_to_bin(#{key_exchange := dhe_psk, cipher := aes_128_cbc, mac := sha256}) -> ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := dhe_psk, +suite_map_to_bin(#{key_exchange := dhe_psk, cipher := aes_256_cbc, mac := sha384}) -> ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384; -suite(#{key_exchange := rsa_psk, +suite_map_to_bin(#{key_exchange := rsa_psk, cipher := aes_128_cbc, mac := sha256}) -> ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := rsa_psk, +suite_map_to_bin(#{key_exchange := rsa_psk, cipher := aes_256_cbc, mac := sha384}) -> ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384; -suite(#{key_exchange := psk, +suite_map_to_bin(#{key_exchange := psk, cipher := null, mac := sha256}) -> ?TLS_PSK_WITH_NULL_SHA256; -suite(#{key_exchange := psk, +suite_map_to_bin(#{key_exchange := psk, cipher := null, mac := sha384}) -> ?TLS_PSK_WITH_NULL_SHA384; -suite(#{key_exchange := dhe_psk, +suite_map_to_bin(#{key_exchange := dhe_psk, cipher := null, mac := sha256}) -> ?TLS_DHE_PSK_WITH_NULL_SHA256; -suite(#{key_exchange := dhe_psk, +suite_map_to_bin(#{key_exchange := dhe_psk, cipher := null, mac := sha384}) -> ?TLS_DHE_PSK_WITH_NULL_SHA384; -suite(#{key_exchange := rsa_psk, +suite_map_to_bin(#{key_exchange := rsa_psk, cipher := null, mac := sha256}) -> ?TLS_RSA_PSK_WITH_NULL_SHA256; -suite(#{key_exchange := rsa_psk, +suite_map_to_bin(#{key_exchange := rsa_psk, cipher := null, mac := sha384}) -> ?TLS_RSA_PSK_WITH_NULL_SHA384; %%% ECDHE PSK Cipher Suites RFC 5489 -suite(#{key_exchange := ecdhe_psk, +suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := rc4_128, mac := sha}) -> ?TLS_ECDHE_PSK_WITH_RC4_128_SHA; -suite(#{key_exchange := ecdhe_psk, +suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher :='3des_ede_cbc', mac := sha}) -> ?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := ecdhe_psk, +suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := aes_128_cbc, mac := sha}) -> ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := ecdhe_psk, +suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := aes_256_cbc, mac := sha}) -> ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := ecdhe_psk, +suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := aes_128_cbc, mac := sha256}) -> ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := ecdhe_psk, +suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := aes_256_cbc, mac := sha384}) -> ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384; -suite(#{key_exchange := ecdhe_psk, +suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := null, mac := sha256}) -> ?TLS_ECDHE_PSK_WITH_NULL_SHA256; -suite(#{key_exchange := ecdhe_psk, +suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := null, mac := sha384}) -> ?TLS_ECDHE_PSK_WITH_NULL_SHA384; %%% ECDHE_PSK with AES-GCM and AES-CCM Cipher Suites, draft-ietf-tls-ecdhe-psk-aead-05 -suite(#{key_exchange := ecdhe_psk, +suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := aes_128_gcm, mac := null, prf := sha256}) -> ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := ecdhe_psk, +suite_map_to_bin(#{key_exchange := ecdhe_psk, cipher := aes_256_gcm, mac := null, prf := sha384}) -> ?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384; - %% suite(#{key_exchange := ecdhe_psk, - %% cipher := aes_128_ccm, - %% mac := null, - %% prf := sha256}) -> - %% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256; - %% suite(#{key_exchange := ecdhe_psk, - %% cipher := aes_256_ccm, - %% mac := null, - %% prf := sha256}) -> - %% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256; +suite_map_to_bin(#{key_exchange := ecdhe_psk, + cipher := aes_128_ccm_8, + mac := null, + prf := sha256}) -> + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256; +suite_map_to_bin(#{key_exchange := ecdhe_psk, + cipher := aes_128_ccm, + mac := null, + prf := sha256}) -> + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256; %%% SRP Cipher Suites RFC 5054 -suite(#{key_exchange := srp_anon, +suite_map_to_bin(#{key_exchange := srp_anon, cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := srp_rsa, +suite_map_to_bin(#{key_exchange := srp_rsa, cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := srp_dss, +suite_map_to_bin(#{key_exchange := srp_dss, cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := srp_anon, +suite_map_to_bin(#{key_exchange := srp_anon, cipher := aes_128_cbc, mac := sha}) -> ?TLS_SRP_SHA_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := srp_rsa, +suite_map_to_bin(#{key_exchange := srp_rsa, cipher := aes_128_cbc, mac := sha}) -> ?TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := srp_dss, +suite_map_to_bin(#{key_exchange := srp_dss, cipher := aes_128_cbc, mac := sha}) -> ?TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := srp_anon, +suite_map_to_bin(#{key_exchange := srp_anon, cipher := aes_256_cbc, mac := sha}) -> ?TLS_SRP_SHA_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := srp_rsa, +suite_map_to_bin(#{key_exchange := srp_rsa, cipher := aes_256_cbc, mac := sha}) -> ?TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := srp_dss, +suite_map_to_bin(#{key_exchange := srp_dss, cipher := aes_256_cbc, mac := sha}) -> ?TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA; %%% RFC 4492 EC TLS suites -suite(#{key_exchange := ecdh_ecdsa, +suite_map_to_bin(#{key_exchange := ecdh_ecdsa, cipher := null, mac := sha}) -> ?TLS_ECDH_ECDSA_WITH_NULL_SHA; -suite(#{key_exchange := ecdh_ecdsa, +suite_map_to_bin(#{key_exchange := ecdh_ecdsa, cipher := rc4_128, mac := sha}) -> ?TLS_ECDH_ECDSA_WITH_RC4_128_SHA; -suite(#{key_exchange := ecdh_ecdsa, +suite_map_to_bin(#{key_exchange := ecdh_ecdsa, cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := ecdh_ecdsa, +suite_map_to_bin(#{key_exchange := ecdh_ecdsa, cipher := aes_128_cbc, mac := sha}) -> ?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := ecdh_ecdsa, +suite_map_to_bin(#{key_exchange := ecdh_ecdsa, cipher := aes_256_cbc, mac := sha}) -> ?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := ecdhe_ecdsa, +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, cipher := null, mac := sha}) -> ?TLS_ECDHE_ECDSA_WITH_NULL_SHA; -suite(#{key_exchange := ecdhe_ecdsa, +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, cipher := rc4_128, mac := sha}) -> ?TLS_ECDHE_ECDSA_WITH_RC4_128_SHA; -suite(#{key_exchange := ecdhe_ecdsa, +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := ecdhe_ecdsa, +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, cipher := aes_128_cbc, mac := sha}) -> ?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := ecdhe_ecdsa, +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, cipher := aes_256_cbc, mac := sha}) -> ?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := ecdh_rsa, +suite_map_to_bin(#{key_exchange := ecdh_rsa, cipher := null, mac := sha}) -> ?TLS_ECDH_RSA_WITH_NULL_SHA; -suite(#{key_exchange := ecdh_rsa, +suite_map_to_bin(#{key_exchange := ecdh_rsa, cipher := rc4_128, mac := sha}) -> ?TLS_ECDH_RSA_WITH_RC4_128_SHA; -suite(#{key_exchange := ecdh_rsa, +suite_map_to_bin(#{key_exchange := ecdh_rsa, cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := ecdh_rsa, +suite_map_to_bin(#{key_exchange := ecdh_rsa, cipher := aes_128_cbc, mac := sha}) -> ?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := ecdh_rsa, +suite_map_to_bin(#{key_exchange := ecdh_rsa, cipher := aes_256_cbc, mac := sha}) -> ?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := ecdhe_rsa, +suite_map_to_bin(#{key_exchange := ecdhe_rsa, cipher := null, mac := sha}) -> ?TLS_ECDHE_RSA_WITH_NULL_SHA; -suite(#{key_exchange := ecdhe_rsa, +suite_map_to_bin(#{key_exchange := ecdhe_rsa, cipher := rc4_128, mac := sha}) -> ?TLS_ECDHE_RSA_WITH_RC4_128_SHA; -suite(#{key_exchange := ecdhe_rsa, +suite_map_to_bin(#{key_exchange := ecdhe_rsa, cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := ecdhe_rsa, +suite_map_to_bin(#{key_exchange := ecdhe_rsa, cipher := aes_128_cbc, mac := sha}) -> ?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := ecdhe_rsa, +suite_map_to_bin(#{key_exchange := ecdhe_rsa, cipher := aes_256_cbc, mac := sha}) -> ?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA; -suite(#{key_exchange := ecdh_anon, +suite_map_to_bin(#{key_exchange := ecdh_anon, cipher := null, mac := sha}) -> ?TLS_ECDH_anon_WITH_NULL_SHA; -suite(#{key_exchange := ecdh_anon, +suite_map_to_bin(#{key_exchange := ecdh_anon, cipher := rc4_128, mac := sha}) -> ?TLS_ECDH_anon_WITH_RC4_128_SHA; -suite(#{key_exchange := ecdh_anon, +suite_map_to_bin(#{key_exchange := ecdh_anon, cipher := '3des_ede_cbc', mac := sha}) -> ?TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA; -suite(#{key_exchange := ecdh_anon, +suite_map_to_bin(#{key_exchange := ecdh_anon, cipher := aes_128_cbc, mac := sha}) -> ?TLS_ECDH_anon_WITH_AES_128_CBC_SHA; -suite(#{key_exchange := ecdh_anon, +suite_map_to_bin(#{key_exchange := ecdh_anon, cipher := aes_256_cbc, mac := sha}) -> ?TLS_ECDH_anon_WITH_AES_256_CBC_SHA; %%% RFC 5289 EC TLS suites -suite(#{key_exchange := ecdhe_ecdsa, +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, cipher := aes_128_cbc, mac:= sha256, prf := sha256}) -> ?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := ecdhe_ecdsa, +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, cipher := aes_256_cbc, mac := sha384, prf := sha384}) -> ?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384; -suite(#{key_exchange := ecdh_ecdsa, +suite_map_to_bin(#{key_exchange := ecdh_ecdsa, cipher := aes_128_cbc, mac := sha256, prf := sha256}) -> ?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := ecdh_ecdsa, +suite_map_to_bin(#{key_exchange := ecdh_ecdsa, cipher := aes_256_cbc, mac := sha384, prf := sha384}) -> ?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384; -suite(#{key_exchange := ecdhe_rsa, +suite_map_to_bin(#{key_exchange := ecdhe_rsa, cipher := aes_128_cbc, mac := sha256, prf := sha256}) -> ?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := ecdhe_rsa, +suite_map_to_bin(#{key_exchange := ecdhe_rsa, cipher := aes_256_cbc, mac := sha384, prf := sha384}) -> ?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384; -suite(#{key_exchange := ecdh_rsa, +suite_map_to_bin(#{key_exchange := ecdh_rsa, cipher := aes_128_cbc, mac := sha256, prf := sha256}) -> ?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256; -suite(#{key_exchange := ecdh_rsa, +suite_map_to_bin(#{key_exchange := ecdh_rsa, cipher := aes_256_cbc, mac := sha384, prf := sha384}) -> ?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384; %% RFC 5288 AES-GCM Cipher Suites -suite(#{key_exchange := rsa, +suite_map_to_bin(#{key_exchange := rsa, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_RSA_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := rsa, +suite_map_to_bin(#{key_exchange := rsa, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_RSA_WITH_AES_256_GCM_SHA384; -suite(#{key_exchange := dhe_rsa, +suite_map_to_bin(#{key_exchange := dhe_rsa, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := dhe_rsa, +suite_map_to_bin(#{key_exchange := dhe_rsa, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384; -suite(#{key_exchange := dh_rsa, +suite_map_to_bin(#{key_exchange := dh_rsa, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_DH_RSA_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := dh_rsa, +suite_map_to_bin(#{key_exchange := dh_rsa, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_DH_RSA_WITH_AES_256_GCM_SHA384; -suite(#{key_exchange := dhe_dss, +suite_map_to_bin(#{key_exchange := dhe_dss, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := dhe_dss, +suite_map_to_bin(#{key_exchange := dhe_dss, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384; -suite(#{key_exchange := dh_dss, +suite_map_to_bin(#{key_exchange := dh_dss, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_DH_DSS_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := dh_dss, +suite_map_to_bin(#{key_exchange := dh_dss, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_DH_DSS_WITH_AES_256_GCM_SHA384; -suite(#{key_exchange := dh_anon, +suite_map_to_bin(#{key_exchange := dh_anon, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_DH_anon_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := dh_anon, +suite_map_to_bin(#{key_exchange := dh_anon, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_DH_anon_WITH_AES_256_GCM_SHA384; %% RFC 5289 ECC AES-GCM Cipher Suites -suite(#{key_exchange := ecdhe_ecdsa, +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := ecdhe_ecdsa, +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384; -suite(#{key_exchange := ecdh_ecdsa, +suite_map_to_bin(#{key_exchange := ecdh_ecdsa, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := ecdh_ecdsa, +suite_map_to_bin(#{key_exchange := ecdh_ecdsa, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384; -suite(#{key_exchange := ecdhe_rsa, +suite_map_to_bin(#{key_exchange := ecdhe_rsa, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := ecdhe_rsa, +suite_map_to_bin(#{key_exchange := ecdhe_rsa, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384; -suite(#{key_exchange := ecdh_rsa, +suite_map_to_bin(#{key_exchange := ecdh_rsa, cipher := aes_128_gcm, mac := aead, prf := sha256}) -> ?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256; -suite(#{key_exchange := ecdh_rsa, +suite_map_to_bin(#{key_exchange := ecdh_rsa, cipher := aes_256_gcm, mac := aead, prf := sha384}) -> ?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384; %% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites -suite(#{key_exchange := ecdhe_rsa, +suite_map_to_bin(#{key_exchange := ecdhe_rsa, cipher := chacha20_poly1305, mac := aead, prf := sha256}) -> ?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256; -suite(#{key_exchange := ecdhe_ecdsa, +suite_map_to_bin(#{key_exchange := ecdhe_ecdsa, cipher := chacha20_poly1305, mac := aead, prf := sha256}) -> ?TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256; -suite(#{key_exchange := dhe_rsa, +suite_map_to_bin(#{key_exchange := dhe_rsa, cipher := chacha20_poly1305, mac := aead, prf := sha256}) -> - ?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256. + ?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256; -%%-------------------------------------------------------------------- --spec openssl_suite(openssl_cipher_suite()) -> cipher_suite(). -%% -%% Description: Return TLS cipher suite definition. -%%-------------------------------------------------------------------- -%% translate constants <-> openssl-strings -openssl_suite("DHE-RSA-AES256-SHA256") -> - ?TLS_DHE_RSA_WITH_AES_256_CBC_SHA256; -openssl_suite("DHE-DSS-AES256-SHA256") -> - ?TLS_DHE_DSS_WITH_AES_256_CBC_SHA256; -openssl_suite("AES256-SHA256") -> - ?TLS_RSA_WITH_AES_256_CBC_SHA256; -openssl_suite("DHE-RSA-AES128-SHA256") -> - ?TLS_DHE_RSA_WITH_AES_128_CBC_SHA256; -openssl_suite("DHE-DSS-AES128-SHA256") -> - ?TLS_DHE_DSS_WITH_AES_128_CBC_SHA256; -openssl_suite("AES128-SHA256") -> - ?TLS_RSA_WITH_AES_128_CBC_SHA256; -openssl_suite("DHE-RSA-AES256-SHA") -> - ?TLS_DHE_RSA_WITH_AES_256_CBC_SHA; -openssl_suite("DHE-DSS-AES256-SHA") -> - ?TLS_DHE_DSS_WITH_AES_256_CBC_SHA; -openssl_suite("AES256-SHA") -> - ?TLS_RSA_WITH_AES_256_CBC_SHA; -openssl_suite("EDH-RSA-DES-CBC3-SHA") -> - ?TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA; -openssl_suite("EDH-DSS-DES-CBC3-SHA") -> - ?TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA; -openssl_suite("DES-CBC3-SHA") -> - ?TLS_RSA_WITH_3DES_EDE_CBC_SHA; -openssl_suite("DHE-RSA-AES128-SHA") -> - ?TLS_DHE_RSA_WITH_AES_128_CBC_SHA; -openssl_suite("DHE-DSS-AES128-SHA") -> - ?TLS_DHE_DSS_WITH_AES_128_CBC_SHA; -openssl_suite("AES128-SHA") -> - ?TLS_RSA_WITH_AES_128_CBC_SHA; -openssl_suite("RC4-SHA") -> - ?TLS_RSA_WITH_RC4_128_SHA; -openssl_suite("RC4-MD5") -> - ?TLS_RSA_WITH_RC4_128_MD5; -openssl_suite("EDH-RSA-DES-CBC-SHA") -> - ?TLS_DHE_RSA_WITH_DES_CBC_SHA; -openssl_suite("DES-CBC-SHA") -> - ?TLS_RSA_WITH_DES_CBC_SHA; +%% RFC 6655 - TLS-1.2 cipher suites +suite_map_to_bin(#{key_exchange := psk, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_128_CCM; +suite_map_to_bin(#{key_exchange := psk, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_256_CCM; +suite_map_to_bin(#{key_exchange := dhe_psk, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_PSK_WITH_AES_128_CCM; +suite_map_to_bin(#{key_exchange := dhe_psk, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_PSK_WITH_AES_256_CCM; +suite_map_to_bin(#{key_exchange := rsa, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_128_CCM; +suite_map_to_bin(#{key_exchange := rsa, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_256_CCM; +suite_map_to_bin(#{key_exchange := dhe_rsa, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_128_CCM; +suite_map_to_bin(#{key_exchange := dhe_rsa, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_256_CCM; -%%% SRP Cipher Suites RFC 5054 +suite_map_to_bin(#{key_exchange := psk, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_128_CCM_8; +suite_map_to_bin(#{key_exchange := psk, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_256_CCM_8; +suite_map_to_bin(#{key_exchange := dhe_psk, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_DHE_WITH_AES_128_CCM_8; +suite_map_to_bin(#{key_exchange := dhe_psk, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_DHE_WITH_AES_256_CCM_8; +suite_map_to_bin(#{key_exchange := rsa, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_128_CCM_8; +suite_map_to_bin(#{key_exchange := rsa, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_256_CCM_8; +suite_map_to_bin(#{key_exchange := dhe_rsa, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_128_CCM_8; +suite_map_to_bin(#{key_exchange := dhe_rsa, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_256_CCM_8; -openssl_suite("SRP-DSS-AES-256-CBC-SHA") -> - ?TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA; -openssl_suite("SRP-RSA-AES-256-CBC-SHA") -> - ?TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA; -openssl_suite("SRP-DSS-3DES-EDE-CBC-SHA") -> - ?TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA; -openssl_suite("SRP-RSA-3DES-EDE-CBC-SHA") -> - ?TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA; -openssl_suite("SRP-DSS-AES-128-CBC-SHA") -> - ?TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA; -openssl_suite("SRP-RSA-AES-128-CBC-SHA") -> - ?TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA; +%% TLS 1.3 Cipher Suites RFC8446 +suite_map_to_bin(#{key_exchange := any, + cipher := aes_128_gcm, + mac := aead, + prf := sha256}) -> + ?TLS_AES_128_GCM_SHA256; +suite_map_to_bin(#{key_exchange := any, + cipher := aes_256_gcm, + mac := aead, + prf := sha384}) -> + ?TLS_AES_256_GCM_SHA384; +suite_map_to_bin(#{key_exchange := any, + cipher := chacha20_poly1305, + mac := aead, + prf := sha256}) -> + ?TLS_CHACHA20_POLY1305_SHA256; +suite_map_to_bin(#{key_exchange := any, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_AES_128_CCM_SHA256. +%% suite_map_to_bin(#{key_exchange := any, +%% cipher := aes_128_ccm_8, +%% mac := aead, +%% prf := sha256}) -> +%% ?TLS_AES_128_CCM_8_SHA256. -%% RFC 4492 EC TLS suites -openssl_suite("ECDH-ECDSA-RC4-SHA") -> - ?TLS_ECDH_ECDSA_WITH_RC4_128_SHA; -openssl_suite("ECDH-ECDSA-DES-CBC3-SHA") -> - ?TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA; -openssl_suite("ECDH-ECDSA-AES128-SHA") -> - ?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA; -openssl_suite("ECDH-ECDSA-AES256-SHA") -> - ?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA; -openssl_suite("ECDHE-ECDSA-RC4-SHA") -> - ?TLS_ECDHE_ECDSA_WITH_RC4_128_SHA; -openssl_suite("ECDHE-ECDSA-DES-CBC3-SHA") -> - ?TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA; -openssl_suite("ECDHE-ECDSA-AES128-SHA") -> - ?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA; -openssl_suite("ECDHE-ECDSA-AES256-SHA") -> - ?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA; +tls_1_3_suite_str_to_map(CipherStr) -> + {Cipher, Mac, Prf} = cipher_str_to_algs(CipherStr, ""), + #{key_exchange => any, + mac => Mac, + cipher => Cipher, + prf => Prf + }. -openssl_suite("ECDHE-RSA-RC4-SHA") -> - ?TLS_ECDHE_RSA_WITH_RC4_128_SHA; -openssl_suite("ECDHE-RSA-DES-CBC3-SHA") -> - ?TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA; -openssl_suite("ECDHE-RSA-AES128-SHA") -> - ?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA; -openssl_suite("ECDHE-RSA-AES256-SHA") -> - ?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA; +pre_tls_1_3_suite_str_to_map(KexStr, Rest) -> + Kex = algo_str_to_atom(KexStr), + [CipherStr, AlgStr] = string:split(Rest, "_", trailing), + {Cipher, Mac, Prf} = cipher_str_to_algs(CipherStr, AlgStr), + #{key_exchange => Kex, + mac => Mac, + cipher => Cipher, + prf => Prf + }. + +cipher_str_to_algs(CipherStr, "CCM"= End) -> %% PRE TLS 1.3 + Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), + {Cipher, aead, sha256}; +cipher_str_to_algs(CipherStr, "8" = End) -> %% PRE TLS 1.3 + Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), + {Cipher, aead, sha256}; +cipher_str_to_algs(CipherStr, "CHACHA20_POLY1305" = End) -> %% PRE TLS 1.3 + Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), + {Cipher, aead, sha256}; +cipher_str_to_algs(CipherStr0, "") -> %% TLS 1.3 + [CipherStr, AlgStr] = string:split(CipherStr0, "_", trailing), + Hash = algo_str_to_atom(AlgStr), + Cipher = algo_str_to_atom(CipherStr), + {Cipher, aead, Hash}; +cipher_str_to_algs(CipherStr, HashStr) -> %% PRE TLS 1.3 + Hash = algo_str_to_atom(HashStr), + Cipher = algo_str_to_atom(CipherStr), + case is_aead_cipher(CipherStr) of + true -> + {Cipher, aead, Hash}; + false -> + {Cipher, Hash, default_prf} + end. -openssl_suite("ECDH-RSA-RC4-SHA") -> - ?TLS_ECDH_RSA_WITH_RC4_128_SHA; -openssl_suite("ECDH-RSA-DES-CBC3-SHA") -> - ?TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA; -openssl_suite("ECDH-RSA-AES128-SHA") -> - ?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA; -openssl_suite("ECDH-RSA-AES256-SHA") -> - ?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA; +%% PRE TLS 1.3 +is_aead_cipher("CHACHA20_POLY1305") -> + true; +is_aead_cipher(CipherStr) -> + [_, Rest] = string:split(CipherStr, "_", trailing), + (Rest == "GCM") orelse (Rest == "CCM") orelse (Rest == "8"). -%% RFC 5289 EC TLS suites -openssl_suite("ECDHE-ECDSA-AES128-SHA256") -> - ?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256; -openssl_suite("ECDHE-ECDSA-AES256-SHA384") -> - ?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384; -openssl_suite("ECDH-ECDSA-AES128-SHA256") -> - ?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256; -openssl_suite("ECDH-ECDSA-AES256-SHA384") -> - ?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384; -openssl_suite("ECDHE-RSA-AES128-SHA256") -> - ?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256; -openssl_suite("ECDHE-RSA-AES256-SHA384") -> - ?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384; -openssl_suite("ECDH-RSA-AES128-SHA256") -> - ?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256; -openssl_suite("ECDH-RSA-AES256-SHA384") -> - ?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384; +openssl_is_aead_cipher("CHACHA20-POLY1305") -> + true; +openssl_is_aead_cipher(CipherStr) -> + case string:split(CipherStr, "-", trailing) of + [_, Rest] -> + (Rest == "GCM") orelse (Rest == "CCM") orelse (Rest == "8"); + [_] -> + false + end. -%% RFC 5288 AES-GCM Cipher Suites -openssl_suite("AES128-GCM-SHA256") -> - ?TLS_RSA_WITH_AES_128_GCM_SHA256; -openssl_suite("AES256-GCM-SHA384") -> - ?TLS_RSA_WITH_AES_256_GCM_SHA384; -openssl_suite("DHE-RSA-AES128-GCM-SHA256") -> - ?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256; -openssl_suite("DHE-RSA-AES256-GCM-SHA384") -> - ?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384; -openssl_suite("DH-RSA-AES128-GCM-SHA256") -> - ?TLS_DH_RSA_WITH_AES_128_GCM_SHA256; -openssl_suite("DH-RSA-AES256-GCM-SHA384") -> - ?TLS_DH_RSA_WITH_AES_256_GCM_SHA384; -openssl_suite("DHE-DSS-AES128-GCM-SHA256") -> - ?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256; -openssl_suite("DHE-DSS-AES256-GCM-SHA384") -> - ?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384; -openssl_suite("DH-DSS-AES128-GCM-SHA256") -> - ?TLS_DH_DSS_WITH_AES_128_GCM_SHA256; -openssl_suite("DH-DSS-AES256-GCM-SHA384") -> - ?TLS_DH_DSS_WITH_AES_256_GCM_SHA384; +algo_str_to_atom(AlgoStr) -> + erlang:list_to_existing_atom(string:to_lower(AlgoStr)). -%% RFC 5289 ECC AES-GCM Cipher Suites -openssl_suite("ECDHE-ECDSA-AES128-GCM-SHA256") -> - ?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256; -openssl_suite("ECDHE-ECDSA-AES256-GCM-SHA384") -> - ?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384; -openssl_suite("ECDH-ECDSA-AES128-GCM-SHA256") -> - ?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256; -openssl_suite("ECDH-ECDSA-AES256-GCM-SHA384") -> - ?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384; -openssl_suite("ECDHE-RSA-AES128-GCM-SHA256") -> - ?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256; -openssl_suite("ECDHE-RSA-AES256-GCM-SHA384") -> - ?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384; -openssl_suite("ECDH-RSA-AES128-GCM-SHA256") -> - ?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256; -openssl_suite("ECDH-RSA-AES256-GCM-SHA384") -> - ?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384. -%%-------------------------------------------------------------------- --spec openssl_suite_name(cipher_suite()) -> openssl_cipher_suite() | internal_erl_cipher_suite(). -%% -%% Description: Return openssl cipher suite name if possible -%%------------------------------------------------------------------- -openssl_suite_name(?TLS_DHE_RSA_WITH_AES_256_CBC_SHA) -> - "DHE-RSA-AES256-SHA"; -openssl_suite_name(?TLS_DHE_DSS_WITH_AES_256_CBC_SHA) -> - "DHE-DSS-AES256-SHA"; -openssl_suite_name(?TLS_RSA_WITH_AES_256_CBC_SHA) -> - "AES256-SHA"; -openssl_suite_name(?TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA) -> - "EDH-RSA-DES-CBC3-SHA"; -openssl_suite_name(?TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA) -> - "EDH-DSS-DES-CBC3-SHA"; -openssl_suite_name(?TLS_RSA_WITH_3DES_EDE_CBC_SHA) -> - "DES-CBC3-SHA"; -openssl_suite_name( ?TLS_DHE_RSA_WITH_AES_128_CBC_SHA) -> - "DHE-RSA-AES128-SHA"; -openssl_suite_name(?TLS_DHE_DSS_WITH_AES_128_CBC_SHA) -> - "DHE-DSS-AES128-SHA"; -openssl_suite_name(?TLS_RSA_WITH_AES_128_CBC_SHA) -> - "AES128-SHA"; -openssl_suite_name(?TLS_RSA_WITH_RC4_128_SHA) -> - "RC4-SHA"; -openssl_suite_name(?TLS_RSA_WITH_RC4_128_MD5) -> - "RC4-MD5"; -openssl_suite_name(?TLS_DHE_RSA_WITH_DES_CBC_SHA) -> - "EDH-RSA-DES-CBC-SHA"; -openssl_suite_name(?TLS_RSA_WITH_DES_CBC_SHA) -> - "DES-CBC-SHA"; -openssl_suite_name(?TLS_RSA_WITH_NULL_SHA256) -> - "NULL-SHA256"; -openssl_suite_name(?TLS_RSA_WITH_AES_128_CBC_SHA256) -> - "AES128-SHA256"; -openssl_suite_name(?TLS_RSA_WITH_AES_256_CBC_SHA256) -> - "AES256-SHA256"; -openssl_suite_name(?TLS_DH_DSS_WITH_AES_128_CBC_SHA256) -> - "DH-DSS-AES128-SHA256"; -openssl_suite_name(?TLS_DH_RSA_WITH_AES_128_CBC_SHA256) -> - "DH-RSA-AES128-SHA256"; -openssl_suite_name(?TLS_DHE_DSS_WITH_AES_128_CBC_SHA256) -> - "DHE-DSS-AES128-SHA256"; -openssl_suite_name(?TLS_DHE_RSA_WITH_AES_128_CBC_SHA256) -> - "DHE-RSA-AES128-SHA256"; -openssl_suite_name(?TLS_DH_DSS_WITH_AES_256_CBC_SHA256) -> - "DH-DSS-AES256-SHA256"; -openssl_suite_name(?TLS_DH_RSA_WITH_AES_256_CBC_SHA256) -> - "DH-RSA-AES256-SHA256"; -openssl_suite_name(?TLS_DHE_DSS_WITH_AES_256_CBC_SHA256) -> - "DHE-DSS-AES256-SHA256"; -openssl_suite_name(?TLS_DHE_RSA_WITH_AES_256_CBC_SHA256) -> - "DHE-RSA-AES256-SHA256"; +openssl_cipher_name(Kex, "AES_128_CBC" ++ _ = CipherStr) when Kex == rsa; + Kex == dhe_rsa; + Kex == ecdhe_rsa; + Kex == ecdhe_ecdsa -> + openssl_name_concat(CipherStr); +openssl_cipher_name(Kex, "AES_256_CBC" ++ _ = CipherStr) when Kex == rsa; + Kex == dhe_rsa; + Kex == ecdhe_rsa; + Kex == ecdhe_ecdsa -> + openssl_name_concat(CipherStr); +openssl_cipher_name(Kex, "AES_128_CBC" ++ _ = CipherStr) when Kex == srp; + Kex == srp_rsa -> + lists:append(string:replace(CipherStr, "_", "-", all)); +openssl_cipher_name(Kex, "AES_256_CBC" ++ _ = CipherStr) when Kex == srp; + Kex == srp_rsa -> + lists:append(string:replace(CipherStr, "_", "-", all)); +openssl_cipher_name(_, "AES_128_CBC" ++ _ = CipherStr) -> + openssl_name_concat(CipherStr) ++ "-CBC"; +openssl_cipher_name(_, "AES_256_CBC" ++ _ = CipherStr) -> + openssl_name_concat(CipherStr) ++ "-CBC"; +openssl_cipher_name(_, "AES_128_GCM" ++ _ = CipherStr) -> + openssl_name_concat(CipherStr) ++ "-GCM"; +openssl_cipher_name(_, "AES_256_GCM" ++ _ = CipherStr) -> + openssl_name_concat(CipherStr) ++ "-GCM"; +openssl_cipher_name(_, "RC4" ++ _) -> + "RC4"; +openssl_cipher_name(_, CipherStr) -> + lists:append(string:replace(CipherStr, "_", "-", all)). -%%% PSK Cipher Suites RFC 4279 -openssl_suite_name(?TLS_PSK_WITH_AES_256_CBC_SHA) -> - "PSK-AES256-CBC-SHA"; -openssl_suite_name(?TLS_PSK_WITH_3DES_EDE_CBC_SHA) -> - "PSK-3DES-EDE-CBC-SHA"; -openssl_suite_name(?TLS_PSK_WITH_AES_128_CBC_SHA) -> - "PSK-AES128-CBC-SHA"; -openssl_suite_name(?TLS_PSK_WITH_RC4_128_SHA) -> - "PSK-RC4-SHA"; +openssl_suite_start(Kex) -> + case openssl_kex_name(Kex) of + "" -> + ""; + Name -> + Name ++ "-" + end. -%%% SRP Cipher Suites RFC 5054 +openssl_kex_name("RSA") -> + ""; +openssl_kex_name(Kex) -> + lists:append(string:replace(Kex, "_", "-", all)). -openssl_suite_name(?TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA) -> - "SRP-RSA-3DES-EDE-CBC-SHA"; -openssl_suite_name(?TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA) -> - "SRP-DSS-3DES-EDE-CBC-SHA"; -openssl_suite_name(?TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA) -> - "SRP-RSA-AES-128-CBC-SHA"; -openssl_suite_name(?TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA) -> - "SRP-DSS-AES-128-CBC-SHA"; -openssl_suite_name(?TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA) -> - "SRP-RSA-AES-256-CBC-SHA"; -openssl_suite_name(?TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA) -> - "SRP-DSS-AES-256-CBC-SHA"; +kex_name_from_openssl(Kex) -> + lists:append(string:replace(Kex, "-", "_", all)). -%% RFC 4492 EC TLS suites -openssl_suite_name(?TLS_ECDH_ECDSA_WITH_RC4_128_SHA) -> - "ECDH-ECDSA-RC4-SHA"; -openssl_suite_name(?TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA) -> - "ECDH-ECDSA-DES-CBC3-SHA"; -openssl_suite_name(?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA) -> - "ECDH-ECDSA-AES128-SHA"; -openssl_suite_name(?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA) -> - "ECDH-ECDSA-AES256-SHA"; +cipher_name_from_openssl("AES128") -> + "AES_128_CBC"; +cipher_name_from_openssl("AES256") -> + "AES_256_CBC"; +cipher_name_from_openssl("AES128-CBC") -> + "AES_128_CBC"; +cipher_name_from_openssl("AES256-CBC") -> + "AES_256_CBC"; +cipher_name_from_openssl("AES-128-CBC") -> + "AES_128_CBC"; +cipher_name_from_openssl("AES-256-CBC") -> + "AES_256_CBC"; +cipher_name_from_openssl("AES128-GCM") -> + "AES_128_GCM"; +cipher_name_from_openssl("AES256-GCM") -> + "AES_256_GCM"; +cipher_name_from_openssl("RC4") -> + "RC4_128"; +cipher_name_from_openssl(Str) -> + Str. -openssl_suite_name(?TLS_ECDHE_ECDSA_WITH_RC4_128_SHA) -> - "ECDHE-ECDSA-RC4-SHA"; -openssl_suite_name(?TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA) -> - "ECDHE-ECDSA-DES-CBC3-SHA"; -openssl_suite_name(?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA) -> - "ECDHE-ECDSA-AES128-SHA"; -openssl_suite_name(?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA) -> - "ECDHE-ECDSA-AES256-SHA"; +openssl_name_concat(Str0) -> + [Str, _] = string:split(Str0, "_", trailing), + [Part1, Part2] = string:split(Str, "_", trailing), + Part1 ++ Part2. -openssl_suite_name(?TLS_ECDH_RSA_WITH_RC4_128_SHA) -> - "ECDH-RSA-RC4-SHA"; -openssl_suite_name(?TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA) -> - "ECDH-RSA-DES-CBC3-SHA"; -openssl_suite_name(?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA) -> - "ECDH-RSA-AES128-SHA"; -openssl_suite_name(?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA) -> - "ECDH-RSA-AES256-SHA"; -openssl_suite_name(?TLS_ECDHE_RSA_WITH_RC4_128_SHA) -> - "ECDHE-RSA-RC4-SHA"; -openssl_suite_name(?TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA) -> - "ECDHE-RSA-DES-CBC3-SHA"; -openssl_suite_name(?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA) -> - "ECDHE-RSA-AES128-SHA"; -openssl_suite_name(?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA) -> - "ECDHE-RSA-AES256-SHA"; +suite_openssl_str_to_map(Kex0, Rest) -> + Kex = algo_str_to_atom(kex_name_from_openssl(Kex0)), + [CipherStr, AlgStr] = string:split(Rest, "-", trailing), + {Cipher, Mac, Prf} = openssl_cipher_str_to_algs(CipherStr, AlgStr), + #{key_exchange => Kex, + mac => Mac, + cipher => Cipher, + prf => Prf + }. -%% RFC 5289 EC TLS suites -openssl_suite_name(?TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256) -> - "ECDHE-ECDSA-AES128-SHA256"; -openssl_suite_name(?TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384) -> - "ECDHE-ECDSA-AES256-SHA384"; -openssl_suite_name(?TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256) -> - "ECDH-ECDSA-AES128-SHA256"; -openssl_suite_name(?TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384) -> - "ECDH-ECDSA-AES256-SHA384"; -openssl_suite_name(?TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256) -> - "ECDHE-RSA-AES128-SHA256"; -openssl_suite_name(?TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) -> - "ECDHE-RSA-AES256-SHA384"; -openssl_suite_name(?TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256) -> - "ECDH-RSA-AES128-SHA256"; -openssl_suite_name(?TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384) -> - "ECDH-RSA-AES256-SHA384"; +%% Does only need own implementation PRE TLS 1.3 +openssl_cipher_str_to_algs(CipherStr, "CCM"= End) -> + Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), + {Cipher, aead, sha256}; +openssl_cipher_str_to_algs(CipherStr, "8" = End) -> + Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), + {Cipher, aead, sha256}; +openssl_cipher_str_to_algs(CipherStr, "POLY1305" = End) -> + Cipher = algo_str_to_atom(CipherStr ++ "_" ++ End), + {Cipher, aead, sha256}; +openssl_cipher_str_to_algs(CipherStr, HashStr) -> + Hash = algo_str_to_atom(HashStr), + Cipher = algo_str_to_atom(cipher_name_from_openssl(CipherStr)), + case openssl_is_aead_cipher(CipherStr) of + true -> + {Cipher, aead, Hash}; + false -> + {Cipher, Hash, openssl_prf(Hash)} + end. + +openssl_prf(sha256)-> + sha256; +openssl_prf(sha384) -> + sha384; +openssl_prf(_) -> + default_prf. -%% RFC 5288 AES-GCM Cipher Suites -openssl_suite_name(?TLS_RSA_WITH_AES_128_GCM_SHA256) -> - "AES128-GCM-SHA256"; -openssl_suite_name(?TLS_RSA_WITH_AES_256_GCM_SHA384) -> - "AES256-GCM-SHA384"; -openssl_suite_name(?TLS_DHE_RSA_WITH_AES_128_GCM_SHA256) -> - "DHE-RSA-AES128-GCM-SHA256"; -openssl_suite_name(?TLS_DHE_RSA_WITH_AES_256_GCM_SHA384) -> - "DHE-RSA-AES256-GCM-SHA384"; -openssl_suite_name(?TLS_DH_RSA_WITH_AES_128_GCM_SHA256) -> - "DH-RSA-AES128-GCM-SHA256"; -openssl_suite_name(?TLS_DH_RSA_WITH_AES_256_GCM_SHA384) -> - "DH-RSA-AES256-GCM-SHA384"; -openssl_suite_name(?TLS_DHE_DSS_WITH_AES_128_GCM_SHA256) -> - "DHE-DSS-AES128-GCM-SHA256"; -openssl_suite_name(?TLS_DHE_DSS_WITH_AES_256_GCM_SHA384) -> - "DHE-DSS-AES256-GCM-SHA384"; -openssl_suite_name(?TLS_DH_DSS_WITH_AES_128_GCM_SHA256) -> - "DH-DSS-AES128-GCM-SHA256"; -openssl_suite_name(?TLS_DH_DSS_WITH_AES_256_GCM_SHA384) -> - "DH-DSS-AES256-GCM-SHA384"; -%% RFC 5289 ECC AES-GCM Cipher Suites -openssl_suite_name(?TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256) -> - "ECDHE-ECDSA-AES128-GCM-SHA256"; -openssl_suite_name(?TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) -> - "ECDHE-ECDSA-AES256-GCM-SHA384"; -openssl_suite_name(?TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256) -> - "ECDH-ECDSA-AES128-GCM-SHA256"; -openssl_suite_name(?TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384) -> - "ECDH-ECDSA-AES256-GCM-SHA384"; -openssl_suite_name(?TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) -> - "ECDHE-RSA-AES128-GCM-SHA256"; -openssl_suite_name(?TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) -> - "ECDHE-RSA-AES256-GCM-SHA384"; -openssl_suite_name(?TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256) -> - "ECDH-RSA-AES128-GCM-SHA256"; -openssl_suite_name(?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384) -> - "ECDH-RSA-AES256-GCM-SHA384"; -%% No oppenssl name -openssl_suite_name(Cipher) -> - suite_definition(Cipher). diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index fbbe0a49c8..a5f754d2e3 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -35,10 +35,11 @@ -include("ssl_internal.hrl"). -include("ssl_srp.hrl"). -include_lib("public_key/include/public_key.hrl"). +-include_lib("kernel/include/logger.hrl"). %% Setup --export([connect/8, handshake/7, handshake/2, handshake/3, +-export([connect/8, handshake/7, handshake/2, handshake/3, handle_common_event/5, handshake_continue/3, handshake_cancel/1, socket_control/4, socket_control/5]). @@ -59,7 +60,7 @@ %% Help functions for tls|dtls_connection.erl -export([handle_session/7, ssl_config/3, - prepare_connection/2, hibernate_after/3, map_extensions/1]). + prepare_connection/2, hibernate_after/3]). %% General gen_statem state functions with extra callback argument %% to determine if it is an SSL/TLS or DTLS gen_statem machine @@ -337,7 +338,9 @@ handle_own_alert(Alert, _, StateName, ignore end, try %% Try to tell the local user - log_alert(SslOpts#ssl_options.log_alert, Role, Connection:protocol_name(), StateName, Alert#alert{role = Role}), + log_alert(SslOpts#ssl_options.log_level, Role, + Connection:protocol_name(), StateName, + Alert#alert{role = Role}), handle_normal_shutdown(Alert,StateName, State) catch _:_ -> ok @@ -379,10 +382,11 @@ handle_alert(#alert{level = ?FATAL} = Alert, StateName, session = Session, socket_options = Opts} = State) -> invalidate_session(Role, Host, Port, Session), - log_alert(SslOpts#ssl_options.log_alert, Role, Connection:protocol_name(), + log_alert(SslOpts#ssl_options.log_level, Role, Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}), Pids = Connection:pids(State), - alert_user(Pids, Transport, Tracker, Socket, StateName, Opts, Pid, From, Alert, Role, Connection), + alert_user(Pids, Transport, Tracker, Socket, StateName, Opts, Pid, From, Alert, + opposite_role(Role), Connection), {stop, {shutdown, normal}, State}; handle_alert(#alert{level = ?WARNING, description = ?CLOSE_NOTIFY} = Alert, @@ -397,7 +401,7 @@ handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, protocol_cb = Connection}, handshake_env = #handshake_env{renegotiation = {true, internal}}, ssl_options = SslOpts} = State) -> - log_alert(SslOpts#ssl_options.log_alert, Role, + log_alert(SslOpts#ssl_options.log_level, Role, Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}), handle_normal_shutdown(Alert, StateName, State), {stop,{shutdown, peer_close}, State}; @@ -408,7 +412,7 @@ handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, handshake_env = #handshake_env{renegotiation = {true, From}} = HsEnv, ssl_options = SslOpts } = State0) -> - log_alert(SslOpts#ssl_options.log_alert, Role, + log_alert(SslOpts#ssl_options.log_level, Role, Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}), gen_statem:reply(From, {error, renegotiation_rejected}), State = Connection:reinit_handshake_data(State0), @@ -419,8 +423,8 @@ handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, protocol_cb = Connection}, handshake_env = #handshake_env{renegotiation = {true, From}} = HsEnv, ssl_options = SslOpts - } = State0) -> - log_alert(SslOpts#ssl_options.log_alert, Role, + } = State0) -> + log_alert(SslOpts#ssl_options.log_level, Role, Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}), gen_statem:reply(From, {error, renegotiation_rejected}), %% Go back to connection! @@ -432,8 +436,9 @@ handle_alert(#alert{level = ?WARNING} = Alert, StateName, #state{static_env = #static_env{role = Role, protocol_cb = Connection}, ssl_options = SslOpts} = State) -> - log_alert(SslOpts#ssl_options.log_alert, Role, - Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}), + log_alert(SslOpts#ssl_options.log_level, Role, + Connection:protocol_name(), StateName, + Alert#alert{role = opposite_role(Role)}), Connection:next_event(StateName, no_record, State). %%==================================================================== @@ -442,8 +447,7 @@ handle_alert(#alert{level = ?WARNING} = Alert, StateName, passive_receive(State0 = #state{user_data_buffer = {_,BufferSize,_}}, StateName, Connection, StartTimerAction) -> case BufferSize of 0 -> - {Record, State} = Connection:next_record(State0), - Connection:next_event(StateName, Record, State, StartTimerAction); + Connection:next_event(StateName, no_record, State0, StartTimerAction); _ -> case read_application_data(<<>>, State0) of {stop, _, _} = ShutdownError -> @@ -699,7 +703,7 @@ handle_session(#server_hello{cipher_suite = CipherSuite, handshake_env = #handshake_env{negotiated_protocol = CurrentProtocol} = HsEnv, connection_env = #connection_env{negotiated_version = ReqVersion} = CEnv} = State0) -> #{key_exchange := KeyAlgorithm} = - ssl_cipher_format:suite_definition(CipherSuite), + ssl_cipher_format:suite_bin_to_map(CipherSuite), PremasterSecret = make_premaster_secret(ReqVersion, KeyAlgorithm), @@ -1022,7 +1026,8 @@ certify(internal, #certificate_request{} = CertRequest, connection_env = #connection_env{negotiated_version = Version}, session = #session{own_certificate = Cert}, ssl_options = #ssl_options{signature_algs = SupportedHashSigns}} = State, Connection) -> - case ssl_handshake:select_hashsign(CertRequest, Cert, SupportedHashSigns, ssl:tls_version(Version)) of + case ssl_handshake:select_hashsign(CertRequest, Cert, + SupportedHashSigns, ssl:tls_version(Version)) of #alert {} = Alert -> handle_own_alert(Alert, Version, ?FUNCTION_NAME, State); NegotiatedHashSign -> @@ -1188,10 +1193,8 @@ cipher(internal, #finished{verify_data = Data} = Finished, cipher(internal, #next_protocol{selected_protocol = SelectedProtocol}, #state{static_env = #static_env{role = server}, handshake_env = #handshake_env{expecting_finished = true, - expecting_next_protocol_negotiation = true} = HsEnv} = State0, Connection) -> - {Record, State} = - Connection:next_record(State0), - Connection:next_event(?FUNCTION_NAME, Record, + expecting_next_protocol_negotiation = true} = HsEnv} = State, Connection) -> + Connection:next_event(?FUNCTION_NAME, no_record, State#state{handshake_env = HsEnv#handshake_env{negotiated_protocol = SelectedProtocol, expecting_next_protocol_negotiation = false}}); cipher(internal, #change_cipher_spec{type = <<1>>}, #state{handshake_env = HsEnv, connection_states = ConnectionStates0} = @@ -1313,10 +1316,10 @@ handle_common_event({timeout, handshake}, close, _StateName, #state{start_or_rec handle_common_event({timeout, recv}, timeout, StateName, #state{start_or_recv_from = RecvFrom} = State, _) -> {next_state, StateName, State#state{start_or_recv_from = undefined, bytes_to_read = undefined}, [{reply, RecvFrom, {error, timeout}}]}; -handle_common_event(_Type, Msg, StateName, #state{connection_env = +handle_common_event(Type, Msg, StateName, #state{connection_env = #connection_env{negotiated_version = Version}} = State, _) -> - Alert = ?ALERT_REC(?FATAL,?UNEXPECTED_MESSAGE, {unexpected_msg, Msg}), + Alert = ?ALERT_REC(?FATAL,?UNEXPECTED_MESSAGE, {unexpected_msg, {Type,Msg}}), handle_own_alert(Alert, Version, StateName, State). handle_call({application_data, _Data}, _, _, _, _) -> @@ -1448,7 +1451,7 @@ handle_info({ErrorTag, Socket, econnaborted}, StateName, handle_info({ErrorTag, Socket, Reason}, StateName, #state{static_env = #static_env{socket = Socket, error_tag = ErrorTag}} = State) -> Report = io_lib:format("SSL: Socket error: ~p ~n", [Reason]), - error_logger:error_report(Report), + ?LOG_ERROR(Report), handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), {stop, {shutdown,normal}, State}; @@ -1478,7 +1481,7 @@ handle_info(allow_renegotiate, StateName, #state{handshake_env = HsEnv} = State) handle_info(Msg, StateName, #state{static_env = #static_env{socket = Socket, error_tag = Tag}} = State) -> Report = io_lib:format("SSL: Got unexpected info: ~p ~n", [{Msg, Tag, Socket}]), - error_logger:info_report(Report), + ?LOG_NOTICE(Report), {next_state, StateName, State}. %%==================================================================== @@ -1570,7 +1573,7 @@ connection_info(#state{static_env = #static_env{protocol_cb = Connection}, connection_env = #connection_env{negotiated_version = {_,_} = Version}, ssl_options = Opts}) -> RecordCB = record_cb(Connection), - CipherSuiteDef = #{key_exchange := KexAlg} = ssl_cipher_format:suite_definition(CipherSuite), + CipherSuiteDef = #{key_exchange := KexAlg} = ssl_cipher_format:suite_bin_to_map(CipherSuite), IsNamedCurveSuite = lists:member(KexAlg, [ecdh_ecdsa, ecdhe_ecdsa, ecdh_rsa, ecdhe_rsa, ecdh_anon]), CurveInfo = case ECCCurve of @@ -1581,7 +1584,7 @@ connection_info(#state{static_env = #static_env{protocol_cb = Connection}, end, [{protocol, RecordCB:protocol_version(Version)}, {session_id, SessionId}, - {cipher_suite, ssl_cipher_format:erl_suite_definition(CipherSuiteDef)}, + {cipher_suite, ssl_cipher_format:suite_legacy(CipherSuiteDef)}, {selected_cipher_suite, CipherSuiteDef}, {sni_hostname, SNIHostname} | CurveInfo] ++ ssl_options_list(Opts). @@ -1593,18 +1596,23 @@ security_info(#state{connection_states = ConnectionStates}) -> ssl_record:current_connection_state(ConnectionStates, read), [{client_random, ClientRand}, {server_random, ServerRand}, {master_secret, MasterSecret}]. -do_server_hello(Type, #hello_extensions{next_protocol_negotiation = NextProtocols} = +do_server_hello(Type, #{next_protocol_negotiation := NextProtocols} = ServerHelloExt, #state{connection_env = #connection_env{negotiated_version = Version}, handshake_env = HsEnv, session = #session{session_id = SessId}, - connection_states = ConnectionStates0} + connection_states = ConnectionStates0, + ssl_options = #ssl_options{versions = [HighestVersion|_]}} = State0, Connection) when is_atom(Type) -> - + %% TLS 1.3 - Section 4.1.3 + %% Override server random values for TLS 1.3 downgrade protection mechanism. + ConnectionStates1 = update_server_random(ConnectionStates0, Version, HighestVersion), + State1 = State0#state{connection_states = ConnectionStates1}, ServerHello = - ssl_handshake:server_hello(SessId, ssl:tls_version(Version), ConnectionStates0, ServerHelloExt), + ssl_handshake:server_hello(SessId, ssl:tls_version(Version), + ConnectionStates1, ServerHelloExt), State = server_hello(ServerHello, - State0#state{handshake_env = HsEnv#handshake_env{expecting_next_protocol_negotiation = + State1#state{handshake_env = HsEnv#handshake_env{expecting_next_protocol_negotiation = NextProtocols =/= undefined}}, Connection), case Type of new -> @@ -1613,6 +1621,60 @@ do_server_hello(Type, #hello_extensions{next_protocol_negotiation = NextProtocol resumed_server_hello(State, Connection) end. +update_server_random(#{pending_read := #{security_parameters := ReadSecParams0} = + ReadState0, + pending_write := #{security_parameters := WriteSecParams0} = + WriteState0} = ConnectionStates, + Version, HighestVersion) -> + ReadRandom = override_server_random( + ReadSecParams0#security_parameters.server_random, + Version, + HighestVersion), + WriteRandom = override_server_random( + WriteSecParams0#security_parameters.server_random, + Version, + HighestVersion), + ReadSecParams = ReadSecParams0#security_parameters{server_random = ReadRandom}, + WriteSecParams = WriteSecParams0#security_parameters{server_random = WriteRandom}, + ReadState = ReadState0#{security_parameters => ReadSecParams}, + WriteState = WriteState0#{security_parameters => WriteSecParams}, + + ConnectionStates#{pending_read => ReadState, pending_write => WriteState}. + +%% TLS 1.3 - Section 4.1.3 +%% +%% If negotiating TLS 1.2, TLS 1.3 servers MUST set the last eight bytes +%% of their Random value to the bytes: +%% +%% 44 4F 57 4E 47 52 44 01 +%% +%% If negotiating TLS 1.1 or below, TLS 1.3 servers MUST and TLS 1.2 +%% servers SHOULD set the last eight bytes of their Random value to the +%% bytes: +%% +%% 44 4F 57 4E 47 52 44 00 +override_server_random(<<Random0:24/binary,_:8/binary>> = Random, {M,N}, {Major,Minor}) + when Major > 3 orelse Major =:= 3 andalso Minor >= 4 -> %% TLS 1.3 or above + if M =:= 3 andalso N =:= 3 -> %% Negotating TLS 1.2 + Down = ?RANDOM_OVERRIDE_TLS12, + <<Random0/binary,Down/binary>>; + M =:= 3 andalso N < 3 -> %% Negotating TLS 1.1 or prior + Down = ?RANDOM_OVERRIDE_TLS11, + <<Random0/binary,Down/binary>>; + true -> + Random + end; +override_server_random(<<Random0:24/binary,_:8/binary>> = Random, {M,N}, {Major,Minor}) + when Major =:= 3 andalso Minor =:= 3 -> %% TLS 1.2 + if M =:= 3 andalso N < 3 -> %% Negotating TLS 1.1 or prior + Down = ?RANDOM_OVERRIDE_TLS11, + <<Random0/binary,Down/binary>>; + true -> + Random + end; +override_server_random(Random, _, _) -> + Random. + new_server_hello(#server_hello{cipher_suite = CipherSuite, compression_method = Compression, session_id = SessionId}, @@ -1649,7 +1711,7 @@ resumed_server_hello(#state{session = Session, server_hello(ServerHello, State0, Connection) -> CipherSuite = ServerHello#server_hello.cipher_suite, - #{key_exchange := KeyAlgorithm} = ssl_cipher_format:suite_definition(CipherSuite), + #{key_exchange := KeyAlgorithm} = ssl_cipher_format:suite_bin_to_map(CipherSuite), #state{handshake_env = HsEnv} = State = Connection:queue_handshake(ServerHello, State0), State#state{handshake_env = HsEnv#handshake_env{kex_algorithm = KeyAlgorithm}}. @@ -1664,7 +1726,7 @@ handle_peer_cert(Role, PeerCert, PublicKeyInfo, State1 = State0#state{handshake_env = HsEnv#handshake_env{public_key_info = PublicKeyInfo}, session = Session#session{peer_certificate = PeerCert}}, - #{key_exchange := KeyAlgorithm} = ssl_cipher_format:suite_definition(CipherSuite), + #{key_exchange := KeyAlgorithm} = ssl_cipher_format:suite_bin_to_map(CipherSuite), State = handle_peer_cert_key(Role, PeerCert, PublicKeyInfo, KeyAlgorithm, State1), Connection:next_event(certify, no_record, State). @@ -2505,22 +2567,6 @@ hibernate_after(connection = StateName, hibernate_after(StateName, State, Actions) -> {next_state, StateName, State, Actions}. -map_extensions(#hello_extensions{renegotiation_info = RenegotiationInfo, - signature_algs = SigAlg, - alpn = Alpn, - next_protocol_negotiation = Next, - srp = SRP, - ec_point_formats = ECPointFmt, - elliptic_curves = ECCCurves, - sni = SNI}) -> - #{renegotiation_info => ssl_handshake:extension_value(RenegotiationInfo), - signature_algs => ssl_handshake:extension_value(SigAlg), - alpn => ssl_handshake:extension_value(Alpn), - srp => ssl_handshake:extension_value(SRP), - next_protocol => ssl_handshake:extension_value(Next), - ec_point_formats => ssl_handshake:extension_value(ECPointFmt), - elliptic_curves => ssl_handshake:extension_value(ECCCurves), - sni => ssl_handshake:extension_value(SNI)}. terminate_alert(normal) -> ?ALERT_REC(?WARNING, ?CLOSE_NOTIFY); @@ -2682,7 +2728,7 @@ ssl_options_list([ciphers = Key | Keys], [Value | Values], Acc) -> ssl_options_list(Keys, Values, [{Key, lists:map( fun(Suite) -> - ssl_cipher_format:suite_definition(Suite) + ssl_cipher_format:suite_bin_to_map(Suite) end, Value)} | Acc]); ssl_options_list([Key | Keys], [Value | Values], Acc) -> @@ -2885,14 +2931,14 @@ alert_user(Pids, Transport, Tracker, Socket, Active, Pid, From, Alert, Role, Con {ssl_error, Connection:socket(Pids, Transport, Socket, Tracker), ReasonCode}) end. -log_alert(true, Role, ProtocolName, StateName, #alert{role = Role} = Alert) -> +log_alert(Level, Role, ProtocolName, StateName, #alert{role = Role} = Alert) -> Txt = ssl_alert:own_alert_txt(Alert), - error_logger:info_report(io_lib:format("~s ~p: In state ~p ~s\n", [ProtocolName, Role, StateName, Txt])); -log_alert(true, Role, ProtocolName, StateName, Alert) -> + Report = io_lib:format("~s ~p: In state ~p ~s\n", [ProtocolName, Role, StateName, Txt]), + ssl_logger:notice(Level, Report); +log_alert(Level, Role, ProtocolName, StateName, Alert) -> Txt = ssl_alert:alert_txt(Alert), - error_logger:info_report(io_lib:format("~s ~p: In state ~p ~s\n", [ProtocolName, Role, StateName, Txt])); -log_alert(false, _, _, _, _) -> - ok. + Report = io_lib:format("~s ~p: In state ~p ~s\n", [ProtocolName, Role, StateName, Txt]), + ssl_logger:notice(Level, Report). invalidate_session(client, Host, Port, Session) -> ssl_manager:invalidate_session(Host, Port, Session); diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl index c90fe926b7..ff7207a8ce 100644 --- a/lib/ssl/src/ssl_connection.hrl +++ b/lib/ssl/src/ssl_connection.hrl @@ -107,20 +107,77 @@ client_certificate_requested = false :: boolean(), protocol_specific = #{} :: map(), session :: #session{} | secret_printout(), + key_share, %% Data shuffling %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% connection_states :: ssl_record:connection_states() | secret_printout(), protocol_buffers :: term() | secret_printout() , %% #protocol_buffers{} from tls_record.hrl or dtls_recor.hr user_data_buffer :: undefined | {[binary()],non_neg_integer(),[binary()]} | secret_printout(), bytes_to_read :: undefined | integer(), %% bytes to read in passive mode - %% recv and start handling - start_or_recv_from :: term() + start_or_recv_from :: term(), + log_level }). - -define(DEFAULT_DIFFIE_HELLMAN_PARAMS, #'DHParameter'{prime = ?DEFAULT_DIFFIE_HELLMAN_PRIME, base = ?DEFAULT_DIFFIE_HELLMAN_GENERATOR}). -define(WAIT_TO_ALLOW_RENEGOTIATION, 12000). + +%%---------------------------------------------------------------------- +%% TLS 1.3 +%%---------------------------------------------------------------------- + +%% TLS 1.3 uses the same state record with the following differences: +%% +%% state :: record() +%% +%% session_cache - not implemented +%% session_cache_cb - not implemented +%% crl_db - not implemented +%% client_hello_version - Bleichenbacher mitigation in TLS 1.2 +%% client_certificate_requested - Built into TLS 1.3 state machine +%% key_algorithm - not used +%% diffie_hellman_params - used in TLS 1.2 ECDH key exchange +%% diffie_hellman_keys - used in TLS 1.2 ECDH key exchange +%% psk_identity - not used +%% srp_params - not used, no srp extension in TLS 1.3 +%% srp_keys - not used, no srp extension in TLS 1.3 +%% premaster_secret - not used +%% renegotiation - TLS 1.3 forbids renegotiation +%% hello - used in user_hello, handshake continue +%% allow_renegotiate - TLS 1.3 forbids renegotiation +%% expecting_next_protocol_negotiation - ALPN replaced NPN, depricated in TLS 1.3 +%% expecting_finished - not implemented, used by abbreviated +%% next_protocol - ALPN replaced NPN, depricated in TLS 1.3 +%% +%% connection_state :: map() +%% +%% compression_state - not used +%% mac_secret - not used +%% sequence_number - not used +%% secure_renegotiation - not used, no renegotiation_info in TLS 1.3 +%% client_verify_data - not used, no renegotiation_info in TLS 1.3 +%% server_verify_data - not used, no renegotiation_info in TLS 1.3 +%% beast_mitigation - not used +%% +%% security_parameters :: map() +%% +%% cipher_type - TLS 1.3 uses only AEAD ciphers +%% iv_size - not used +%% key_size - not used +%% key_material_length - not used +%% expanded_key_material_length - used in SSL 3.0 +%% mac_algorithm - not used +%% prf_algorithm - not used +%% hash_size - not used +%% compression_algorithm - not used +%% master_secret - used for multiple secret types in TLS 1.3 +%% client_random - not used +%% server_random - not used +%% exportable - not used +%% +%% cipher_state :: record() +%% nonce - used for sequence_number + -endif. % -ifdef(ssl_connection). diff --git a/lib/ssl/src/ssl_crl_hash_dir.erl b/lib/ssl/src/ssl_crl_hash_dir.erl index bb62737232..9478ff9b78 100644 --- a/lib/ssl/src/ssl_crl_hash_dir.erl +++ b/lib/ssl/src/ssl_crl_hash_dir.erl @@ -20,6 +20,7 @@ -module(ssl_crl_hash_dir). -include_lib("public_key/include/public_key.hrl"). +-include_lib("kernel/include/logger.hrl"). -behaviour(ssl_crl_cache_api). @@ -55,7 +56,7 @@ select(Issuer, {_DbHandle, [{dir, Dir}]}) -> %% is happy with that, but if it's true, this is an error. []; {error, Error} -> - error_logger:error_report( + ?LOG_ERROR( [{cannot_find_crl, Error}, {dir, Dir}, {module, ?MODULE}, @@ -86,7 +87,7 @@ find_crls(Issuer, Hash, Dir, N, Acc) -> error:Error -> %% Something is wrong with the file. Report %% it, and try the next one. - error_logger:error_report( + ?LOG_ERROR( [{crl_parse_error, Error}, {filename, Filename}, {module, ?MODULE}, diff --git a/lib/ssl/src/ssl_dh_groups.erl b/lib/ssl/src/ssl_dh_groups.erl new file mode 100644 index 0000000000..20d53de430 --- /dev/null +++ b/lib/ssl/src/ssl_dh_groups.erl @@ -0,0 +1,467 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2007-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ssl_dh_groups). + +-include_lib("public_key/include/public_key.hrl"). + +-export([modp2048_generator/0, modp2048_prime/0, + ffdhe2048_generator/0, ffdhe2048_prime/0, + ffdhe3072_generator/0, ffdhe3072_prime/0, + ffdhe4096_generator/0, ffdhe4096_prime/0, + ffdhe6144_generator/0, ffdhe6144_prime/0, + ffdhe8192_generator/0, ffdhe8192_prime/0, + dh_params/1]). + +%% RFC3526 - 2048-bit MODP Group +%% This group is assigned id 14. +%% +%% This prime is: 2^2048 - 2^1984 - 1 + 2^64 * { [2^1918 pi] + 124476 } +%% +%% Its hexadecimal value is: +%% +%% FFFFFFFF FFFFFFFF C90FDAA2 2168C234 C4C6628B 80DC1CD1 +%% 29024E08 8A67CC74 020BBEA6 3B139B22 514A0879 8E3404DD +%% EF9519B3 CD3A431B 302B0A6D F25F1437 4FE1356D 6D51C245 +%% E485B576 625E7EC6 F44C42E9 A637ED6B 0BFF5CB6 F406B7ED +%% EE386BFB 5A899FA5 AE9F2411 7C4B1FE6 49286651 ECE45B3D +%% C2007CB8 A163BF05 98DA4836 1C55D39A 69163FA8 FD24CF5F +%% 83655D23 DCA3AD96 1C62F356 208552BB 9ED52907 7096966D +%% 670C354E 4ABC9804 F1746C08 CA18217C 32905E46 2E36CE3B +%% E39E772C 180E8603 9B2783A2 EC07A28F B5C55DF0 6F4C52C9 +%% DE2BCBF6 95581718 3995497C EA956AE5 15D22618 98FA0510 +%% 15728E5A 8AACAA68 FFFFFFFF FFFFFFFF +%% +%% The generator is: 2. +modp2048_generator() -> + 2. + +modp2048_prime() -> + P = "FFFFFFFF" "FFFFFFFF" "C90FDAA2" "2168C234" "C4C6628B" "80DC1CD1" + "29024E08" "8A67CC74" "020BBEA6" "3B139B22" "514A0879" "8E3404DD" + "EF9519B3" "CD3A431B" "302B0A6D" "F25F1437" "4FE1356D" "6D51C245" + "E485B576" "625E7EC6" "F44C42E9" "A637ED6B" "0BFF5CB6" "F406B7ED" + "EE386BFB" "5A899FA5" "AE9F2411" "7C4B1FE6" "49286651" "ECE45B3D" + "C2007CB8" "A163BF05" "98DA4836" "1C55D39A" "69163FA8" "FD24CF5F" + "83655D23" "DCA3AD96" "1C62F356" "208552BB" "9ED52907" "7096966D" + "670C354E" "4ABC9804" "F1746C08" "CA18217C" "32905E46" "2E36CE3B" + "E39E772C" "180E8603" "9B2783A2" "EC07A28F" "B5C55DF0" "6F4C52C9" + "DE2BCBF6" "95581718" "3995497C" "EA956AE5" "15D22618" "98FA0510" + "15728E5A" "8AACAA68" "FFFFFFFF" "FFFFFFFF", + list_to_integer(P, 16). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% RFC8446 - TLS 1.3 +%%% RFC7919 - Negotiated FFDHE for TLS +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%% ffdhe2048 +%% --------- +%% The 2048-bit group has registry value 256 and is calculated from the +%% following formula: +%% +%% The modulus is: +%% +%% p = 2^2048 - 2^1984 + {[2^1918 * e] + 560316 } * 2^64 - 1 +%% +%% The hexadecimal representation of p is: +%% +%% FFFFFFFF FFFFFFFF ADF85458 A2BB4A9A AFDC5620 273D3CF1 +%% D8B9C583 CE2D3695 A9E13641 146433FB CC939DCE 249B3EF9 +%% 7D2FE363 630C75D8 F681B202 AEC4617A D3DF1ED5 D5FD6561 +%% 2433F51F 5F066ED0 85636555 3DED1AF3 B557135E 7F57C935 +%% 984F0C70 E0E68B77 E2A689DA F3EFE872 1DF158A1 36ADE735 +%% 30ACCA4F 483A797A BC0AB182 B324FB61 D108A94B B2C8E3FB +%% B96ADAB7 60D7F468 1D4F42A3 DE394DF4 AE56EDE7 6372BB19 +%% 0B07A7C8 EE0A6D70 9E02FCE1 CDF7E2EC C03404CD 28342F61 +%% 9172FE9C E98583FF 8E4F1232 EEF28183 C3FE3B1B 4C6FAD73 +%% 3BB5FCBC 2EC22005 C58EF183 7D1683B2 C6F34A26 C1B2EFFA +%% 886B4238 61285C97 FFFFFFFF FFFFFFFF +%% +%% The generator is: g = 2 +%% +%% The group size is: q = (p-1)/2 +%% +%% The estimated symmetric-equivalent strength of this group is 103 +%% bits. +ffdhe2048_generator() -> + 2. + +ffdhe2048_prime() -> + P = "FFFFFFFF" "FFFFFFFF" "ADF85458" "A2BB4A9A" "AFDC5620" "273D3CF1" + "D8B9C583" "CE2D3695" "A9E13641" "146433FB" "CC939DCE" "249B3EF9" + "7D2FE363" "630C75D8" "F681B202" "AEC4617A" "D3DF1ED5" "D5FD6561" + "2433F51F" "5F066ED0" "85636555" "3DED1AF3" "B557135E" "7F57C935" + "984F0C70" "E0E68B77" "E2A689DA" "F3EFE872" "1DF158A1" "36ADE735" + "30ACCA4F" "483A797A" "BC0AB182" "B324FB61" "D108A94B" "B2C8E3FB" + "B96ADAB7" "60D7F468" "1D4F42A3" "DE394DF4" "AE56EDE7" "6372BB19" + "0B07A7C8" "EE0A6D70" "9E02FCE1" "CDF7E2EC" "C03404CD" "28342F61" + "9172FE9C" "E98583FF" "8E4F1232" "EEF28183" "C3FE3B1B" "4C6FAD73" + "3BB5FCBC" "2EC22005" "C58EF183" "7D1683B2" "C6F34A26" "C1B2EFFA" + "886B4238" "61285C97" "FFFFFFFF" "FFFFFFFF", + list_to_integer(P, 16). + + +%% ffdhe3072 +%% --------- +%% The 3072-bit prime has registry value 257 and is calculated from the +%% following formula: +%% +%% The modulus is: +%% +%% p = 2^3072 - 2^3008 + {[2^2942 * e] + 2625351} * 2^64 - 1 +%% +%% The hexadecimal representation of p is: +%% +%% FFFFFFFF FFFFFFFF ADF85458 A2BB4A9A AFDC5620 273D3CF1 +%% D8B9C583 CE2D3695 A9E13641 146433FB CC939DCE 249B3EF9 +%% 7D2FE363 630C75D8 F681B202 AEC4617A D3DF1ED5 D5FD6561 +%% 2433F51F 5F066ED0 85636555 3DED1AF3 B557135E 7F57C935 +%% 984F0C70 E0E68B77 E2A689DA F3EFE872 1DF158A1 36ADE735 +%% 30ACCA4F 483A797A BC0AB182 B324FB61 D108A94B B2C8E3FB +%% B96ADAB7 60D7F468 1D4F42A3 DE394DF4 AE56EDE7 6372BB19 +%% 0B07A7C8 EE0A6D70 9E02FCE1 CDF7E2EC C03404CD 28342F61 +%% 9172FE9C E98583FF 8E4F1232 EEF28183 C3FE3B1B 4C6FAD73 +%% 3BB5FCBC 2EC22005 C58EF183 7D1683B2 C6F34A26 C1B2EFFA +%% 886B4238 611FCFDC DE355B3B 6519035B BC34F4DE F99C0238 +%% 61B46FC9 D6E6C907 7AD91D26 91F7F7EE 598CB0FA C186D91C +%% AEFE1309 85139270 B4130C93 BC437944 F4FD4452 E2D74DD3 +%% 64F2E21E 71F54BFF 5CAE82AB 9C9DF69E E86D2BC5 22363A0D +%% ABC52197 9B0DEADA 1DBF9A42 D5C4484E 0ABCD06B FA53DDEF +%% 3C1B20EE 3FD59D7C 25E41D2B 66C62E37 FFFFFFFF FFFFFFFF +%% +%% The generator is: g = 2 +%% +%% The group size is: q = (p-1)/2 +%% +%% The estimated symmetric-equivalent strength of this group is 125 +%% bits. +ffdhe3072_generator() -> + 2. + +ffdhe3072_prime() -> + P = "FFFFFFFF" "FFFFFFFF" "ADF85458" "A2BB4A9A" "AFDC5620" "273D3CF1" + "D8B9C583" "CE2D3695" "A9E13641" "146433FB" "CC939DCE" "249B3EF9" + "7D2FE363" "630C75D8" "F681B202" "AEC4617A" "D3DF1ED5" "D5FD6561" + "2433F51F" "5F066ED0" "85636555" "3DED1AF3" "B557135E" "7F57C935" + "984F0C70" "E0E68B77" "E2A689DA" "F3EFE872" "1DF158A1" "36ADE735" + "30ACCA4F" "483A797A" "BC0AB182" "B324FB61" "D108A94B" "B2C8E3FB" + "B96ADAB7" "60D7F468" "1D4F42A3" "DE394DF4" "AE56EDE7" "6372BB19" + "0B07A7C8" "EE0A6D70" "9E02FCE1" "CDF7E2EC" "C03404CD" "28342F61" + "9172FE9C" "E98583FF" "8E4F1232" "EEF28183" "C3FE3B1B" "4C6FAD73" + "3BB5FCBC" "2EC22005" "C58EF183" "7D1683B2" "C6F34A26" "C1B2EFFA" + "886B4238" "611FCFDC" "DE355B3B" "6519035B" "BC34F4DE" "F99C0238" + "61B46FC9" "D6E6C907" "7AD91D26" "91F7F7EE" "598CB0FA" "C186D91C" + "AEFE1309" "85139270" "B4130C93" "BC437944" "F4FD4452" "E2D74DD3" + "64F2E21E" "71F54BFF" "5CAE82AB" "9C9DF69E" "E86D2BC5" "22363A0D" + "ABC52197" "9B0DEADA" "1DBF9A42" "D5C4484E" "0ABCD06B" "FA53DDEF" + "3C1B20EE" "3FD59D7C" "25E41D2B" "66C62E37" "FFFFFFFF" "FFFFFFFF", + list_to_integer(P, 16). + + +%% ffdhe4096 +%% --------- +%% The 4096-bit group has registry value 258 and is calculated from the +%% following formula: +%% +%% The modulus is: +%% +%% p = 2^4096 - 2^4032 + {[2^3966 * e] + 5736041} * 2^64 - 1 +%% +%% The hexadecimal representation of p is: +%% +%% FFFFFFFF FFFFFFFF ADF85458 A2BB4A9A AFDC5620 273D3CF1 +%% D8B9C583 CE2D3695 A9E13641 146433FB CC939DCE 249B3EF9 +%% 7D2FE363 630C75D8 F681B202 AEC4617A D3DF1ED5 D5FD6561 +%% 2433F51F 5F066ED0 85636555 3DED1AF3 B557135E 7F57C935 +%% 984F0C70 E0E68B77 E2A689DA F3EFE872 1DF158A1 36ADE735 +%% 30ACCA4F 483A797A BC0AB182 B324FB61 D108A94B B2C8E3FB +%% B96ADAB7 60D7F468 1D4F42A3 DE394DF4 AE56EDE7 6372BB19 +%% 0B07A7C8 EE0A6D70 9E02FCE1 CDF7E2EC C03404CD 28342F61 +%% 9172FE9C E98583FF 8E4F1232 EEF28183 C3FE3B1B 4C6FAD73 +%% 3BB5FCBC 2EC22005 C58EF183 7D1683B2 C6F34A26 C1B2EFFA +%% 886B4238 611FCFDC DE355B3B 6519035B BC34F4DE F99C0238 +%% 61B46FC9 D6E6C907 7AD91D26 91F7F7EE 598CB0FA C186D91C +%% AEFE1309 85139270 B4130C93 BC437944 F4FD4452 E2D74DD3 +%% 64F2E21E 71F54BFF 5CAE82AB 9C9DF69E E86D2BC5 22363A0D +%% ABC52197 9B0DEADA 1DBF9A42 D5C4484E 0ABCD06B FA53DDEF +%% 3C1B20EE 3FD59D7C 25E41D2B 669E1EF1 6E6F52C3 164DF4FB +%% 7930E9E4 E58857B6 AC7D5F42 D69F6D18 7763CF1D 55034004 +%% 87F55BA5 7E31CC7A 7135C886 EFB4318A ED6A1E01 2D9E6832 +%% A907600A 918130C4 6DC778F9 71AD0038 092999A3 33CB8B7A +%% 1A1DB93D 7140003C 2A4ECEA9 F98D0ACC 0A8291CD CEC97DCF +%% 8EC9B55A 7F88A46B 4DB5A851 F44182E1 C68A007E 5E655F6A +%% FFFFFFFF FFFFFFFF +%% +%% The generator is: g = 2 +%% +%% The group size is: q = (p-1)/2 +%% +%% The estimated symmetric-equivalent strength of this group is 150 +%% bits. +ffdhe4096_generator() -> + 2. + +ffdhe4096_prime() -> + P = "FFFFFFFF" "FFFFFFFF" "ADF85458" "A2BB4A9A" "AFDC5620" "273D3CF1" + "D8B9C583" "CE2D3695" "A9E13641" "146433FB" "CC939DCE" "249B3EF9" + "7D2FE363" "630C75D8" "F681B202" "AEC4617A" "D3DF1ED5" "D5FD6561" + "2433F51F" "5F066ED0" "85636555" "3DED1AF3" "B557135E" "7F57C935" + "984F0C70" "E0E68B77" "E2A689DA" "F3EFE872" "1DF158A1" "36ADE735" + "30ACCA4F" "483A797A" "BC0AB182" "B324FB61" "D108A94B" "B2C8E3FB" + "B96ADAB7" "60D7F468" "1D4F42A3" "DE394DF4" "AE56EDE7" "6372BB19" + "0B07A7C8" "EE0A6D70" "9E02FCE1" "CDF7E2EC" "C03404CD" "28342F61" + "9172FE9C" "E98583FF" "8E4F1232" "EEF28183" "C3FE3B1B" "4C6FAD73" + "3BB5FCBC" "2EC22005" "C58EF183" "7D1683B2" "C6F34A26" "C1B2EFFA" + "886B4238" "611FCFDC" "DE355B3B" "6519035B" "BC34F4DE" "F99C0238" + "61B46FC9" "D6E6C907" "7AD91D26" "91F7F7EE" "598CB0FA" "C186D91C" + "AEFE1309" "85139270" "B4130C93" "BC437944" "F4FD4452" "E2D74DD3" + "64F2E21E" "71F54BFF" "5CAE82AB" "9C9DF69E" "E86D2BC5" "22363A0D" + "ABC52197" "9B0DEADA" "1DBF9A42" "D5C4484E" "0ABCD06B" "FA53DDEF" + "3C1B20EE" "3FD59D7C" "25E41D2B" "669E1EF1" "6E6F52C3" "164DF4FB" + "7930E9E4" "E58857B6" "AC7D5F42" "D69F6D18" "7763CF1D" "55034004" + "87F55BA5" "7E31CC7A" "7135C886" "EFB4318A" "ED6A1E01" "2D9E6832" + "A907600A" "918130C4" "6DC778F9" "71AD0038" "092999A3" "33CB8B7A" + "1A1DB93D" "7140003C" "2A4ECEA9" "F98D0ACC" "0A8291CD" "CEC97DCF" + "8EC9B55A" "7F88A46B" "4DB5A851" "F44182E1" "C68A007E" "5E655F6A" + "FFFFFFFF" "FFFFFFFF", + list_to_integer(P, 16). + + +%% ffdhe6144 +%% --------- +%% The 6144-bit group has registry value 259 and is calculated from the +%% following formula: +%% +%% The modulus is: +%% +%% p = 2^6144 - 2^6080 + {[2^6014 * e] + 15705020} * 2^64 - 1 +%% +%% The hexadecimal representation of p is: +%% +%% FFFFFFFF FFFFFFFF ADF85458 A2BB4A9A AFDC5620 273D3CF1 +%% D8B9C583 CE2D3695 A9E13641 146433FB CC939DCE 249B3EF9 +%% 7D2FE363 630C75D8 F681B202 AEC4617A D3DF1ED5 D5FD6561 +%% 2433F51F 5F066ED0 85636555 3DED1AF3 B557135E 7F57C935 +%% 984F0C70 E0E68B77 E2A689DA F3EFE872 1DF158A1 36ADE735 +%% 30ACCA4F 483A797A BC0AB182 B324FB61 D108A94B B2C8E3FB +%% B96ADAB7 60D7F468 1D4F42A3 DE394DF4 AE56EDE7 6372BB19 +%% 0B07A7C8 EE0A6D70 9E02FCE1 CDF7E2EC C03404CD 28342F61 +%% 9172FE9C E98583FF 8E4F1232 EEF28183 C3FE3B1B 4C6FAD73 +%% 3BB5FCBC 2EC22005 C58EF183 7D1683B2 C6F34A26 C1B2EFFA +%% 886B4238 611FCFDC DE355B3B 6519035B BC34F4DE F99C0238 +%% 61B46FC9 D6E6C907 7AD91D26 91F7F7EE 598CB0FA C186D91C +%% AEFE1309 85139270 B4130C93 BC437944 F4FD4452 E2D74DD3 +%% 64F2E21E 71F54BFF 5CAE82AB 9C9DF69E E86D2BC5 22363A0D +%% ABC52197 9B0DEADA 1DBF9A42 D5C4484E 0ABCD06B FA53DDEF +%% 3C1B20EE 3FD59D7C 25E41D2B 669E1EF1 6E6F52C3 164DF4FB +%% 7930E9E4 E58857B6 AC7D5F42 D69F6D18 7763CF1D 55034004 +%% 87F55BA5 7E31CC7A 7135C886 EFB4318A ED6A1E01 2D9E6832 +%% A907600A 918130C4 6DC778F9 71AD0038 092999A3 33CB8B7A +%% 1A1DB93D 7140003C 2A4ECEA9 F98D0ACC 0A8291CD CEC97DCF +%% 8EC9B55A 7F88A46B 4DB5A851 F44182E1 C68A007E 5E0DD902 +%% 0BFD64B6 45036C7A 4E677D2C 38532A3A 23BA4442 CAF53EA6 +%% 3BB45432 9B7624C8 917BDD64 B1C0FD4C B38E8C33 4C701C3A +%% CDAD0657 FCCFEC71 9B1F5C3E 4E46041F 388147FB 4CFDB477 +%% A52471F7 A9A96910 B855322E DB6340D8 A00EF092 350511E3 +%% 0ABEC1FF F9E3A26E 7FB29F8C 183023C3 587E38DA 0077D9B4 +%% 763E4E4B 94B2BBC1 94C6651E 77CAF992 EEAAC023 2A281BF6 +%% B3A739C1 22611682 0AE8DB58 47A67CBE F9C9091B 462D538C +%% D72B0374 6AE77F5E 62292C31 1562A846 505DC82D B854338A +%% E49F5235 C95B9117 8CCF2DD5 CACEF403 EC9D1810 C6272B04 +%% 5B3B71F9 DC6B80D6 3FDD4A8E 9ADB1E69 62A69526 D43161C1 +%% A41D570D 7938DAD4 A40E329C D0E40E65 FFFFFFFF FFFFFFFF +%% +%% The generator is: g = 2 +%% +%% The group size is: q = (p-1)/2 +%% +%% The estimated symmetric-equivalent strength of this group is 175 +%% bits. +ffdhe6144_generator() -> + 2. + +ffdhe6144_prime() -> + P = "FFFFFFFF" "FFFFFFFF" "ADF85458" "A2BB4A9A" "AFDC5620" "273D3CF1" + "D8B9C583" "CE2D3695" "A9E13641" "146433FB" "CC939DCE" "249B3EF9" + "7D2FE363" "630C75D8" "F681B202" "AEC4617A" "D3DF1ED5" "D5FD6561" + "2433F51F" "5F066ED0" "85636555" "3DED1AF3" "B557135E" "7F57C935" + "984F0C70" "E0E68B77" "E2A689DA" "F3EFE872" "1DF158A1" "36ADE735" + "30ACCA4F" "483A797A" "BC0AB182" "B324FB61" "D108A94B" "B2C8E3FB" + "B96ADAB7" "60D7F468" "1D4F42A3" "DE394DF4" "AE56EDE7" "6372BB19" + "0B07A7C8" "EE0A6D70" "9E02FCE1" "CDF7E2EC" "C03404CD" "28342F61" + "9172FE9C" "E98583FF" "8E4F1232" "EEF28183" "C3FE3B1B" "4C6FAD73" + "3BB5FCBC" "2EC22005" "C58EF183" "7D1683B2" "C6F34A26" "C1B2EFFA" + "886B4238" "611FCFDC" "DE355B3B" "6519035B" "BC34F4DE" "F99C0238" + "61B46FC9" "D6E6C907" "7AD91D26" "91F7F7EE" "598CB0FA" "C186D91C" + "AEFE1309" "85139270" "B4130C93" "BC437944" "F4FD4452" "E2D74DD3" + "64F2E21E" "71F54BFF" "5CAE82AB" "9C9DF69E" "E86D2BC5" "22363A0D" + "ABC52197" "9B0DEADA" "1DBF9A42" "D5C4484E" "0ABCD06B" "FA53DDEF" + "3C1B20EE" "3FD59D7C" "25E41D2B" "669E1EF1" "6E6F52C3" "164DF4FB" + "7930E9E4" "E58857B6" "AC7D5F42" "D69F6D18" "7763CF1D" "55034004" + "87F55BA5" "7E31CC7A" "7135C886" "EFB4318A" "ED6A1E01" "2D9E6832" + "A907600A" "918130C4" "6DC778F9" "71AD0038" "092999A3" "33CB8B7A" + "1A1DB93D" "7140003C" "2A4ECEA9" "F98D0ACC" "0A8291CD" "CEC97DCF" + "8EC9B55A" "7F88A46B" "4DB5A851" "F44182E1" "C68A007E" "5E0DD902" + "0BFD64B6" "45036C7A" "4E677D2C" "38532A3A" "23BA4442" "CAF53EA6" + "3BB45432" "9B7624C8" "917BDD64" "B1C0FD4C" "B38E8C33" "4C701C3A" + "CDAD0657" "FCCFEC71" "9B1F5C3E" "4E46041F" "388147FB" "4CFDB477" + "A52471F7" "A9A96910" "B855322E" "DB6340D8" "A00EF092" "350511E3" + "0ABEC1FF" "F9E3A26E" "7FB29F8C" "183023C3" "587E38DA" "0077D9B4" + "763E4E4B" "94B2BBC1" "94C6651E" "77CAF992" "EEAAC023" "2A281BF6" + "B3A739C1" "22611682" "0AE8DB58" "47A67CBE" "F9C9091B" "462D538C" + "D72B0374" "6AE77F5E" "62292C31" "1562A846" "505DC82D" "B854338A" + "E49F5235" "C95B9117" "8CCF2DD5" "CACEF403" "EC9D1810" "C6272B04" + "5B3B71F9" "DC6B80D6" "3FDD4A8E" "9ADB1E69" "62A69526" "D43161C1" + "A41D570D" "7938DAD4" "A40E329C" "D0E40E65" "FFFFFFFF" "FFFFFFFF", + list_to_integer(P, 16). + + +%% ffdhe8192 +%% --------- +%% The 8192-bit group has registry value 260 and is calculated from the +%% following formula: +%% +%% The modulus is: +%% +%% p = 2^8192 - 2^8128 + {[2^8062 * e] + 10965728} * 2^64 - 1 +%% +%% The hexadecimal representation of p is: +%% +%% FFFFFFFF FFFFFFFF ADF85458 A2BB4A9A AFDC5620 273D3CF1 +%% D8B9C583 CE2D3695 A9E13641 146433FB CC939DCE 249B3EF9 +%% 7D2FE363 630C75D8 F681B202 AEC4617A D3DF1ED5 D5FD6561 +%% 2433F51F 5F066ED0 85636555 3DED1AF3 B557135E 7F57C935 +%% 984F0C70 E0E68B77 E2A689DA F3EFE872 1DF158A1 36ADE735 +%% 30ACCA4F 483A797A BC0AB182 B324FB61 D108A94B B2C8E3FB +%% B96ADAB7 60D7F468 1D4F42A3 DE394DF4 AE56EDE7 6372BB19 +%% 0B07A7C8 EE0A6D70 9E02FCE1 CDF7E2EC C03404CD 28342F61 +%% 9172FE9C E98583FF 8E4F1232 EEF28183 C3FE3B1B 4C6FAD73 +%% 3BB5FCBC 2EC22005 C58EF183 7D1683B2 C6F34A26 C1B2EFFA +%% 886B4238 611FCFDC DE355B3B 6519035B BC34F4DE F99C0238 +%% 61B46FC9 D6E6C907 7AD91D26 91F7F7EE 598CB0FA C186D91C +%% AEFE1309 85139270 B4130C93 BC437944 F4FD4452 E2D74DD3 +%% 64F2E21E 71F54BFF 5CAE82AB 9C9DF69E E86D2BC5 22363A0D +%% ABC52197 9B0DEADA 1DBF9A42 D5C4484E 0ABCD06B FA53DDEF +%% 3C1B20EE 3FD59D7C 25E41D2B 669E1EF1 6E6F52C3 164DF4FB +%% 7930E9E4 E58857B6 AC7D5F42 D69F6D18 7763CF1D 55034004 +%% 87F55BA5 7E31CC7A 7135C886 EFB4318A ED6A1E01 2D9E6832 +%% A907600A 918130C4 6DC778F9 71AD0038 092999A3 33CB8B7A +%% 1A1DB93D 7140003C 2A4ECEA9 F98D0ACC 0A8291CD CEC97DCF +%% 8EC9B55A 7F88A46B 4DB5A851 F44182E1 C68A007E 5E0DD902 +%% 0BFD64B6 45036C7A 4E677D2C 38532A3A 23BA4442 CAF53EA6 +%% 3BB45432 9B7624C8 917BDD64 B1C0FD4C B38E8C33 4C701C3A +%% CDAD0657 FCCFEC71 9B1F5C3E 4E46041F 388147FB 4CFDB477 +%% A52471F7 A9A96910 B855322E DB6340D8 A00EF092 350511E3 +%% 0ABEC1FF F9E3A26E 7FB29F8C 183023C3 587E38DA 0077D9B4 +%% 763E4E4B 94B2BBC1 94C6651E 77CAF992 EEAAC023 2A281BF6 +%% B3A739C1 22611682 0AE8DB58 47A67CBE F9C9091B 462D538C +%% D72B0374 6AE77F5E 62292C31 1562A846 505DC82D B854338A +%% E49F5235 C95B9117 8CCF2DD5 CACEF403 EC9D1810 C6272B04 +%% 5B3B71F9 DC6B80D6 3FDD4A8E 9ADB1E69 62A69526 D43161C1 +%% A41D570D 7938DAD4 A40E329C CFF46AAA 36AD004C F600C838 +%% 1E425A31 D951AE64 FDB23FCE C9509D43 687FEB69 EDD1CC5E +%% 0B8CC3BD F64B10EF 86B63142 A3AB8829 555B2F74 7C932665 +%% CB2C0F1C C01BD702 29388839 D2AF05E4 54504AC7 8B758282 +%% 2846C0BA 35C35F5C 59160CC0 46FD8251 541FC68C 9C86B022 +%% BB709987 6A460E74 51A8A931 09703FEE 1C217E6C 3826E52C +%% 51AA691E 0E423CFC 99E9E316 50C1217B 624816CD AD9A95F9 +%% D5B80194 88D9C0A0 A1FE3075 A577E231 83F81D4A 3F2FA457 +%% 1EFC8CE0 BA8A4FE8 B6855DFE 72B0A66E DED2FBAB FBE58A30 +%% FAFABE1C 5D71A87E 2F741EF8 C1FE86FE A6BBFDE5 30677F0D +%% 97D11D49 F7A8443D 0822E506 A9F4614E 011E2A94 838FF88C +%% D68C8BB7 C5C6424C FFFFFFFF FFFFFFFF +%% +%% The generator is: g = 2 +%% +%% The group size is: q = (p-1)/2 +%% +%% The estimated symmetric-equivalent strength of this group is 192 +%% bits. +ffdhe8192_generator() -> + 2. + +ffdhe8192_prime() -> + P = "FFFFFFFF" "FFFFFFFF" "ADF85458" "A2BB4A9A" "AFDC5620" "273D3CF1" + "D8B9C583" "CE2D3695" "A9E13641" "146433FB" "CC939DCE" "249B3EF9" + "7D2FE363" "630C75D8" "F681B202" "AEC4617A" "D3DF1ED5" "D5FD6561" + "2433F51F" "5F066ED0" "85636555" "3DED1AF3" "B557135E" "7F57C935" + "984F0C70" "E0E68B77" "E2A689DA" "F3EFE872" "1DF158A1" "36ADE735" + "30ACCA4F" "483A797A" "BC0AB182" "B324FB61" "D108A94B" "B2C8E3FB" + "B96ADAB7" "60D7F468" "1D4F42A3" "DE394DF4" "AE56EDE7" "6372BB19" + "0B07A7C8" "EE0A6D70" "9E02FCE1" "CDF7E2EC" "C03404CD" "28342F61" + "9172FE9C" "E98583FF" "8E4F1232" "EEF28183" "C3FE3B1B" "4C6FAD73" + "3BB5FCBC" "2EC22005" "C58EF183" "7D1683B2" "C6F34A26" "C1B2EFFA" + "886B4238" "611FCFDC" "DE355B3B" "6519035B" "BC34F4DE" "F99C0238" + "61B46FC9" "D6E6C907" "7AD91D26" "91F7F7EE" "598CB0FA" "C186D91C" + "AEFE1309" "85139270" "B4130C93" "BC437944" "F4FD4452" "E2D74DD3" + "64F2E21E" "71F54BFF" "5CAE82AB" "9C9DF69E" "E86D2BC5" "22363A0D" + "ABC52197" "9B0DEADA" "1DBF9A42" "D5C4484E" "0ABCD06B" "FA53DDEF" + "3C1B20EE" "3FD59D7C" "25E41D2B" "669E1EF1" "6E6F52C3" "164DF4FB" + "7930E9E4" "E58857B6" "AC7D5F42" "D69F6D18" "7763CF1D" "55034004" + "87F55BA5" "7E31CC7A" "7135C886" "EFB4318A" "ED6A1E01" "2D9E6832" + "A907600A" "918130C4" "6DC778F9" "71AD0038" "092999A3" "33CB8B7A" + "1A1DB93D" "7140003C" "2A4ECEA9" "F98D0ACC" "0A8291CD" "CEC97DCF" + "8EC9B55A" "7F88A46B" "4DB5A851" "F44182E1" "C68A007E" "5E0DD902" + "0BFD64B6" "45036C7A" "4E677D2C" "38532A3A" "23BA4442" "CAF53EA6" + "3BB45432" "9B7624C8" "917BDD64" "B1C0FD4C" "B38E8C33" "4C701C3A" + "CDAD0657" "FCCFEC71" "9B1F5C3E" "4E46041F" "388147FB" "4CFDB477" + "A52471F7" "A9A96910" "B855322E" "DB6340D8" "A00EF092" "350511E3" + "0ABEC1FF" "F9E3A26E" "7FB29F8C" "183023C3" "587E38DA" "0077D9B4" + "763E4E4B" "94B2BBC1" "94C6651E" "77CAF992" "EEAAC023" "2A281BF6" + "B3A739C1" "22611682" "0AE8DB58" "47A67CBE" "F9C9091B" "462D538C" + "D72B0374" "6AE77F5E" "62292C31" "1562A846" "505DC82D" "B854338A" + "E49F5235" "C95B9117" "8CCF2DD5" "CACEF403" "EC9D1810" "C6272B04" + "5B3B71F9" "DC6B80D6" "3FDD4A8E" "9ADB1E69" "62A69526" "D43161C1" + "A41D570D" "7938DAD4" "A40E329C" "CFF46AAA" "36AD004C" "F600C838" + "1E425A31" "D951AE64" "FDB23FCE" "C9509D43" "687FEB69" "EDD1CC5E" + "0B8CC3BD" "F64B10EF" "86B63142" "A3AB8829" "555B2F74" "7C932665" + "CB2C0F1C" "C01BD702" "29388839" "D2AF05E4" "54504AC7" "8B758282" + "2846C0BA" "35C35F5C" "59160CC0" "46FD8251" "541FC68C" "9C86B022" + "BB709987" "6A460E74" "51A8A931" "09703FEE" "1C217E6C" "3826E52C" + "51AA691E" "0E423CFC" "99E9E316" "50C1217B" "624816CD" "AD9A95F9" + "D5B80194" "88D9C0A0" "A1FE3075" "A577E231" "83F81D4A" "3F2FA457" + "1EFC8CE0" "BA8A4FE8" "B6855DFE" "72B0A66E" "DED2FBAB" "FBE58A30" + "FAFABE1C" "5D71A87E" "2F741EF8" "C1FE86FE" "A6BBFDE5" "30677F0D" + "97D11D49" "F7A8443D" "0822E506" "A9F4614E" "011E2A94" "838FF88C" + "D68C8BB7" "C5C6424C" "FFFFFFFF" "FFFFFFFF", + list_to_integer(P, 16). + +dh_params(ffdhe2048) -> + #'DHParameter'{ + prime = ffdhe2048_prime(), + base = ffdhe2048_generator()}; +dh_params(ffdhe3072) -> + #'DHParameter'{ + prime = ffdhe3072_prime(), + base = ffdhe3072_generator()}; +dh_params(ffdhe4096) -> + #'DHParameter'{ + prime = ffdhe4096_prime(), + base = ffdhe4096_generator()}; +dh_params(ffdhe6144) -> + #'DHParameter'{ + prime = ffdhe6144_prime(), + base = ffdhe6144_generator()}; +dh_params(ffdhe8192) -> + #'DHParameter'{ + prime = ffdhe8192_prime(), + base = ffdhe8192_generator()}. diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 29db1b07c4..7b34991f4f 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -30,6 +30,7 @@ -include("ssl_alert.hrl"). -include("ssl_internal.hrl"). -include("ssl_srp.hrl"). +-include("tls_handshake_1_3.hrl"). -include_lib("public_key/include/public_key.hrl"). -export_type([ssl_handshake/0, ssl_handshake_history/0, @@ -38,7 +39,7 @@ -type oid() :: tuple(). -type public_key_params() :: #'Dss-Parms'{} | {namedCurve, oid()} | #'ECParameters'{} | term(). -type public_key_info() :: {oid(), #'RSAPublicKey'{} | integer() | #'ECPoint'{}, public_key_params()}. --type ssl_handshake_history() :: {[binary()], [binary()]}. +-type ssl_handshake_history() :: {iodata(), iodata()}. -type ssl_handshake() :: #server_hello{} | #server_hello_done{} | #certificate{} | #certificate_request{} | #client_key_exchange{} | #finished{} | #certificate_verify{} | @@ -53,14 +54,14 @@ -export([certify/7, certificate_verify/6, verify_signature/5, master_secret/4, server_key_exchange_hash/2, verify_connection/6, init_handshake_history/0, update_handshake_history/2, verify_server_key/5, - select_version/3, extension_value/1 + select_version/3, select_supported_version/2, extension_value/1 ]). %% Encode --export([encode_handshake/2, encode_hello_extensions/1, +-export([encode_handshake/2, encode_hello_extensions/1, encode_extensions/1, encode_extensions/2, encode_client_protocol_negotiation/2, encode_protocols_advertised_on_server/1]). %% Decode --export([decode_handshake/3, decode_hello_extensions/1, +-export([decode_handshake/3, decode_vector/1, decode_hello_extensions/4, decode_extensions/3, decode_server_key/3, decode_client_key/3, decode_suites/2 ]). @@ -71,13 +72,18 @@ premaster_secret/2, premaster_secret/3, premaster_secret/4]). %% Extensions handling --export([client_hello_extensions/5, +-export([client_hello_extensions/6, handle_client_hello_extensions/9, %% Returns server hello extensions handle_server_hello_extensions/9, select_curve/2, select_curve/3, select_hashsign/4, select_hashsign/5, - select_hashsign_algs/3 + select_hashsign_algs/3, empty_extensions/2, add_server_share/3 ]). +-export([get_cert_params/1, + server_name/3, + validation_fun_and_state/9, + handle_path_validation_error/7]). + %%==================================================================== %% Create handshake messages %%==================================================================== @@ -176,7 +182,7 @@ client_certificate_verify(OwnCert, MasterSecret, Version, %% Description: Creates a certificate_request message, called by the server. %%-------------------------------------------------------------------- certificate_request(CipherSuite, CertDbHandle, CertDbRef, HashSigns, Version) -> - Types = certificate_types(ssl_cipher_format:suite_definition(CipherSuite), Version), + Types = certificate_types(ssl_cipher_format:suite_bin_to_map(CipherSuite), Version), Authorities = certificate_authorities(CertDbHandle, CertDbRef), #certificate_request{ certificate_types = Types, @@ -504,6 +510,21 @@ verify_server_key(#server_key_params{params_bin = EncParams, select_version(RecordCB, ClientVersion, Versions) -> do_select_version(RecordCB, ClientVersion, Versions). + +%% Called by TLS 1.2/1.3 Server when "supported_versions" is present +%% in ClientHello. +%% Input lists are ordered (highest first) +select_supported_version([], _ServerVersions) -> + undefined; +select_supported_version([ClientVersion|T], ServerVersions) -> + case lists:member(ClientVersion, ServerVersions) of + true -> + ClientVersion; + false -> + select_supported_version(T, ServerVersions) + end. + + %%==================================================================== %% Encode handshake %%==================================================================== @@ -517,7 +538,7 @@ encode_handshake(#server_hello{server_version = {Major, Minor}, session_id = Session_ID, cipher_suite = CipherSuite, compression_method = Comp_method, - extensions = #hello_extensions{} = Extensions}, _Version) -> + extensions = Extensions}, _Version) -> SID_length = byte_size(Session_ID), ExtensionsBin = encode_hello_extensions(Extensions), {?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, @@ -567,71 +588,126 @@ encode_handshake(#certificate_verify{signature = BinSig, hashsign_algorithm = Ha encode_handshake(#finished{verify_data = VerifyData}, _Version) -> {?FINISHED, VerifyData}. -encode_hello_extensions(#hello_extensions{} = Extensions) -> - encode_hello_extensions(hello_extensions_list(Extensions), <<>>). -encode_hello_extensions([], <<>>) -> - <<>>; -encode_hello_extensions([], Acc) -> +encode_hello_extensions(Extensions) -> + encode_extensions(hello_extensions_list(Extensions), <<>>). + +encode_extensions(Exts) -> + encode_extensions(Exts, <<>>). + +encode_extensions([], <<>>) -> + <<?UINT16(0)>>; +encode_extensions([], Acc) -> Size = byte_size(Acc), <<?UINT16(Size), Acc/binary>>; - -encode_hello_extensions([#alpn{extension_data = ExtensionData} | Rest], Acc) -> - Len = byte_size(ExtensionData), +encode_extensions([#alpn{extension_data = ExtensionData} | Rest], Acc) -> + Len = byte_size(ExtensionData), ExtLen = Len + 2, - encode_hello_extensions(Rest, <<?UINT16(?ALPN_EXT), ?UINT16(ExtLen), ?UINT16(Len), - ExtensionData/binary, Acc/binary>>); -encode_hello_extensions([#next_protocol_negotiation{extension_data = ExtensionData} | Rest], Acc) -> + encode_extensions(Rest, <<?UINT16(?ALPN_EXT), ?UINT16(ExtLen), ?UINT16(Len), + ExtensionData/binary, Acc/binary>>); +encode_extensions([#next_protocol_negotiation{extension_data = ExtensionData} | Rest], Acc) -> Len = byte_size(ExtensionData), - encode_hello_extensions(Rest, <<?UINT16(?NEXTPROTONEG_EXT), ?UINT16(Len), + encode_extensions(Rest, <<?UINT16(?NEXTPROTONEG_EXT), ?UINT16(Len), ExtensionData/binary, Acc/binary>>); -encode_hello_extensions([#renegotiation_info{renegotiated_connection = undefined} | Rest], Acc) -> - encode_hello_extensions(Rest, Acc); -encode_hello_extensions([#renegotiation_info{renegotiated_connection = ?byte(0) = Info} | Rest], Acc) -> +encode_extensions([#renegotiation_info{renegotiated_connection = undefined} | Rest], Acc) -> + encode_extensions(Rest, Acc); +encode_extensions([#renegotiation_info{renegotiated_connection = ?byte(0) = Info} | Rest], Acc) -> Len = byte_size(Info), - encode_hello_extensions(Rest, <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), Info/binary, Acc/binary>>); + encode_extensions(Rest, <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), Info/binary, Acc/binary>>); -encode_hello_extensions([#renegotiation_info{renegotiated_connection = Info} | Rest], Acc) -> +encode_extensions([#renegotiation_info{renegotiated_connection = Info} | Rest], Acc) -> InfoLen = byte_size(Info), Len = InfoLen +1, - encode_hello_extensions(Rest, <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), ?BYTE(InfoLen), + encode_extensions(Rest, <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), ?BYTE(InfoLen), Info/binary, Acc/binary>>); -encode_hello_extensions([#elliptic_curves{elliptic_curve_list = EllipticCurves} | Rest], Acc) -> +encode_extensions([#elliptic_curves{elliptic_curve_list = EllipticCurves} | Rest], Acc) -> EllipticCurveList = << <<(tls_v1:oid_to_enum(X)):16>> || X <- EllipticCurves>>, ListLen = byte_size(EllipticCurveList), Len = ListLen + 2, - encode_hello_extensions(Rest, <<?UINT16(?ELLIPTIC_CURVES_EXT), + encode_extensions(Rest, <<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), ?UINT16(ListLen), EllipticCurveList/binary, Acc/binary>>); -encode_hello_extensions([#ec_point_formats{ec_point_format_list = ECPointFormats} | Rest], Acc) -> +encode_extensions([#supported_groups{supported_groups = SupportedGroups} | Rest], Acc) -> + + SupportedGroupList = << <<(tls_v1:group_to_enum(X)):16>> || X <- SupportedGroups>>, + ListLen = byte_size(SupportedGroupList), + Len = ListLen + 2, + encode_extensions(Rest, <<?UINT16(?ELLIPTIC_CURVES_EXT), + ?UINT16(Len), ?UINT16(ListLen), + SupportedGroupList/binary, Acc/binary>>); +encode_extensions([#ec_point_formats{ec_point_format_list = ECPointFormats} | Rest], Acc) -> ECPointFormatList = list_to_binary(ECPointFormats), ListLen = byte_size(ECPointFormatList), Len = ListLen + 1, - encode_hello_extensions(Rest, <<?UINT16(?EC_POINT_FORMATS_EXT), + encode_extensions(Rest, <<?UINT16(?EC_POINT_FORMATS_EXT), ?UINT16(Len), ?BYTE(ListLen), ECPointFormatList/binary, Acc/binary>>); -encode_hello_extensions([#srp{username = UserName} | Rest], Acc) -> +encode_extensions([#srp{username = UserName} | Rest], Acc) -> SRPLen = byte_size(UserName), Len = SRPLen + 1, - encode_hello_extensions(Rest, <<?UINT16(?SRP_EXT), ?UINT16(Len), ?BYTE(SRPLen), + encode_extensions(Rest, <<?UINT16(?SRP_EXT), ?UINT16(Len), ?BYTE(SRPLen), UserName/binary, Acc/binary>>); -encode_hello_extensions([#hash_sign_algos{hash_sign_algos = HashSignAlgos} | Rest], Acc) -> +encode_extensions([#hash_sign_algos{hash_sign_algos = HashSignAlgos} | Rest], Acc) -> SignAlgoList = << <<(ssl_cipher:hash_algorithm(Hash)):8, (ssl_cipher:sign_algorithm(Sign)):8>> || {Hash, Sign} <- HashSignAlgos >>, ListLen = byte_size(SignAlgoList), Len = ListLen + 2, - encode_hello_extensions(Rest, <<?UINT16(?SIGNATURE_ALGORITHMS_EXT), + encode_extensions(Rest, <<?UINT16(?SIGNATURE_ALGORITHMS_EXT), ?UINT16(Len), ?UINT16(ListLen), SignAlgoList/binary, Acc/binary>>); -encode_hello_extensions([#sni{hostname = Hostname} | Rest], Acc) -> +encode_extensions([#signature_algorithms{ + signature_scheme_list = SignatureSchemes} | Rest], Acc) -> + SignSchemeList = << <<(ssl_cipher:signature_scheme(SignatureScheme)):16 >> || + SignatureScheme <- SignatureSchemes >>, + ListLen = byte_size(SignSchemeList), + Len = ListLen + 2, + encode_extensions(Rest, <<?UINT16(?SIGNATURE_ALGORITHMS_EXT), + ?UINT16(Len), ?UINT16(ListLen), SignSchemeList/binary, Acc/binary>>); +encode_extensions([#signature_algorithms_cert{ + signature_scheme_list = SignatureSchemes} | Rest], Acc) -> + SignSchemeList = << <<(ssl_cipher:signature_scheme(SignatureScheme)):16 >> || + SignatureScheme <- SignatureSchemes >>, + ListLen = byte_size(SignSchemeList), + Len = ListLen + 2, + encode_extensions(Rest, <<?UINT16(?SIGNATURE_ALGORITHMS_CERT_EXT), + ?UINT16(Len), ?UINT16(ListLen), SignSchemeList/binary, Acc/binary>>); +encode_extensions([#sni{hostname = Hostname} | Rest], Acc) -> HostLen = length(Hostname), HostnameBin = list_to_binary(Hostname), % Hostname type (1 byte) + Hostname length (2 bytes) + Hostname (HostLen bytes) ServerNameLength = 1 + 2 + HostLen, % ServerNameListSize (2 bytes) + ServerNameLength ExtLength = 2 + ServerNameLength, - encode_hello_extensions(Rest, <<?UINT16(?SNI_EXT), ?UINT16(ExtLength), - ?UINT16(ServerNameLength), - ?BYTE(?SNI_NAMETYPE_HOST_NAME), - ?UINT16(HostLen), HostnameBin/binary, - Acc/binary>>). + encode_extensions(Rest, <<?UINT16(?SNI_EXT), ?UINT16(ExtLength), + ?UINT16(ServerNameLength), + ?BYTE(?SNI_NAMETYPE_HOST_NAME), + ?UINT16(HostLen), HostnameBin/binary, + Acc/binary>>); +encode_extensions([#client_hello_versions{versions = Versions0} | Rest], Acc) -> + Versions = encode_versions(Versions0), + VerLen = byte_size(Versions), + Len = VerLen + 1, + encode_extensions(Rest, <<?UINT16(?SUPPORTED_VERSIONS_EXT), + ?UINT16(Len), ?BYTE(VerLen), Versions/binary, Acc/binary>>); +encode_extensions([#server_hello_selected_version{selected_version = Version0} | Rest], Acc) -> + Version = encode_versions([Version0]), + Len = byte_size(Version), %% 2 + encode_extensions(Rest, <<?UINT16(?SUPPORTED_VERSIONS_EXT), + ?UINT16(Len), Version/binary, Acc/binary>>); +encode_extensions([#key_share_client_hello{client_shares = ClientShares0} | Rest], Acc) -> + ClientShares = encode_client_shares(ClientShares0), + ClientSharesLen = byte_size(ClientShares), + Len = ClientSharesLen + 2, + encode_extensions(Rest, <<?UINT16(?KEY_SHARE_EXT), + ?UINT16(Len), ?UINT16(ClientSharesLen), + ClientShares/binary, Acc/binary>>); +encode_extensions([#key_share_server_hello{server_share = ServerShare0} | Rest], Acc) -> + ServerShare = encode_key_share_entry(ServerShare0), + Len = byte_size(ServerShare), + encode_extensions(Rest, <<?UINT16(?KEY_SHARE_EXT), + ?UINT16(Len), ServerShare/binary, Acc/binary>>); +encode_extensions([#key_share_hello_retry_request{selected_group = Group0} | Rest], Acc) -> + Group = tls_v1:group_to_enum(Group0), + encode_extensions(Rest, <<?UINT16(?KEY_SHARE_EXT), + ?UINT16(2), ?UINT16(Group), Acc/binary>>). + encode_client_protocol_negotiation(undefined, _) -> undefined; @@ -657,7 +733,7 @@ decode_handshake(_, ?NEXT_PROTOCOL, <<?BYTE(SelectedProtocolLength), ?BYTE(PaddingLength), _Padding:PaddingLength/binary>>) -> #next_protocol{selected_protocol = SelectedProtocol}; -decode_handshake(_Version, ?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, +decode_handshake(Version, ?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, ?BYTE(SID_length), Session_ID:SID_length/binary, Cipher_suite:2/binary, ?BYTE(Comp_method)>>) -> #server_hello{ @@ -666,14 +742,13 @@ decode_handshake(_Version, ?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:3 session_id = Session_ID, cipher_suite = Cipher_suite, compression_method = Comp_method, - extensions = #hello_extensions{}}; + extensions = empty_extensions(Version, server_hello)}; -decode_handshake(_Version, ?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, +decode_handshake(Version, ?SERVER_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, ?BYTE(SID_length), Session_ID:SID_length/binary, Cipher_suite:2/binary, ?BYTE(Comp_method), ?UINT16(ExtLen), Extensions:ExtLen/binary>>) -> - - HelloExtensions = decode_hello_extensions(Extensions), + HelloExtensions = decode_hello_extensions(Extensions, Version, {Major, Minor}, server_hello), #server_hello{ server_version = {Major,Minor}, @@ -716,17 +791,49 @@ decode_handshake(_Version, ?FINISHED, VerifyData) -> decode_handshake(_, Message, _) -> throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, {unknown_or_malformed_handshake, Message})). + +%%-------------------------------------------------------------------- +-spec decode_vector(binary()) -> binary(). +%% +%% Description: Remove length tag from TLS Vector type. Needed +%% for client hello when extensions in older versions may be empty. +%% +%%-------------------------------------------------------------------- +decode_vector(<<>>) -> + <<>>; +decode_vector(<<?UINT16(Len), Vector:Len/binary>>) -> + Vector. + +%%-------------------------------------------------------------------- +-spec decode_hello_extensions(binary(), ssl_record:ssl_version(), + ssl_record:ssl_version(), atom()) -> map(). +%% +%% Description: Decodes TLS hello extensions +%%-------------------------------------------------------------------- +decode_hello_extensions(Extensions, LocalVersion, LegacyVersion, MessageType0) -> + %% Convert legacy atoms + MessageType = + case MessageType0 of + client -> client_hello; + server -> server_hello; + T -> T + end, + %% RFC 8446 - 4.2.1 + %% Servers MUST be prepared to receive ClientHellos that include this extension but + %% do not include 0x0304 in the list of versions. + %% Clients MUST check for this extension prior to processing the rest of the + %% ServerHello (although they will have to parse the ServerHello in order to read + %% the extension). + Version = process_supported_versions_extension(Extensions, LocalVersion, LegacyVersion), + decode_extensions(Extensions, Version, MessageType, empty_extensions(Version, MessageType)). + %%-------------------------------------------------------------------- --spec decode_hello_extensions({client, binary()} | binary()) -> #hello_extensions{}. +-spec decode_extensions(binary(),tuple(), atom()) -> map(). %% %% Description: Decodes TLS hello extensions %%-------------------------------------------------------------------- -decode_hello_extensions({client, <<>>}) -> - #hello_extensions{}; -decode_hello_extensions({client, <<?UINT16(ExtLen), Extensions:ExtLen/binary>>}) -> - decode_hello_extensions(Extensions); -decode_hello_extensions(Extensions) -> - dec_hello_extensions(Extensions, #hello_extensions{}). +decode_extensions(Extensions, Version, MessageType) -> + decode_extensions(Extensions, Version, MessageType, empty_extensions()). %%-------------------------------------------------------------------- -spec decode_server_key(binary(), ssl:kex_algo(), ssl_record:ssl_version()) -> @@ -776,7 +883,7 @@ available_suites(ServerCert, UserSuites, Version, undefined, Curve) -> filter_unavailable_ecc_suites(Curve, Suites); available_suites(ServerCert, UserSuites, Version, HashSigns, Curve) -> Suites = available_suites(ServerCert, UserSuites, Version, undefined, Curve), - filter_hashsigns(Suites, [ssl_cipher_format:suite_definition(Suite) || Suite <- Suites], HashSigns, + filter_hashsigns(Suites, [ssl_cipher_format:suite_bin_to_map(Suite) || Suite <- Suites], HashSigns, Version, []). available_signature_algs(undefined, _) -> @@ -936,88 +1043,216 @@ premaster_secret(EncSecret, #{algorithm := rsa} = Engine) -> %%==================================================================== %% Extensions handling %%==================================================================== -client_hello_extensions(Version, CipherSuites, - #ssl_options{signature_algs = SupportedHashSigns, - eccs = SupportedECCs} = SslOpts, ConnectionStates, Renegotiation) -> - {EcPointFormats, EllipticCurves} = - case advertises_ec_ciphers(lists:map(fun ssl_cipher_format:suite_definition/1, CipherSuites)) of - true -> - client_ecc_extensions(SupportedECCs); - false -> - {undefined, undefined} - end, +client_hello_extensions(Version, CipherSuites, SslOpts, ConnectionStates, Renegotiation, KeyShare) -> + HelloExtensions0 = add_tls12_extensions(Version, SslOpts, ConnectionStates, Renegotiation), + HelloExtensions1 = add_common_extensions(Version, HelloExtensions0, CipherSuites, SslOpts), + maybe_add_tls13_extensions(Version, HelloExtensions1, SslOpts, KeyShare). + + +add_tls12_extensions(_Version, + SslOpts, + ConnectionStates, + Renegotiation) -> SRP = srp_user(SslOpts), + #{renegotiation_info => renegotiation_info(tls_record, client, + ConnectionStates, Renegotiation), + srp => SRP, + alpn => encode_alpn(SslOpts#ssl_options.alpn_advertised_protocols, Renegotiation), + next_protocol_negotiation => + encode_client_protocol_negotiation(SslOpts#ssl_options.next_protocol_selector, + Renegotiation), + sni => sni(SslOpts#ssl_options.server_name_indication) + }. + - #hello_extensions{ - renegotiation_info = renegotiation_info(tls_record, client, - ConnectionStates, Renegotiation), - srp = SRP, - signature_algs = available_signature_algs(SupportedHashSigns, Version), - ec_point_formats = EcPointFormats, - elliptic_curves = EllipticCurves, - alpn = encode_alpn(SslOpts#ssl_options.alpn_advertised_protocols, Renegotiation), - next_protocol_negotiation = - encode_client_protocol_negotiation(SslOpts#ssl_options.next_protocol_selector, - Renegotiation), - sni = sni(SslOpts#ssl_options.server_name_indication)}. +add_common_extensions({3,4}, + HelloExtensions, + _CipherSuites, + #ssl_options{eccs = SupportedECCs, + supported_groups = Groups, + signature_algs = SignatureSchemes}) -> + {EcPointFormats, _} = + client_ecc_extensions(SupportedECCs), + HelloExtensions#{ec_point_formats => EcPointFormats, + elliptic_curves => Groups, + signature_algs => signature_algs_ext(SignatureSchemes)}; + +add_common_extensions(Version, + HelloExtensions, + CipherSuites, + #ssl_options{eccs = SupportedECCs, + signature_algs = SupportedHashSigns}) -> + + {EcPointFormats, EllipticCurves} = + case advertises_ec_ciphers( + lists:map(fun ssl_cipher_format:suite_bin_to_map/1, + CipherSuites)) of + true -> + client_ecc_extensions(SupportedECCs); + false -> + {undefined, undefined} + end, + HelloExtensions#{ec_point_formats => EcPointFormats, + elliptic_curves => EllipticCurves, + signature_algs => available_signature_algs(SupportedHashSigns, Version)}. + + +maybe_add_tls13_extensions({3,4}, + HelloExtensions0, + #ssl_options{signature_algs_cert = SignatureSchemes, + versions = SupportedVersions}, + KeyShare) -> + HelloExtensions = + HelloExtensions0#{client_hello_versions => + #client_hello_versions{versions = SupportedVersions}, + signature_algs_cert => + signature_algs_cert(SignatureSchemes)}, + maybe_add_key_share(HelloExtensions, KeyShare); +maybe_add_tls13_extensions(_, HelloExtensions, _, _) -> + HelloExtensions. + + +%% TODO: Add support for PSK key establishment + +%% RFC 8446 (TLS 1.3) - 4.2.8. Key Share +%% +%% 4.2.8.1. Diffie-Hellman Parameters +%% Diffie-Hellman [DH76] parameters for both clients and servers are +%% encoded in the opaque key_exchange field of a KeyShareEntry in a +%% KeyShare structure. The opaque value contains the Diffie-Hellman +%% public value (Y = g^X mod p) for the specified group (see [RFC7919] +%% for group definitions) encoded as a big-endian integer and padded to +%% the left with zeros to the size of p in bytes. +%% +%% 4.2.8.2. ECDHE Parameters +%% +%% ECDHE parameters for both clients and servers are encoded in the +%% opaque key_exchange field of a KeyShareEntry in a KeyShare structure. +%% +%% For secp256r1, secp384r1, and secp521r1, the contents are the +%% serialized value of the following struct: +%% +%% struct { +%% uint8 legacy_form = 4; +%% opaque X[coordinate_length]; +%% opaque Y[coordinate_length]; +%% } UncompressedPointRepresentation; +%% +%% X and Y, respectively, are the binary representations of the x and y +%% values in network byte order. There are no internal length markers, +%% so each number representation occupies as many octets as implied by +%% the curve parameters. For P-256, this means that each of X and Y use +%% 32 octets, padded on the left by zeros if necessary. For P-384, they +%% take 48 octets each. For P-521, they take 66 octets each. +maybe_add_key_share(HelloExtensions, undefined) -> + HelloExtensions; +maybe_add_key_share(HelloExtensions, KeyShare) -> + #key_share_client_hello{client_shares = ClientShares0} = KeyShare, + %% Keep only public keys + ClientShares = lists:map(fun kse_remove_private_key/1, ClientShares0), + HelloExtensions#{key_share => #key_share_client_hello{ + client_shares = ClientShares}}. + +add_server_share(server_hello, Extensions, KeyShare) -> + #key_share_server_hello{server_share = ServerShare0} = KeyShare, + %% Keep only public keys + ServerShare = kse_remove_private_key(ServerShare0), + Extensions#{key_share => #key_share_server_hello{ + server_share = ServerShare}}; +add_server_share(hello_retry_request, Extensions, + #key_share_server_hello{ + server_share = #key_share_entry{group = Group}}) -> + Extensions#{key_share => #key_share_hello_retry_request{ + selected_group = Group}}. + + +kse_remove_private_key(#key_share_entry{ + group = Group, + key_exchange = + #'ECPrivateKey'{publicKey = PublicKey}}) -> + #key_share_entry{ + group = Group, + key_exchange = PublicKey}; +kse_remove_private_key(#key_share_entry{ + group = Group, + key_exchange = + {PublicKey, _}}) -> + #key_share_entry{ + group = Group, + key_exchange = PublicKey}. + +signature_algs_ext(undefined) -> + undefined; +signature_algs_ext(SignatureSchemes0) -> + %% The SSL option signature_algs contains both hash-sign algorithms (tuples) and + %% signature schemes (atoms) if TLS 1.3 is configured. + %% Filter out all hash-sign tuples when creating the signature_algs extension. + %% (TLS 1.3 specific record type) + SignatureSchemes = lists:filter(fun is_atom/1, SignatureSchemes0), + #signature_algorithms{signature_scheme_list = SignatureSchemes}. + +signature_algs_cert(undefined) -> + undefined; +signature_algs_cert(SignatureSchemes) -> + #signature_algorithms_cert{signature_scheme_list = SignatureSchemes}. handle_client_hello_extensions(RecordCB, Random, ClientCipherSuites, - #hello_extensions{renegotiation_info = Info, - srp = SRP, - ec_point_formats = ECCFormat, - alpn = ALPN, - next_protocol_negotiation = NextProtocolNegotiation}, Version, + Exts, Version, #ssl_options{secure_renegotiate = SecureRenegotation, alpn_preferred_protocols = ALPNPreferredProtocols} = Opts, #session{cipher_suite = NegotiatedCipherSuite, compression_method = Compression} = Session0, ConnectionStates0, Renegotiation) -> - Session = handle_srp_extension(SRP, Session0), - ConnectionStates = handle_renegotiation_extension(server, RecordCB, Version, Info, - Random, NegotiatedCipherSuite, + Session = handle_srp_extension(maps:get(srp, Exts, undefined), Session0), + ConnectionStates = handle_renegotiation_extension(server, RecordCB, Version, maps:get(renegotiation_info, Exts, undefined), + Random, NegotiatedCipherSuite, ClientCipherSuites, Compression, - ConnectionStates0, Renegotiation, SecureRenegotation), - - ServerHelloExtensions = #hello_extensions{ - renegotiation_info = renegotiation_info(RecordCB, server, + ConnectionStates0, Renegotiation, SecureRenegotation), + + Empty = empty_extensions(Version, server_hello), + ServerHelloExtensions = Empty#{renegotiation_info => renegotiation_info(RecordCB, server, ConnectionStates, Renegotiation), - ec_point_formats = server_ecc_extension(Version, ECCFormat) - }, - + ec_point_formats => server_ecc_extension(Version, + maps:get(ec_point_formats, Exts, undefined)) + }, + %% If we receive an ALPN extension and have ALPN configured for this connection, %% we handle it. Otherwise we check for the NPN extension. + ALPN = maps:get(alpn, Exts, undefined), if ALPN =/= undefined, ALPNPreferredProtocols =/= undefined -> Protocol = handle_alpn_extension(ALPNPreferredProtocols, decode_alpn(ALPN)), {Session, ConnectionStates, Protocol, - ServerHelloExtensions#hello_extensions{alpn=encode_alpn([Protocol], Renegotiation)}}; + ServerHelloExtensions#{alpn => encode_alpn([Protocol], Renegotiation)}}; true -> - ProtocolsToAdvertise = handle_next_protocol_extension(NextProtocolNegotiation, Renegotiation, Opts), + NextProtocolNegotiation = maps:get(next_protocol_negotiation, Exts, undefined), + ProtocolsToAdvertise = handle_next_protocol_extension(NextProtocolNegotiation, Renegotiation, Opts), {Session, ConnectionStates, undefined, - ServerHelloExtensions#hello_extensions{next_protocol_negotiation= - encode_protocols_advertised_on_server(ProtocolsToAdvertise)}} + ServerHelloExtensions#{next_protocol_negotiation => + encode_protocols_advertised_on_server(ProtocolsToAdvertise)}} end. handle_server_hello_extensions(RecordCB, Random, CipherSuite, Compression, - #hello_extensions{renegotiation_info = Info, - alpn = ALPN, - next_protocol_negotiation = NextProtocolNegotiation}, Version, + Exts, Version, #ssl_options{secure_renegotiate = SecureRenegotation, next_protocol_selector = NextProtoSelector}, ConnectionStates0, Renegotiation) -> - ConnectionStates = handle_renegotiation_extension(client, RecordCB, Version, Info, Random, + ConnectionStates = handle_renegotiation_extension(client, RecordCB, Version, + maps:get(renegotiation_info, Exts, undefined), Random, CipherSuite, undefined, Compression, ConnectionStates0, Renegotiation, SecureRenegotation), %% If we receive an ALPN extension then this is the protocol selected, %% otherwise handle the NPN extension. + ALPN = maps:get(alpn, Exts, undefined), case decode_alpn(ALPN) of %% ServerHello contains exactly one protocol: the one selected. %% We also ignore the ALPN extension during renegotiation (see encode_alpn/2). [Protocol] when not Renegotiation -> {ConnectionStates, alpn, Protocol}; undefined -> + NextProtocolNegotiation = maps:get(next_protocol_negotiation, Exts, undefined), Protocol = handle_next_protocol(NextProtocolNegotiation, NextProtoSelector, Renegotiation), {ConnectionStates, npn, Protocol}; {error, Reason} -> @@ -1062,26 +1297,50 @@ select_hashsign(_, _, KeyExAlgo, _, _Version) when KeyExAlgo == dh_anon; {null, anon}; %% The signature_algorithms extension was introduced with TLS 1.2. Ignore it if we have %% negotiated a lower version. -select_hashsign(HashSigns, Cert, KeyExAlgo, - undefined, {Major, Minor} = Version) when Major >= 3 andalso Minor >= 3-> - select_hashsign(HashSigns, Cert, KeyExAlgo, tls_v1:default_signature_algs(Version), Version); -select_hashsign(#hash_sign_algos{hash_sign_algos = HashSigns}, Cert, KeyExAlgo, SupportedHashSigns, - {Major, Minor}) when Major >= 3 andalso Minor >= 3 -> - #'OTPCertificate'{tbsCertificate = TBSCert} = public_key:pkix_decode_cert(Cert, otp), - #'OTPSubjectPublicKeyInfo'{algorithm = {_, SubjAlgo, _}} = - TBSCert#'OTPTBSCertificate'.subjectPublicKeyInfo, - - SubSign = sign_algo(SubjAlgo), - - case lists:filter(fun({_, S} = Algos) when S == SubSign -> - is_acceptable_hash_sign(Algos, KeyExAlgo, SupportedHashSigns); - (_) -> - false - end, HashSigns) of - [] -> - ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm); - [HashSign | _] -> - HashSign +select_hashsign({ClientHashSigns, ClientSignatureSchemes}, + Cert, KeyExAlgo, undefined, {Major, Minor} = Version) + when Major >= 3 andalso Minor >= 3-> + select_hashsign({ClientHashSigns, ClientSignatureSchemes}, Cert, KeyExAlgo, + tls_v1:default_signature_algs(Version), Version); +select_hashsign({#hash_sign_algos{hash_sign_algos = ClientHashSigns}, + ClientSignatureSchemes0}, + Cert, KeyExAlgo, SupportedHashSigns, {Major, Minor}) + when Major >= 3 andalso Minor >= 3 -> + ClientSignatureSchemes = get_signature_scheme(ClientSignatureSchemes0), + {SignAlgo0, Param, PublicKeyAlgo0} = get_cert_params(Cert), + SignAlgo = sign_algo(SignAlgo0), + PublicKeyAlgo = public_key_algo(PublicKeyAlgo0), + + %% RFC 5246 (TLS 1.2) + %% If the client provided a "signature_algorithms" extension, then all + %% certificates provided by the server MUST be signed by a + %% hash/signature algorithm pair that appears in that extension. + %% + %% RFC 8446 (TLS 1.3) + %% TLS 1.3 provides two extensions for indicating which signature + %% algorithms may be used in digital signatures. The + %% "signature_algorithms_cert" extension applies to signatures in + %% certificates and the "signature_algorithms" extension, which + %% originally appeared in TLS 1.2, applies to signatures in + %% CertificateVerify messages. + %% + %% If no "signature_algorithms_cert" extension is + %% present, then the "signature_algorithms" extension also applies to + %% signatures appearing in certificates. + case is_supported_sign(SignAlgo, Param, ClientHashSigns, ClientSignatureSchemes) of + true -> + case lists:filter(fun({_, S} = Algos) when S == PublicKeyAlgo -> + is_acceptable_hash_sign(Algos, KeyExAlgo, SupportedHashSigns); + (_) -> + false + end, ClientHashSigns) of + [] -> + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm); + [HashSign | _] -> + HashSign + end; + false -> + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm) end; select_hashsign(_, Cert, _, _, Version) -> #'OTPCertificate'{tbsCertificate = TBSCert} = public_key:pkix_decode_cert(Cert, otp), @@ -1095,21 +1354,23 @@ select_hashsign(_, Cert, _, _, Version) -> %% %% Description: Handles signature algorithms selection for certificate requests (client) %%-------------------------------------------------------------------- -select_hashsign(#certificate_request{hashsign_algorithms = #hash_sign_algos{hash_sign_algos = HashSigns}, - certificate_types = Types}, Cert, SupportedHashSigns, +select_hashsign(#certificate_request{ + hashsign_algorithms = #hash_sign_algos{ + hash_sign_algos = HashSigns}, + certificate_types = Types}, + Cert, + SupportedHashSigns, {Major, Minor}) when Major >= 3 andalso Minor >= 3-> - #'OTPCertificate'{tbsCertificate = TBSCert} = public_key:pkix_decode_cert(Cert, otp), - #'OTPCertificate'{tbsCertificate = TBSCert, - signatureAlgorithm = {_,SignAlgo, _}} = public_key:pkix_decode_cert(Cert, otp), - #'OTPSubjectPublicKeyInfo'{algorithm = {_, SubjAlgo, _}} = - TBSCert#'OTPTBSCertificate'.subjectPublicKeyInfo, - - Sign = sign_algo(SignAlgo), - SubSign = sign_algo(SubjAlgo), - - case is_acceptable_cert_type(SubSign, HashSigns, Types) andalso is_supported_sign(Sign, HashSigns) of + {SignAlgo0, Param, PublicKeyAlgo0} = get_cert_params(Cert), + SignAlgo = sign_algo(SignAlgo0), + PublicKeyAlgo = public_key_algo(PublicKeyAlgo0), + + case is_acceptable_cert_type(PublicKeyAlgo, Types) andalso + %% certificate_request has no "signature_algorithms_cert" + %% extension in TLS 1.2. + is_supported_sign(SignAlgo, Param, HashSigns, undefined) of true -> - case lists:filter(fun({_, S} = Algos) when S == SubSign -> + case lists:filter(fun({_, S} = Algos) when S == PublicKeyAlgo -> is_acceptable_hash_sign(Algos, SupportedHashSigns); (_) -> false @@ -1122,8 +1383,38 @@ select_hashsign(#certificate_request{hashsign_algorithms = #hash_sign_algos{hash false -> ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm) end; -select_hashsign(#certificate_request{}, Cert, _, Version) -> - select_hashsign(undefined, Cert, undefined, [], Version). +select_hashsign(#certificate_request{certificate_types = Types}, Cert, _, Version) -> + {_, _, PublicKeyAlgo0} = get_cert_params(Cert), + PublicKeyAlgo = public_key_algo(PublicKeyAlgo0), + + %% Check cert even for TLS 1.0/1.1 + case is_acceptable_cert_type(PublicKeyAlgo, Types) of + true -> + select_hashsign(undefined, Cert, undefined, [], Version); + false -> + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm) + end. + + +%% Gets the relevant parameters of a certificate: +%% - signature algorithm +%% - parameters of the signature algorithm +%% - public key algorithm (key type) +get_cert_params(Cert) -> + #'OTPCertificate'{tbsCertificate = TBSCert, + signatureAlgorithm = + {_,SignAlgo, Param}} = public_key:pkix_decode_cert(Cert, otp), + #'OTPSubjectPublicKeyInfo'{algorithm = {_, PublicKeyAlgo, _}} = + TBSCert#'OTPTBSCertificate'.subjectPublicKeyInfo, + {SignAlgo, Param, PublicKeyAlgo}. + + +get_signature_scheme(undefined) -> + undefined; +get_signature_scheme(#signature_algorithms_cert{ + signature_scheme_list = ClientSignatureSchemes}) -> + ClientSignatureSchemes. + %%-------------------------------------------------------------------- -spec select_hashsign_algs({atom(), atom()}| undefined, oid(), ssl_record:ssl_version()) -> @@ -1172,6 +1463,8 @@ extension_value(#ec_point_formats{ec_point_format_list = List}) -> List; extension_value(#elliptic_curves{elliptic_curve_list = List}) -> List; +extension_value(#supported_groups{supported_groups = SupportedGroups}) -> + SupportedGroups; extension_value(#hash_sign_algos{hash_sign_algos = Algos}) -> Algos; extension_value(#alpn{extension_data = Data}) -> @@ -1192,33 +1485,30 @@ int_to_bin(I) -> L = (length(integer_to_list(I, 16)) + 1) div 2, <<I:(L*8)>>. -certificate_types(_, {N, M}) when N >= 3 andalso M >= 3 -> - case proplists:get_bool(ecdsa, - proplists:get_value(public_keys, crypto:supports())) of - true -> - <<?BYTE(?ECDSA_SIGN), ?BYTE(?RSA_SIGN), ?BYTE(?DSS_SIGN)>>; - false -> - <<?BYTE(?RSA_SIGN), ?BYTE(?DSS_SIGN)>> - end; - -certificate_types(#{key_exchange := KeyExchange}, _) when KeyExchange == rsa; - KeyExchange == dh_rsa; - KeyExchange == dhe_rsa; - KeyExchange == ecdhe_rsa -> - <<?BYTE(?RSA_SIGN)>>; - -certificate_types(#{key_exchange := KeyExchange}, _) when KeyExchange == dh_dss; - KeyExchange == dhe_dss; - KeyExchange == srp_dss -> - <<?BYTE(?DSS_SIGN)>>; - -certificate_types(#{key_exchange := KeyExchange}, _) when KeyExchange == dh_ecdsa; - KeyExchange == dhe_ecdsa; - KeyExchange == ecdh_ecdsa; - KeyExchange == ecdhe_ecdsa -> - <<?BYTE(?ECDSA_SIGN)>>; +%% TLS 1.0+ +%% The end-entity certificate provided by the client MUST contain a +%% key that is compatible with certificate_types. +certificate_types(_, {N, M}) when N >= 3 andalso M >= 1 -> + ECDSA = supported_cert_type_or_empty(ecdsa, ?ECDSA_SIGN), + RSA = supported_cert_type_or_empty(rsa, ?RSA_SIGN), + DSS = supported_cert_type_or_empty(dss, ?DSS_SIGN), + <<ECDSA/binary,RSA/binary,DSS/binary>>; +%% SSL 3.0 certificate_types(_, _) -> - <<?BYTE(?RSA_SIGN)>>. + RSA = supported_cert_type_or_empty(rsa, ?RSA_SIGN), + DSS = supported_cert_type_or_empty(dss, ?DSS_SIGN), + <<RSA/binary,DSS/binary>>. + +%% Returns encoded certificate_type if algorithm is supported +supported_cert_type_or_empty(Algo, Type) -> + case proplists:get_bool( + Algo, + proplists:get_value(public_keys, crypto:supports())) of + true -> + <<?BYTE(Type)>>; + false -> + <<>> + end. certificate_authorities(CertDbHandle, CertDbRef) -> Authorities = certificate_authorities_from_db(CertDbHandle, CertDbRef), @@ -1761,16 +2051,32 @@ encode_alpn(undefined, _) -> encode_alpn(Protocols, _) -> #alpn{extension_data = lists:foldl(fun encode_protocol/2, <<>>, Protocols)}. -hello_extensions_list(#hello_extensions{renegotiation_info = RenegotiationInfo, - srp = SRP, - signature_algs = HashSigns, - ec_point_formats = EcPointFormats, - elliptic_curves = EllipticCurves, - alpn = ALPN, - next_protocol_negotiation = NextProtocolNegotiation, - sni = Sni}) -> - [Ext || Ext <- [RenegotiationInfo, SRP, HashSigns, - EcPointFormats, EllipticCurves, ALPN, NextProtocolNegotiation, Sni], Ext =/= undefined]. + +encode_versions(Versions) -> + encode_versions(lists:reverse(Versions), <<>>). +%% +encode_versions([], Acc) -> + Acc; +encode_versions([{M,N}|T], Acc) -> + encode_versions(T, <<?BYTE(M),?BYTE(N),Acc/binary>>). + +encode_client_shares(ClientShares) -> + encode_client_shares(ClientShares, <<>>). +%% +encode_client_shares([], Acc) -> + Acc; +encode_client_shares([KeyShareEntry0|T], Acc) -> + KeyShareEntry = encode_key_share_entry(KeyShareEntry0), + encode_client_shares(T, <<Acc/binary,KeyShareEntry/binary>>). + +encode_key_share_entry(#key_share_entry{ + group = Group, + key_exchange = KeyExchange}) -> + Len = byte_size(KeyExchange), + <<?UINT16((tls_v1:group_to_enum(Group))),?UINT16(Len),KeyExchange/binary>>. + +hello_extensions_list(HelloExtensions) -> + [Ext || {_, Ext} <- maps:to_list(HelloExtensions), Ext =/= undefined]. %%-------------Decode handshakes--------------------------------- dec_server_key(<<?UINT16(PLen), P:PLen/binary, @@ -1910,16 +2216,60 @@ dec_server_key_signature(Params, <<?UINT16(Len), Signature:Len/binary>>, _) -> dec_server_key_signature(_, _, _) -> throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, failed_to_decrypt_server_key_sign)). -dec_hello_extensions(<<>>, Acc) -> +%% Processes a ClientHello/ServerHello message and returns the version to be used +%% in the decoding functions. The following rules apply: +%% - IF supported_versions extension is absent: +%% RETURN the lowest of (LocalVersion and LegacyVersion) +%% - IF supported_versions estension is present: +%% RETURN the lowest of (LocalVersion and first element of supported versions) +process_supported_versions_extension(<<>>, LocalVersion, LegacyVersion) + when LegacyVersion =< LocalVersion -> + LegacyVersion; +process_supported_versions_extension(<<>>, LocalVersion, _LegacyVersion) -> + LocalVersion; +process_supported_versions_extension(<<?UINT16(?SUPPORTED_VERSIONS_EXT), ?UINT16(Len), + ExtData:Len/binary, _Rest/binary>>, + LocalVersion, _LegacyVersion) when Len > 2 -> + <<?BYTE(_),Versions0/binary>> = ExtData, + [Highest|_] = decode_versions(Versions0), + if Highest =< LocalVersion -> + Highest; + true -> + LocalVersion + end; +process_supported_versions_extension(<<?UINT16(?SUPPORTED_VERSIONS_EXT), ?UINT16(Len), + ?BYTE(Major),?BYTE(Minor), _Rest/binary>>, + LocalVersion, _LegacyVersion) when Len =:= 2 -> + SelectedVersion = {Major, Minor}, + if SelectedVersion =< LocalVersion -> + SelectedVersion; + true -> + LocalVersion + end; +process_supported_versions_extension(<<?UINT16(_), ?UINT16(Len), + _ExtData:Len/binary, Rest/binary>>, + LocalVersion, LegacyVersion) -> + process_supported_versions_extension(Rest, LocalVersion, LegacyVersion); +%% Tolerate protocol encoding errors and skip parsing the rest of the extension. +process_supported_versions_extension(_, LocalVersion, LegacyVersion) + when LegacyVersion =< LocalVersion -> + LegacyVersion; +process_supported_versions_extension(_, LocalVersion, _) -> + LocalVersion. + +decode_extensions(<<>>, _Version, _MessageType, Acc) -> Acc; -dec_hello_extensions(<<?UINT16(?ALPN_EXT), ?UINT16(ExtLen), ?UINT16(Len), ExtensionData:Len/binary, Rest/binary>>, Acc) - when Len + 2 =:= ExtLen -> +decode_extensions(<<?UINT16(?ALPN_EXT), ?UINT16(ExtLen), ?UINT16(Len), + ExtensionData:Len/binary, Rest/binary>>, Version, MessageType, Acc) + when Len + 2 =:= ExtLen -> ALPN = #alpn{extension_data = ExtensionData}, - dec_hello_extensions(Rest, Acc#hello_extensions{alpn = ALPN}); -dec_hello_extensions(<<?UINT16(?NEXTPROTONEG_EXT), ?UINT16(Len), ExtensionData:Len/binary, Rest/binary>>, Acc) -> + decode_extensions(Rest, Version, MessageType, Acc#{alpn => ALPN}); +decode_extensions(<<?UINT16(?NEXTPROTONEG_EXT), ?UINT16(Len), + ExtensionData:Len/binary, Rest/binary>>, Version, MessageType, Acc) -> NextP = #next_protocol_negotiation{extension_data = ExtensionData}, - dec_hello_extensions(Rest, Acc#hello_extensions{next_protocol_negotiation = NextP}); -dec_hello_extensions(<<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), Info:Len/binary, Rest/binary>>, Acc) -> + decode_extensions(Rest, Version, MessageType, Acc#{next_protocol_negotiation => NextP}); +decode_extensions(<<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), + Info:Len/binary, Rest/binary>>, Version, MessageType, Acc) -> RenegotiateInfo = case Len of 1 -> % Initial handshake Info; % should be <<0>> will be matched in handle_renegotiation_info @@ -1928,25 +2278,54 @@ dec_hello_extensions(<<?UINT16(?RENEGOTIATION_EXT), ?UINT16(Len), Info:Len/binar <<?BYTE(VerifyLen), VerifyInfo/binary>> = Info, VerifyInfo end, - dec_hello_extensions(Rest, Acc#hello_extensions{renegotiation_info = - #renegotiation_info{renegotiated_connection = - RenegotiateInfo}}); + decode_extensions(Rest, Version, MessageType, + Acc#{renegotiation_info => + #renegotiation_info{renegotiated_connection = + RenegotiateInfo}}); -dec_hello_extensions(<<?UINT16(?SRP_EXT), ?UINT16(Len), ?BYTE(SRPLen), SRP:SRPLen/binary, Rest/binary>>, Acc) +decode_extensions(<<?UINT16(?SRP_EXT), ?UINT16(Len), ?BYTE(SRPLen), + SRP:SRPLen/binary, Rest/binary>>, Version, MessageType, Acc) when Len == SRPLen + 1 -> - dec_hello_extensions(Rest, Acc#hello_extensions{srp = #srp{username = SRP}}); + decode_extensions(Rest, Version, MessageType, Acc#{srp => #srp{username = SRP}}); -dec_hello_extensions(<<?UINT16(?SIGNATURE_ALGORITHMS_EXT), ?UINT16(Len), - ExtData:Len/binary, Rest/binary>>, Acc) -> +decode_extensions(<<?UINT16(?SIGNATURE_ALGORITHMS_EXT), ?UINT16(Len), + ExtData:Len/binary, Rest/binary>>, Version, MessageType, Acc) + when Version < {3,4} -> SignAlgoListLen = Len - 2, <<?UINT16(SignAlgoListLen), SignAlgoList/binary>> = ExtData, HashSignAlgos = [{ssl_cipher:hash_algorithm(Hash), ssl_cipher:sign_algorithm(Sign)} || <<?BYTE(Hash), ?BYTE(Sign)>> <= SignAlgoList], - dec_hello_extensions(Rest, Acc#hello_extensions{signature_algs = - #hash_sign_algos{hash_sign_algos = HashSignAlgos}}); - -dec_hello_extensions(<<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), - ExtData:Len/binary, Rest/binary>>, Acc) -> + decode_extensions(Rest, Version, MessageType, + Acc#{signature_algs => + #hash_sign_algos{hash_sign_algos = + HashSignAlgos}}); + +decode_extensions(<<?UINT16(?SIGNATURE_ALGORITHMS_EXT), ?UINT16(Len), + ExtData:Len/binary, Rest/binary>>, Version, MessageType, Acc) + when Version =:= {3,4} -> + SignSchemeListLen = Len - 2, + <<?UINT16(SignSchemeListLen), SignSchemeList/binary>> = ExtData, + SignSchemes = [ssl_cipher:signature_scheme(SignScheme) || + <<?UINT16(SignScheme)>> <= SignSchemeList], + decode_extensions(Rest, Version, MessageType, + Acc#{signature_algs => + #signature_algorithms{ + signature_scheme_list = SignSchemes}}); + +decode_extensions(<<?UINT16(?SIGNATURE_ALGORITHMS_CERT_EXT), ?UINT16(Len), + ExtData:Len/binary, Rest/binary>>, Version, MessageType, Acc) -> + SignSchemeListLen = Len - 2, + <<?UINT16(SignSchemeListLen), SignSchemeList/binary>> = ExtData, + SignSchemes = [ssl_cipher:signature_scheme(SignScheme) || + <<?UINT16(SignScheme)>> <= SignSchemeList], + decode_extensions(Rest, Version, MessageType, + Acc#{signature_algs_cert => + #signature_algorithms_cert{ + signature_scheme_list = SignSchemes}}); + +decode_extensions(<<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), + ExtData:Len/binary, Rest/binary>>, Version, MessageType, Acc) + when Version < {3,4} -> <<?UINT16(_), EllipticCurveList/binary>> = ExtData, %% Ignore unknown curves Pick = fun(Enum) -> @@ -1958,31 +2337,103 @@ dec_hello_extensions(<<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), end end, EllipticCurves = lists:filtermap(Pick, [ECC || <<ECC:16>> <= EllipticCurveList]), - dec_hello_extensions(Rest, Acc#hello_extensions{elliptic_curves = - #elliptic_curves{elliptic_curve_list = - EllipticCurves}}); -dec_hello_extensions(<<?UINT16(?EC_POINT_FORMATS_EXT), ?UINT16(Len), - ExtData:Len/binary, Rest/binary>>, Acc) -> + decode_extensions(Rest, Version, MessageType, + Acc#{elliptic_curves => + #elliptic_curves{elliptic_curve_list = + EllipticCurves}}); + +decode_extensions(<<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), + ExtData:Len/binary, Rest/binary>>, Version, MessageType, Acc) + when Version =:= {3,4} -> + <<?UINT16(_), GroupList/binary>> = ExtData, + %% Ignore unknown curves + Pick = fun(Enum) -> + case tls_v1:enum_to_group(Enum) of + undefined -> + false; + Group -> + {true, Group} + end + end, + SupportedGroups = lists:filtermap(Pick, [Group || <<Group:16>> <= GroupList]), + decode_extensions(Rest, Version, MessageType, + Acc#{elliptic_curves => + #supported_groups{supported_groups = + SupportedGroups}}); + +decode_extensions(<<?UINT16(?EC_POINT_FORMATS_EXT), ?UINT16(Len), + ExtData:Len/binary, Rest/binary>>, Version, MessageType, Acc) -> <<?BYTE(_), ECPointFormatList/binary>> = ExtData, ECPointFormats = binary_to_list(ECPointFormatList), - dec_hello_extensions(Rest, Acc#hello_extensions{ec_point_formats = - #ec_point_formats{ec_point_format_list = - ECPointFormats}}); + decode_extensions(Rest, Version, MessageType, + Acc#{ec_point_formats => + #ec_point_formats{ec_point_format_list = + ECPointFormats}}); + +decode_extensions(<<?UINT16(?SNI_EXT), ?UINT16(Len), + Rest/binary>>, Version, MessageType, Acc) when Len == 0 -> + decode_extensions(Rest, Version, MessageType, + Acc#{sni => #sni{hostname = ""}}); %% Server may send an empy SNI + +decode_extensions(<<?UINT16(?SNI_EXT), ?UINT16(Len), + ExtData:Len/binary, Rest/binary>>, Version, MessageType, Acc) -> + <<?UINT16(_), NameList/binary>> = ExtData, + decode_extensions(Rest, Version, MessageType, + Acc#{sni => dec_sni(NameList)}); + +decode_extensions(<<?UINT16(?SUPPORTED_VERSIONS_EXT), ?UINT16(Len), + ExtData:Len/binary, Rest/binary>>, Version, MessageType, Acc) when Len > 2 -> + <<?BYTE(_),Versions/binary>> = ExtData, + decode_extensions(Rest, Version, MessageType, + Acc#{client_hello_versions => + #client_hello_versions{ + versions = decode_versions(Versions)}}); + +decode_extensions(<<?UINT16(?SUPPORTED_VERSIONS_EXT), ?UINT16(Len), + ?UINT16(SelectedVersion), Rest/binary>>, Version, MessageType, Acc) + when Len =:= 2, SelectedVersion =:= 16#0304 -> + decode_extensions(Rest, Version, MessageType, + Acc#{server_hello_selected_version => + #server_hello_selected_version{selected_version = + {3,4}}}); + +decode_extensions(<<?UINT16(?KEY_SHARE_EXT), ?UINT16(Len), + ExtData:Len/binary, Rest/binary>>, + Version, MessageType = client_hello, Acc) -> + <<?UINT16(_),ClientShares/binary>> = ExtData, + decode_extensions(Rest, Version, MessageType, + Acc#{key_share => + #key_share_client_hello{ + client_shares = decode_client_shares(ClientShares)}}); + +decode_extensions(<<?UINT16(?KEY_SHARE_EXT), ?UINT16(Len), + ExtData:Len/binary, Rest/binary>>, + Version, MessageType = server_hello, Acc) -> + <<?UINT16(Group),?UINT16(KeyLen),KeyExchange:KeyLen/binary>> = ExtData, + decode_extensions(Rest, Version, MessageType, + Acc#{key_share => + #key_share_server_hello{ + server_share = + #key_share_entry{ + group = tls_v1:enum_to_group(Group), + key_exchange = KeyExchange}}}); + +decode_extensions(<<?UINT16(?KEY_SHARE_EXT), ?UINT16(Len), + ExtData:Len/binary, Rest/binary>>, + Version, MessageType = hello_retry_request, Acc) -> + <<?UINT16(Group)>> = ExtData, + decode_extensions(Rest, Version, MessageType, + Acc#{key_share => + #key_share_hello_retry_request{ + selected_group = tls_v1:enum_to_group(Group)}}); -dec_hello_extensions(<<?UINT16(?SNI_EXT), ?UINT16(Len), Rest/binary>>, Acc) when Len == 0 -> - dec_hello_extensions(Rest, Acc#hello_extensions{sni = #sni{hostname = ""}}); %% Server may send an empy SNI -dec_hello_extensions(<<?UINT16(?SNI_EXT), ?UINT16(Len), - ExtData:Len/binary, Rest/binary>>, Acc) -> - <<?UINT16(_), NameList/binary>> = ExtData, - dec_hello_extensions(Rest, Acc#hello_extensions{sni = dec_sni(NameList)}); %% Ignore data following the ClientHello (i.e., %% extensions) if not understood. - -dec_hello_extensions(<<?UINT16(_), ?UINT16(Len), _Unknown:Len/binary, Rest/binary>>, Acc) -> - dec_hello_extensions(Rest, Acc); +decode_extensions(<<?UINT16(_), ?UINT16(Len), _Unknown:Len/binary, Rest/binary>>, Version, MessageType, Acc) -> + decode_extensions(Rest, Version, MessageType, Acc); %% This theoretically should not happen if the protocol is followed, but if it does it is ignored. -dec_hello_extensions(_, Acc) -> +decode_extensions(_, _, _, Acc) -> Acc. dec_hashsign(<<?BYTE(HashAlgo), ?BYTE(SignAlgo)>>) -> @@ -2000,6 +2451,26 @@ decode_alpn(undefined) -> decode_alpn(#alpn{extension_data=Data}) -> decode_protocols(Data, []). +decode_versions(Versions) -> + decode_versions(Versions, []). +%% +decode_versions(<<>>, Acc) -> + lists:reverse(Acc); +decode_versions(<<?BYTE(M),?BYTE(N),Rest/binary>>, Acc) -> + decode_versions(Rest, [{M,N}|Acc]). + + +decode_client_shares(ClientShares) -> + decode_client_shares(ClientShares, []). +%% +decode_client_shares(<<>>, Acc) -> + lists:reverse(Acc); +decode_client_shares(<<?UINT16(Group),?UINT16(Len),KeyExchange:Len/binary,Rest/binary>>, Acc) -> + decode_client_shares(Rest, [#key_share_entry{ + group = tls_v1:enum_to_group(Group), + key_exchange= KeyExchange + }|Acc]). + decode_next_protocols({next_protocol_negotiation, Protocols}) -> decode_protocols(Protocols, []). @@ -2253,17 +2724,6 @@ handle_srp_extension(undefined, Session) -> handle_srp_extension(#srp{username = Username}, Session) -> Session#session{srp_username = Username}. - -sign_algo(?rsaEncryption) -> - rsa; -sign_algo(?'id-ecPublicKey') -> - ecdsa; -sign_algo(?'id-dsa') -> - dsa; -sign_algo(Alg) -> - {_, Sign} =public_key:pkix_sign_types(Alg), - Sign. - is_acceptable_hash_sign( _, KeyExAlgo, _) when KeyExAlgo == psk; KeyExAlgo == dhe_psk; @@ -2279,15 +2739,80 @@ is_acceptable_hash_sign(Algos,_, SupportedHashSigns) -> is_acceptable_hash_sign(Algos, SupportedHashSigns) -> lists:member(Algos, SupportedHashSigns). -is_acceptable_cert_type(Sign, _HashSigns, Types) -> +is_acceptable_cert_type(Sign, Types) -> lists:member(sign_type(Sign), binary_to_list(Types)). -is_supported_sign(Sign, HashSigns) -> - [] =/= lists:dropwhile(fun({_, S}) when S =/= Sign -> - true; - (_)-> - false - end, HashSigns). +%% signature_algorithms_cert = undefined +is_supported_sign(SignAlgo, _, HashSigns, undefined) -> + lists:member(SignAlgo, HashSigns); + +%% {'SignatureAlgorithm',{1,2,840,113549,1,1,11},'NULL'} +is_supported_sign({Hash, Sign}, 'NULL', _, SignatureSchemes) -> + Fun = fun (Scheme, Acc) -> + {H0, S0, _} = ssl_cipher:scheme_to_components(Scheme), + S1 = case S0 of + rsa_pkcs1 -> rsa; + S -> S + end, + H1 = case H0 of + sha1 -> sha; + H -> H + end, + Acc orelse (Sign =:= S1 andalso + Hash =:= H1) + end, + lists:foldl(Fun, false, SignatureSchemes); + +%% TODO: Implement validation for the curve used in the signature +%% RFC 3279 - 2.2.3 ECDSA Signature Algorithm +%% When the ecdsa-with-SHA1 algorithm identifier appears as the +%% algorithm field in an AlgorithmIdentifier, the encoding MUST omit the +%% parameters field. That is, the AlgorithmIdentifier SHALL be a +%% SEQUENCE of one component: the OBJECT IDENTIFIER ecdsa-with-SHA1. +%% +%% The elliptic curve parameters in the subjectPublicKeyInfo field of +%% the certificate of the issuer SHALL apply to the verification of the +%% signature. +is_supported_sign({Hash, Sign}, _Param, _, SignatureSchemes) -> + Fun = fun (Scheme, Acc) -> + {H0, S0, _} = ssl_cipher:scheme_to_components(Scheme), + S1 = case S0 of + rsa_pkcs1 -> rsa; + S -> S + end, + H1 = case H0 of + sha1 -> sha; + H -> H + end, + Acc orelse (Sign =:= S1 andalso + Hash =:= H1) + end, + lists:foldl(Fun, false, SignatureSchemes). + +%% SupportedPublicKeyAlgorithms PUBLIC-KEY-ALGORITHM-CLASS ::= { +%% dsa | rsa-encryption | dh | kea | ec-public-key } +public_key_algo(?rsaEncryption) -> + rsa; +public_key_algo(?'id-ecPublicKey') -> + ecdsa; +public_key_algo(?'id-dsa') -> + dsa. + +%% SupportedSignatureAlgorithms SIGNATURE-ALGORITHM-CLASS ::= { +%% dsa-with-sha1 | dsaWithSHA1 | md2-with-rsa-encryption | +%% md5-with-rsa-encryption | sha1-with-rsa-encryption | sha-1with-rsa-encryption | +%% sha224-with-rsa-encryption | +%% sha256-with-rsa-encryption | +%% sha384-with-rsa-encryption | +%% sha512-with-rsa-encryption | +%% ecdsa-with-sha1 | +%% ecdsa-with-sha224 | +%% ecdsa-with-sha256 | +%% ecdsa-with-sha384 | +%% ecdsa-with-sha512 } +sign_algo(Alg) -> + public_key:pkix_sign_types(Alg). + sign_type(rsa) -> ?RSA_SIGN; sign_type(dsa) -> @@ -2306,6 +2831,11 @@ client_ecc_extensions(SupportedECCs) -> CryptoSupport = proplists:get_value(public_keys, crypto:supports()), case proplists:get_bool(ecdh, CryptoSupport) of true -> + %% RFC 8422 - 5.1. Client Hello Extensions + %% Clients SHOULD send both the Supported Elliptic Curves Extension and the + %% Supported Point Formats Extension. If the Supported Point Formats + %% Extension is indeed sent, it MUST contain the value 0 (uncompressed) + %% as one of the items in the list of point formats. EcPointFormats = #ec_point_formats{ec_point_format_list = [?ECPOINT_UNCOMPRESSED]}, EllipticCurves = SupportedECCs, {EcPointFormats, EllipticCurves}; @@ -2460,7 +2990,7 @@ handle_renegotiation_info(_RecordCB, ConnectionStates, SecureRenegotation) -> cert_curve(_, _, no_suite) -> {no_curve, no_suite}; cert_curve(Cert, ECCCurve0, CipherSuite) -> - case ssl_cipher_format:suite_definition(CipherSuite) of + case ssl_cipher_format:suite_bin_to_map(CipherSuite) of #{key_exchange := Kex} when Kex == ecdh_ecdsa; Kex == ecdh_rsa -> OtpCert = public_key:pkix_decode_cert(Cert, otp), @@ -2473,4 +3003,51 @@ cert_curve(Cert, ECCCurve0, CipherSuite) -> {ECCCurve0, CipherSuite} end. - +empty_extensions() -> + #{}. + +empty_extensions({3,4}, client_hello) -> + #{ + sni => undefined, + %% max_fragment_length => undefined, + %% status_request => undefined, + elliptic_curves => undefined, + signature_algs => undefined, + %% use_srtp => undefined, + %% heartbeat => undefined, + alpn => undefined, + %% signed_cert_timestamp => undefined, + %% client_cert_type => undefined, + %% server_cert_type => undefined, + %% padding => undefined, + key_share => undefined, + pre_shared_key => undefined, + %% psk_key_exhange_modes => undefined, + %% early_data => undefined, + %% cookie => undefined, + client_hello_versions => undefined, + %% cert_authorities => undefined, + %% post_handshake_auth => undefined, + signature_algs_cert => undefined + }; +empty_extensions({3, 3}, client_hello) -> + Ext = empty_extensions({3,2}, client_hello), + Ext#{signature_algs => undefined}; +empty_extensions(_, client_hello) -> + #{renegotiation_info => undefined, + alpn => undefined, + next_protocol_negotiation => undefined, + srp => undefined, + ec_point_formats => undefined, + elliptic_curves => undefined, + sni => undefined}; +empty_extensions({3,4}, server_hello) -> + #{server_hello_selected_version => undefined, + key_share => undefined, + pre_shared_key => undefined + }; +empty_extensions(_, server_hello) -> + #{renegotiation_info => undefined, + alpn => undefined, + next_protocol_negotiation => undefined, + ec_point_formats => undefined}. diff --git a/lib/ssl/src/ssl_handshake.hrl b/lib/ssl/src/ssl_handshake.hrl index a191fcf766..b248edcaa9 100644 --- a/lib/ssl/src/ssl_handshake.hrl +++ b/lib/ssl/src/ssl_handshake.hrl @@ -47,14 +47,15 @@ srp_username, is_resumable, time_stamp, - ecc + ecc, %% TLS 1.3 Group + sign_alg, %% TLS 1.3 Signature Algorithm + dh_public_value %% TLS 1.3 DH Public Value from peer }). -define(NUM_OF_SESSION_ID_BYTES, 32). % TSL 1.1 & SSL 3 -define(NUM_OF_PREMASTERSECRET_BYTES, 48). --define(DEFAULT_DIFFIE_HELLMAN_GENERATOR, 2). --define(DEFAULT_DIFFIE_HELLMAN_PRIME, - 16#FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3DC2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F83655D23DCA3AD961C62F356208552BB9ED529077096966D670C354E4ABC9804F1746C08CA18217C32905E462E36CE3BE39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9DE2BCBF6955817183995497CEA956AE515D2261898FA051015728E5A8AACAA68FFFFFFFFFFFFFFFF). +-define(DEFAULT_DIFFIE_HELLMAN_GENERATOR, ssl_dh_groups:modp2048_generator()). +-define(DEFAULT_DIFFIE_HELLMAN_PRIME, ssl_dh_groups:modp2048_prime()). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Handsake protocol - RFC 4346 section 7.4 @@ -105,7 +106,11 @@ srp, ec_point_formats, elliptic_curves, - sni + sni, + client_hello_versions, + server_hello_selected_version, + signature_algs_cert, + key_share }). -record(server_hello, { @@ -313,12 +318,12 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -define(SIGNATURE_ALGORITHMS_EXT, 13). --record(hash_sign_algos, { - hash_sign_algos - }). +-record(hash_sign_algos, {hash_sign_algos}). +%% RFC 8446 (TLS 1.3) +-record(signature_algorithms, {signature_scheme_list}). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Application-Layer Protocol Negotiation RFC 7301 +%% RFC 7301 Application-Layer Protocol Negotiation %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -define(ALPN_EXT, 16). @@ -338,9 +343,8 @@ -record(next_protocol, {selected_protocol}). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% ECC Extensions RFC 4492 section 4 and 5 +%% ECC Extensions RFC 8422 section 4 and 5 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -define(ELLIPTIC_CURVES_EXT, 10). -define(EC_POINT_FORMATS_EXT, 11). @@ -348,11 +352,18 @@ elliptic_curve_list }). +%% RFC 8446 (TLS 1.3) renamed the "elliptic_curve" extension. +-record(supported_groups, { + supported_groups + }). + -record(ec_point_formats, { ec_point_format_list }). -define(ECPOINT_UNCOMPRESSED, 0). +%% Defined in RFC 4492, deprecated by RFC 8422 +%% RFC 8422 compliant implementations MUST not support the two formats below -define(ECPOINT_ANSIX962_COMPRESSED_PRIME, 1). -define(ECPOINT_ANSIX962_COMPRESSED_CHAR2, 2). @@ -365,10 +376,11 @@ -define(NAMED_CURVE, 3). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Server name indication RFC 6066 section 3 +%% RFC 6066 Server name indication %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --define(SNI_EXT, 16#0000). +%% section 3 +-define(SNI_EXT, 0). %% enum { host_name(0), (255) } NameType; -define(SNI_NAMETYPE_HOST_NAME, 0). @@ -377,4 +389,56 @@ hostname = undefined }). +%% Other possible values from RFC 6066, not supported +-define(MAX_FRAGMENT_LENGTH, 1). +-define(CLIENT_CERTIFICATE_URL, 2). +-define(TRUSTED_CA_KEYS, 3). +-define(TRUNCATED_HMAC, 4). +-define(STATUS_REQUEST, 5). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% RFC 7250 Using Raw Public Keys in Transport Layer Security (TLS) +%% and Datagram Transport Layer Security (DTLS) +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Not supported +-define(CLIENT_CERTIFICATE_TYPE, 19). +-define(SERVER_CERTIFICATE_TYPE, 20). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% RFC 6520 Transport Layer Security (TLS) and +%% Datagram Transport Layer Security (DTLS) Heartbeat Extension +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Not supported +-define(HS_HEARTBEAT, 15). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% RFC 6962 Certificate Transparency +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Not supported +-define(SIGNED_CERTIFICATE_TIMESTAMP, 18). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% RFC 7685 A Transport Layer Security (TLS) ClientHello Padding Extension +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Not supported +-define(PADDING, 21). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Supported Versions RFC 8446 (TLS 1.3) section 4.2.1 also affects TLS-1.2 +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +-define(SUPPORTED_VERSIONS_EXT, 43). + +-record(client_hello_versions, {versions}). +-record(server_hello_selected_version, {selected_version}). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Signature Algorithms RFC 8446 (TLS 1.3) section 4.2.3 also affects TLS-1.2 +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +-define(SIGNATURE_ALGORITHMS_CERT_EXT, 50). + +-record(signature_algorithms_cert, {signature_scheme_list}). + -endif. % -ifdef(ssl_handshake). diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index ddd7a8eb7b..06c3ccae45 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -25,6 +25,7 @@ -include_lib("public_key/include/public_key.hrl"). +-define(VSN, "8.2.6"). -define(SECRET_PRINTOUT, "***"). -type reason() :: any(). @@ -70,14 +71,40 @@ -define(FALSE, 1). %% sslv3 is considered insecure due to lack of padding check (Poodle attack) -%% Keep as interop with legacy software but do not support as default --define(ALL_AVAILABLE_VERSIONS, ['tlsv1.2', 'tlsv1.1', tlsv1, sslv3]). +%% Keep as interop with legacy software but do not support as default +%% tlsv1.0 and tlsv1.1 is now also considered legacy +%% tlsv1.3 is under development (experimental). +-define(ALL_AVAILABLE_VERSIONS, ['tlsv1.3', 'tlsv1.2', 'tlsv1.1', tlsv1, sslv3]). -define(ALL_AVAILABLE_DATAGRAM_VERSIONS, ['dtlsv1.2', dtlsv1]). --define(ALL_SUPPORTED_VERSIONS, ['tlsv1.2', 'tlsv1.1', tlsv1]). --define(MIN_SUPPORTED_VERSIONS, ['tlsv1.1', tlsv1]). --define(ALL_DATAGRAM_SUPPORTED_VERSIONS, ['dtlsv1.2', dtlsv1]). +%% Defines the default versions when not specified by an ssl option. +-define(ALL_SUPPORTED_VERSIONS, ['tlsv1.2']). +-define(MIN_SUPPORTED_VERSIONS, ['tlsv1.1']). + +%% Versions allowed in TLSCiphertext.version (TLS 1.2 and prior) and +%% TLSCiphertext.legacy_record_version (TLS 1.3). +%% TLS 1.3 sets TLSCiphertext.legacy_record_version to 0x0303 for all records +%% generated other than an than an initial ClientHello, where it MAY also be 0x0301. +%% Thus, the allowed range is limited to 0x0300 - 0x0303. +-define(ALL_TLS_RECORD_VERSIONS, ['tlsv1.2', 'tlsv1.1', tlsv1, sslv3]). + +-define(ALL_DATAGRAM_SUPPORTED_VERSIONS, ['dtlsv1.2']). -define(MIN_DATAGRAM_SUPPORTED_VERSIONS, [dtlsv1]). +%% TLS 1.3 - Section 4.1.3 +%% +%% If negotiating TLS 1.2, TLS 1.3 servers MUST set the last eight bytes +%% of their Random value to the bytes: +%% +%% 44 4F 57 4E 47 52 44 01 +%% +%% If negotiating TLS 1.1 or below, TLS 1.3 servers MUST and TLS 1.2 +%% servers SHOULD set the last eight bytes of their Random value to the +%% bytes: +%% +%% 44 4F 57 4E 47 52 44 00 +-define(RANDOM_OVERRIDE_TLS12, <<16#44,16#4F,16#57,16#4E,16#47,16#52,16#44,16#01>>). +-define(RANDOM_OVERRIDE_TLS11, <<16#44,16#4F,16#57,16#4E,16#47,16#52,16#44,16#00>>). + -define('24H_in_msec', 86400000). -define('24H_in_sec', 86400). @@ -127,7 +154,7 @@ alpn_preferred_protocols = undefined :: [binary()] | undefined, next_protocols_advertised = undefined :: [binary()] | undefined, next_protocol_selector = undefined, %% fun([binary()]) -> binary()) - log_alert :: boolean(), + log_level = notice :: atom(), server_name_indication = undefined, sni_hosts :: [{inet:hostname(), [tuple()]}] | 'undefined', sni_fun :: function() | undefined, @@ -142,9 +169,11 @@ crl_check :: boolean() | peer | best_effort | 'undefined', crl_cache, signature_algs, + signature_algs_cert, eccs, - honor_ecc_order :: boolean(), - max_handshake_size :: integer(), + supported_groups, %% RFC 8422, RFC 8446 + honor_ecc_order :: boolean() | 'undefined', + max_handshake_size :: integer() | 'undefined', handshake, customize_hostname_check %% , @@ -169,17 +198,7 @@ connection_cb }). --type key_map() :: #{algorithm := rsa | dss | ecdsa, - %% engine and key_id ought to - %% be :=, but putting it in - %% the spec gives dialyzer warning - %% of correct code! - engine => crypto:engine_ref(), - key_id => crypto:key_id(), - password => crypto:password() - }. -type state_name() :: hello | abbreviated | certify | cipher | connection. - -type gen_fsm_state_return() :: {next_state, state_name(), any()} | {next_state, state_name(), any(), timeout()} | {stop, any(), any()}. diff --git a/lib/ssl/src/ssl_logger.erl b/lib/ssl/src/ssl_logger.erl new file mode 100644 index 0000000000..987693b96b --- /dev/null +++ b/lib/ssl/src/ssl_logger.erl @@ -0,0 +1,446 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1999-2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ssl_logger). + +-export([debug/4, + format/2, + notice/2]). + +-define(DEC2HEX(X), + if ((X) >= 0) andalso ((X) =< 9) -> (X) + $0; + ((X) >= 10) andalso ((X) =< 15) -> (X) + $a - 10 + end). + +-define(rec_info(T,R),lists:zip(record_info(fields,T),tl(tuple_to_list(R)))). + +-include("tls_record.hrl"). +-include("ssl_cipher.hrl"). +-include("ssl_internal.hrl"). +-include("tls_handshake.hrl"). +-include("dtls_handshake.hrl"). +-include("tls_handshake_1_3.hrl"). +-include_lib("kernel/include/logger.hrl"). + +%%------------------------------------------------------------------------- +%% External API +%%------------------------------------------------------------------------- + +%% SSL log formatter +format(#{level:= _Level, msg:= {report, Msg}, meta:= _Meta}, _Config0) -> + #{direction := Direction, + protocol := Protocol, + message := Content} = Msg, + case Protocol of + 'record' -> + BinMsg = + case Content of + #ssl_tls{} -> + [tls_record:build_tls_record(Content)]; + _ when is_list(Content) -> + lists:flatten(Content) + end, + format_tls_record(Direction, BinMsg); + 'handshake' -> + format_handshake(Direction, Content); + _Other -> + [] + end. + +%% Stateful logging +debug(Level, Direction, Protocol, Message) + when (Direction =:= inbound orelse Direction =:= outbound) andalso + (Protocol =:= 'record' orelse Protocol =:= 'handshake') -> + case logger:compare_levels(Level, debug) of + lt -> + ?LOG_DEBUG(#{direction => Direction, + protocol => Protocol, + message => Message}, + #{domain => [otp,ssl,Protocol]}); + eq -> + ?LOG_DEBUG(#{direction => Direction, + protocol => Protocol, + message => Message}, + #{domain => [otp,ssl,Protocol]}); + _ -> + ok + end. + +%% Stateful logging +notice(Level, Report) -> + case logger:compare_levels(Level, notice) of + lt -> + ?LOG_NOTICE(Report); + eq -> + ?LOG_NOTICE(Report); + _ -> + ok + end. + + +%%------------------------------------------------------------------------- +%% Handshake Protocol +%%------------------------------------------------------------------------- +format_handshake(Direction, BinMsg) -> + {Header, Message} = parse_handshake(Direction, BinMsg), + io_lib:format("~s~n~s~n", [Header, Message]). + + +parse_handshake(Direction, #client_hello{ + client_version = Version0, + cipher_suites = CipherSuites0, + extensions = Extensions + } = ClientHello) -> + Version = get_client_version(Version0, Extensions), + Header = io_lib:format("~s ~s Handshake, ClientHello", + [header_prefix(Direction), + version(Version)]), + CipherSuites = parse_cipher_suites(CipherSuites0), + Message = io_lib:format("~p", + [?rec_info(client_hello, + ClientHello#client_hello{cipher_suites = CipherSuites})]), + {Header, Message}; +parse_handshake(Direction, #server_hello{ + server_version = Version0, + cipher_suite = CipherSuite0, + extensions = Extensions + } = ServerHello) -> + Version = get_server_version(Version0, Extensions), + Header = io_lib:format("~s ~s Handshake, ServerHello", + [header_prefix(Direction), + version(Version)]), + CipherSuite = format_cipher(CipherSuite0), + Message = io_lib:format("~p", + [?rec_info(server_hello, + ServerHello#server_hello{cipher_suite = CipherSuite})]), + {Header, Message}; +parse_handshake(Direction, #hello_verify_request{} = HelloVerifyRequest) -> + Header = io_lib:format("~s Handshake, HelloVerifyRequest", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(hello_verify_request, HelloVerifyRequest)]), + {Header, Message}; +parse_handshake(Direction, #certificate{} = Certificate) -> + Header = io_lib:format("~s Handshake, Certificate", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(certificate, Certificate)]), + {Header, Message}; +parse_handshake(Direction, #server_key_exchange{} = ServerKeyExchange) -> + Header = io_lib:format("~s Handshake, ServerKeyExchange", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(server_key_exchange, ServerKeyExchange)]), + {Header, Message}; +parse_handshake(Direction, #server_key_params{} = ServerKeyExchange) -> + Header = io_lib:format("~s Handshake, ServerKeyExchange", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(server_key_params, ServerKeyExchange)]), + {Header, Message}; +parse_handshake(Direction, #certificate_request{} = CertificateRequest) -> + Header = io_lib:format("~s Handshake, CertificateRequest", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(certificate_request, CertificateRequest)]), + {Header, Message}; +parse_handshake(Direction, #server_hello_done{} = ServerHelloDone) -> + Header = io_lib:format("~s Handshake, ServerHelloDone", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(server_hello_done, ServerHelloDone)]), + {Header, Message}; +parse_handshake(Direction, #client_key_exchange{} = ClientKeyExchange) -> + Header = io_lib:format("~s Handshake, ClientKeyExchange", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(client_key_exchange, ClientKeyExchange)]), + {Header, Message}; +parse_handshake(Direction, #certificate_verify{} = CertificateVerify) -> + Header = io_lib:format("~s Handshake, CertificateVerify", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(certificate_verify, CertificateVerify)]), + {Header, Message}; +parse_handshake(Direction, #finished{} = Finished) -> + Header = io_lib:format("~s Handshake, Finished", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(finished, Finished)]), + {Header, Message}; +parse_handshake(Direction, #hello_request{} = HelloRequest) -> + Header = io_lib:format("~s Handshake, HelloRequest", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(hello_request, HelloRequest)]), + {Header, Message}; +parse_handshake(Direction, #certificate_request_1_3{} = CertificateRequest) -> + Header = io_lib:format("~s Handshake, CertificateRequest", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(certificate_request_1_3, CertificateRequest)]), + {Header, Message}; +parse_handshake(Direction, #certificate_1_3{} = Certificate) -> + Header = io_lib:format("~s Handshake, Certificate", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(certificate_1_3, Certificate)]), + {Header, Message}; +parse_handshake(Direction, #certificate_verify_1_3{} = CertificateVerify) -> + Header = io_lib:format("~s Handshake, CertificateVerify", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(certificate_verify_1_3, CertificateVerify)]), + {Header, Message}; +parse_handshake(Direction, #encrypted_extensions{} = EncryptedExtensions) -> + Header = io_lib:format("~s Handshake, EncryptedExtensions", + [header_prefix(Direction)]), + Message = io_lib:format("~p", [?rec_info(encrypted_extensions, EncryptedExtensions)]), + {Header, Message}. + + +parse_cipher_suites([_|_] = Ciphers) -> + [format_cipher(C) || C <- Ciphers]. + +format_cipher(C0) -> + try ssl_cipher_format:suite_bin_to_map(C0) of + Map -> + ssl_cipher_format:suite_map_to_str(Map) + catch + error:function_clause -> + format_uknown_cipher_suite(C0) + end. + +get_client_version(Version, Extensions) -> + CHVersions = maps:get(client_hello_versions, Extensions, undefined), + case CHVersions of + #client_hello_versions{versions = [Highest|_]} -> + Highest; + undefined -> + Version + end. + +get_server_version(Version, Extensions) -> + SHVersion = maps:get(server_hello_selected_version, Extensions, undefined), + case SHVersion of + #server_hello_selected_version{selected_version = SelectedVersion} -> + SelectedVersion; + undefined -> + Version + end. + +version({3,4}) -> + "TLS 1.3"; +version({3,3}) -> + "TLS 1.2"; +version({3,2}) -> + "TLS 1.1"; +version({3,1}) -> + "TLS 1.0"; +version({3,0}) -> + "SSL 3.0"; +version({254,253}) -> + "DTLS 1.2"; +version({254,255}) -> + "DTLS 1.0"; +version({M,N}) -> + io_lib:format("TLS/DTLS [0x0~B0~B]", [M,N]). + +header_prefix(inbound) -> + "<<<"; +header_prefix(outbound) -> + ">>>". + + +%%------------------------------------------------------------------------- +%% TLS Record Protocol +%%------------------------------------------------------------------------- +format_tls_record(Direction, BinMsg) -> + {Message, Size} = convert_to_hex('tls_record', BinMsg), + Header = io_lib:format("~s (~B bytes) ~s~n", + [header_prefix_tls_record(Direction), + Size, + tls_record_version(BinMsg)]), + Header ++ Message. + + +header_prefix_tls_record(inbound) -> + "reading"; +header_prefix_tls_record(outbound) -> + "writing". + + +tls_record_version([<<?BYTE(B),?BYTE(3),?BYTE(3),_/binary>>|_]) -> + io_lib:format("TLS 1.2 Record Protocol, ~s", [msg_type(B)]); +tls_record_version([<<?BYTE(B),?BYTE(3),?BYTE(2),_/binary>>|_]) -> + io_lib:format("TLS 1.1 Record Protocol, ~s", [msg_type(B)]); +tls_record_version([<<?BYTE(B),?BYTE(3),?BYTE(1),_/binary>>|_]) -> + io_lib:format("TLS 1.0 Record Protocol, ~s", [msg_type(B)]); +tls_record_version([<<?BYTE(B),?BYTE(3),?BYTE(0),_/binary>>|_]) -> + io_lib:format("SSL 3.0 Record Protocol, ~s", [msg_type(B)]); +tls_record_version([<<?BYTE(B),?BYTE(254),?BYTE(253),_/binary>>|_]) -> + io_lib:format("DTLS 1.2 Record Protocol, ~s", [msg_type(B)]); +tls_record_version([<<?BYTE(B),?BYTE(254),?BYTE(255),_/binary>>|_]) -> + io_lib:format("DTLS 1.0 Record Protocol, ~s", [msg_type(B)]); +tls_record_version([<<?BYTE(B),?BYTE(M),?BYTE(N),_/binary>>|_]) -> + io_lib:format("TLS/DTLS [0x0~B0~B] Record Protocol, ~s", [M, N, msg_type(B)]). + + +msg_type(20) -> "change_cipher_spec"; +msg_type(21) -> "alert"; +msg_type(22) -> "handshake"; +msg_type(23) -> "application_data"; +msg_type(_) -> unknown. + + +%%------------------------------------------------------------------------- +%% Hex encoding functions +%%------------------------------------------------------------------------- +convert_to_hex(Protocol, BinMsg) -> + convert_to_hex(Protocol, BinMsg, [], [], 0). +%% +convert_to_hex(P, [], Row0, Acc, C) when C rem 16 =:= 0 -> + Row = lists:reverse(end_row(P, Row0)), + {lists:reverse(Acc) ++ Row ++ io_lib:nl(), C}; +convert_to_hex(P, [], Row0, Acc, C) -> + Row = lists:reverse(end_row(P, Row0)), + Padding = calculate_padding(Row0, Acc), + PaddedRow = string:pad(Row, Padding, leading, $ ), + {lists:reverse(Acc) ++ PaddedRow ++ io_lib:nl(), C}; +convert_to_hex(P, [H|T], Row, Acc, C) when is_list(H) -> + convert_to_hex(P, H ++ T, Row, Acc, C); +convert_to_hex(P, [<<>>|T], Row, Acc, C) -> + convert_to_hex(P, T, Row, Acc, C); + +%% First line +convert_to_hex(P, [<<A:4,B:4,R/binary>>|T], Row, Acc, C) when C =:= 0 -> + convert_to_hex(P, [<<R/binary>>|T], + update_row(<<A:4,B:4>>, Row), + prepend_first_row(P, A, B, Acc, C), + C + 1); +%% New line +convert_to_hex(P, [<<A:4,B:4,R/binary>>|T], Row, Acc, C) when C rem 16 =:= 0 -> + convert_to_hex(P, [<<R/binary>>|T], + update_row(<<A:4,B:4>>, []), + prepend_row(P, A, B, Row, Acc, C), + C + 1); +%% Add 8th hex with extra whitespace +%% 0000 - 16 03 02 00 bd 01 00 00 b9 ... +%% ^^^^ +convert_to_hex(P, [<<A:4,B:4,R/binary>>|T], Row, Acc, C) when C rem 8 =:= 7 -> + convert_to_hex(P, [<<R/binary>>|T], + update_row(<<A:4,B:4>>, Row), + prepend_eighths_hex(A, B, Acc), + C + 1); +convert_to_hex(P, [<<A:4,B:4,R/binary>>|T], Row, Acc, C) -> + convert_to_hex(P, [<<R/binary>>|T], + update_row(<<A:4,B:4>>, Row), + prepend_hex(A, B, Acc), + C + 1); +%% First line +convert_to_hex(P, [H|T], Row, Acc, C) when is_integer(H), C =:= 0 -> + convert_to_hex(P, T, + update_row(H, Row), + prepend_first_row(P, H, Acc, C), + C + 1); +%% New line +convert_to_hex(P, [H|T], Row, Acc, C) when is_integer(H), C rem 16 =:= 0 -> + convert_to_hex(P, T, + update_row(H, []), + prepend_row(P, H, Row, Acc, C), + C + 1); +%% Add 8th hex with extra whitespace +%% 0000 - 16 03 02 00 bd 01 00 00 b9 ... +%% ^^^^ +convert_to_hex(P, [H|T], Row, Acc, C) when is_integer(H), C rem 8 =:= 7 -> + convert_to_hex(P, T, + update_row(H, Row), + prepend_eighths_hex(H, Acc), + C + 1); +convert_to_hex(P, [H|T], Row, Acc, C) when is_integer(H) -> + convert_to_hex(P, T, + update_row(H, Row), + prepend_hex(H, Acc), + C + 1). + + +row_prefix(_ , N) -> + S = string:pad(string:to_lower(erlang:integer_to_list(N, 16)),4,leading,$0), + lists:reverse(lists:flatten(S ++ " - ")). + + +end_row(_, Row) -> + Row ++ " ". + + +%% Calculate padding of the "printable character" lines in order to be +%% visually aligned. +calculate_padding(Row, Acc) -> + %% Number of new line characters + NNL = (length(Acc) div 75) * length(io_lib:nl()), + %% Length of the last printed line + Length = (length(Acc) - NNL) rem 75, + %% Adjusted length of the last printed line + PaddedLength = 75 - (16 - length(Row)), %% Length + %% Padding + PaddedLength - Length. + + +%%------------------------------------------------------------------------- +%% Functions operating on reversed lists +%%------------------------------------------------------------------------- +update_row(B, Row) when is_binary(B) -> + case binary_to_list(B) of + [C] when 32 =< C, C =< 126 -> + [C|Row]; + _Else -> + [$.|Row] + end; +update_row(C, Row) when 32 =< C, C =< 126 -> + [C|Row]; +update_row(_, Row) -> + [$.|Row]. + + +prepend_first_row(P, A, B, Acc, C) -> + prepend_hex(A, B,row_prefix(P, C) ++ Acc). +%% +prepend_first_row(P, N, Acc, C) -> + prepend_hex(N,row_prefix(P, C) ++ Acc). + +prepend_row(P, A, B, Row, Acc, C) -> + prepend_hex(A, B,row_prefix(P, C) ++ io_lib:nl() ++ end_row(P, Row) ++ Acc). +%% +prepend_row(P, N, Row, Acc, C) -> + prepend_hex(N,row_prefix(P, C) ++ io_lib:nl() ++ end_row(P, Row) ++ Acc). + + + +prepend_hex(A, B, Acc) -> + [$ ,?DEC2HEX(B),?DEC2HEX(A)|Acc]. +%% +prepend_hex(N, Acc) -> + " " ++ number_to_hex(N) ++ Acc. + + +prepend_eighths_hex(A, B, Acc) -> + [$ ,$ ,?DEC2HEX(B),?DEC2HEX(A)|Acc]. +%% +prepend_eighths_hex(N, Acc) -> + " " ++ number_to_hex(N) ++ Acc. + +number_to_hex(N) -> + case string:to_lower(erlang:integer_to_list(N, 16)) of + H when length(H) < 2 -> + lists:append(H, "0"); + H -> + lists:reverse(H) + end. + +format_uknown_cipher_suite(<<?BYTE(X), ?BYTE(Y)>>) -> + "0x" ++ number_to_hex(X) ++ "0x" ++ number_to_hex(Y). + diff --git a/lib/ssl/src/ssl_manager.erl b/lib/ssl/src/ssl_manager.erl index c56675b691..456a560bf6 100644 --- a/lib/ssl/src/ssl_manager.erl +++ b/lib/ssl/src/ssl_manager.erl @@ -516,10 +516,10 @@ last_delay_timer({{_,_},_}, TRef, {LastServer, _}) -> last_delay_timer({_,_}, TRef, {_, LastClient}) -> {TRef, LastClient}. -%% If we can not generate a not allready in use session ID in +%% If we cannot generate a not allready in use session ID in %% ?GEN_UNIQUE_ID_MAX_TRIES we make the new session uncacheable The %% value of ?GEN_UNIQUE_ID_MAX_TRIES is stolen from open SSL which -%% states : "If we can not find a session id in +%% states : "If we cannot find a session id in %% ?GEN_UNIQUE_ID_MAX_TRIES either the RAND code is broken or someone %% is trying to open roughly very close to 2^128 (or 2^256) SSL %% sessions to our server" @@ -530,7 +530,7 @@ new_id(Port, Tries, Cache, CacheCb) -> case CacheCb:lookup(Cache, {Port, Id}) of undefined -> Now = erlang:monotonic_time(), - %% New sessions can not be set to resumable + %% New sessions cannot be set to resumable %% until handshake is compleate and the %% other session values are set. CacheCb:update(Cache, {Port, Id}, #session{session_id = Id, diff --git a/lib/ssl/src/ssl_pkix_db.erl b/lib/ssl/src/ssl_pkix_db.erl index f7ddbd060e..dec48fa914 100644 --- a/lib/ssl/src/ssl_pkix_db.erl +++ b/lib/ssl/src/ssl_pkix_db.erl @@ -27,6 +27,7 @@ -include("ssl_internal.hrl"). -include_lib("public_key/include/public_key.hrl"). -include_lib("kernel/include/file.hrl"). +-include_lib("kernel/include/logger.hrl"). -export([create/1, create_pem_cache/1, add_crls/3, remove_crls/2, remove/1, add_trusted_certs/3, @@ -311,7 +312,7 @@ decode_certs(Ref, Cert) -> error:_ -> Report = io_lib:format("SSL WARNING: Ignoring a CA cert as " "it could not be correctly decoded.~n", []), - error_logger:info_report(Report), + ?LOG_NOTICE(Report), undefined end. diff --git a/lib/ssl/src/ssl_record.erl b/lib/ssl/src/ssl_record.erl index 1a36b2dba8..867d2cfc5a 100644 --- a/lib/ssl/src/ssl_record.erl +++ b/lib/ssl/src/ssl_record.erl @@ -25,6 +25,7 @@ -module(ssl_record). -include("ssl_record.hrl"). +-include("ssl_connection.hrl"). -include("ssl_internal.hrl"). -include("ssl_cipher.hrl"). -include("ssl_alert.hrl"). @@ -39,7 +40,8 @@ set_renegotiation_flag/2, set_client_verify_data/3, set_server_verify_data/3, - empty_connection_state/2, initial_connection_state/2, record_protocol_role/1]). + empty_connection_state/2, initial_connection_state/2, record_protocol_role/1, + step_encryption_state/1]). %% Compression -export([compress/3, uncompress/3, compressions/0]). @@ -120,6 +122,22 @@ activate_pending_connection_state(#{current_write := Current, }. %%-------------------------------------------------------------------- +-spec step_encryption_state(#state{}) -> #state{}. +%% +%% Description: Activates the next encyrption state (e.g. handshake +%% encryption). +%%-------------------------------------------------------------------- +step_encryption_state(#state{connection_states = + #{pending_read := PendingRead, + pending_write := PendingWrite} = ConnStates} = State) -> + NewRead = PendingRead#{sequence_number => 0}, + NewWrite = PendingWrite#{sequence_number => 0}, + State#state{connection_states = + ConnStates#{current_read => NewRead, + current_write => NewWrite}}. + + +%%-------------------------------------------------------------------- -spec set_security_params(#security_parameters{}, #security_parameters{}, connection_states()) -> connection_states(). %% @@ -280,13 +298,12 @@ compress(?NULL, Data, CS) -> {Data, CS}. %%-------------------------------------------------------------------- --spec compressions() -> [binary()]. +-spec compressions() -> [integer()]. %% %% Description: return a list of compressions supported (currently none) %%-------------------------------------------------------------------- compressions() -> - [?byte(?NULL)]. - + [?NULL]. %%==================================================================== %% Payload encryption/decryption @@ -378,7 +395,7 @@ decipher_aead(Type, #cipher_state{key = Key} = CipherState, AAD0, CipherFragment try Nonce = decrypt_nonce(Type, CipherState, CipherFragment), {AAD, CipherText, CipherTag} = aead_ciphertext_split(Type, CipherState, CipherFragment, AAD0), - case ssl_cipher:aead_decrypt(Type, Key, Nonce, CipherText, CipherTag, AAD) of + case ssl_cipher:aead_decrypt(Type, Key, Nonce, CipherText, CipherTag, AAD) of Content when is_binary(Content) -> Content; _ -> @@ -454,34 +471,43 @@ initial_security_params(ConnectionEnd) -> -define(end_additional_data(AAD, Len), << (begin(AAD)end)/binary, ?UINT16(begin(Len)end) >>). -do_cipher_aead(?CHACHA20_POLY1305 = Type, Fragment, #cipher_state{key=Key} = CipherState, AAD0) -> +do_cipher_aead(?CHACHA20_POLY1305 = Type, Fragment, #cipher_state{key=Key, tag_len = TagLen} = CipherState, AAD0) -> AAD = ?end_additional_data(AAD0, erlang:iolist_size(Fragment)), - Nonce = encrypt_nonce(Type, CipherState), - {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD), + Nonce = chacha_nonce(CipherState), + {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD, TagLen), {<<Content/binary, CipherTag/binary>>, CipherState}; -do_cipher_aead(Type, Fragment, #cipher_state{key=Key, nonce = ExplicitNonce} = CipherState, AAD0) -> +do_cipher_aead(Type, Fragment, #cipher_state{key=Key, tag_len = TagLen, nonce = ExplicitNonce} = CipherState, AAD0) -> AAD = ?end_additional_data(AAD0, erlang:iolist_size(Fragment)), Nonce = encrypt_nonce(Type, CipherState), - {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD), + {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD, TagLen), {<<ExplicitNonce:64/integer, Content/binary, CipherTag/binary>>, CipherState#cipher_state{nonce = ExplicitNonce + 1}}. -encrypt_nonce(?CHACHA20_POLY1305, #cipher_state{nonce = Nonce, iv = IV}) -> - crypto:exor(<<?UINT32(0), Nonce/binary>>, IV); -encrypt_nonce(?AES_GCM, #cipher_state{iv = IV, nonce = ExplicitNonce}) -> + +chacha_nonce(#cipher_state{nonce = Nonce, iv = IV}) -> + crypto:exor(<<?UINT32(0), Nonce/binary>>, IV). + +encrypt_nonce(Type, #cipher_state{iv = IV, nonce = ExplicitNonce}) when Type == ?AES_GCM; + Type == ?AES_CCM; + Type == ?AES_CCM_8 -> <<Salt:4/bytes, _/binary>> = IV, <<Salt/binary, ExplicitNonce:64/integer>>. -decrypt_nonce(?CHACHA20_POLY1305, #cipher_state{nonce = Nonce, iv = IV}, _) -> - crypto:exor(<<Nonce:96/unsigned-big-integer>>, IV); -decrypt_nonce(?AES_GCM, #cipher_state{iv = <<Salt:4/bytes, _/binary>>}, <<ExplicitNonce:8/bytes, _/binary>>) -> - <<Salt/binary, ExplicitNonce/binary>>. +decrypt_nonce(?CHACHA20_POLY1305, CipherState, _) -> + chacha_nonce(CipherState); +decrypt_nonce(Type, #cipher_state{iv = <<Salt:4/bytes, _/binary>>}, <<ExplicitNonce:8/bytes, _/binary>>) when + Type == ?AES_GCM; + Type == ?AES_CCM; + Type == ?AES_CCM_8 -> + <<Salt/binary, ExplicitNonce/binary>>. -compile({inline, [aead_ciphertext_split/4]}). aead_ciphertext_split(?CHACHA20_POLY1305, #cipher_state{tag_len = Len}, CipherTextFragment, AAD) -> CipherLen = byte_size(CipherTextFragment) - Len, <<CipherText:CipherLen/bytes, CipherTag:Len/bytes>> = CipherTextFragment, {?end_additional_data(AAD, CipherLen), CipherText, CipherTag}; -aead_ciphertext_split(?AES_GCM, #cipher_state{tag_len = Len}, CipherTextFragment, AAD) -> +aead_ciphertext_split(Type, #cipher_state{tag_len = Len}, CipherTextFragment, AAD) when Type == ?AES_GCM; + Type == ?AES_CCM; + Type == ?AES_CCM_8 -> CipherLen = byte_size(CipherTextFragment) - (Len + 8), %% 8 is length of explicit Nonce << _:8/bytes, CipherText:CipherLen/bytes, CipherTag:Len/bytes>> = CipherTextFragment, {?end_additional_data(AAD, CipherLen), CipherText, CipherTag}. diff --git a/lib/ssl/src/ssl_record.hrl b/lib/ssl/src/ssl_record.hrl index a927fba0de..6d4d47cedb 100644 --- a/lib/ssl/src/ssl_record.hrl +++ b/lib/ssl/src/ssl_record.hrl @@ -74,7 +74,7 @@ -define(INITIAL_BYTES, 5). -define(MAX_SEQENCE_NUMBER, 18446744073709551615). %% (1 bsl 64) - 1 = 18446744073709551615 -%% Sequence numbers can not wrap so when max is about to be reached we should renegotiate. +%% Sequence numbers cannot wrap so when max is about to be reached we should renegotiate. %% We will renegotiate a little before so that there will be sequence numbers left %% for the rehandshake and a little data. Currently we decided to renegotiate a little more %% often as we can have a cheaper test to check if it is time to renegotiate. It will still @@ -96,6 +96,11 @@ -define(AES_CBC, 7). -define(AES_GCM, 8). -define(CHACHA20_POLY1305, 9). +%% Following two are not defined in any RFC but we want to have the +%% same type of handling internaly, all of these "bulk_cipher_algorithm" +%% enums are only used internaly anyway. +-define(AES_CCM, 10). +-define(AES_CCM_8, 11). %% CipherType -define(STREAM, 0). @@ -140,6 +145,7 @@ -define(ALERT, 21). -define(HANDSHAKE, 22). -define(APPLICATION_DATA, 23). +-define(HEARTBEAT, 24). -define(KNOWN_RECORD_TYPE(Type), (is_integer(Type) andalso (20 =< (Type)) andalso ((Type) =< 23))). -define(MAX_PLAIN_TEXT_LENGTH, 16384). diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 52e5db731a..872a557e67 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -37,7 +37,8 @@ -include("ssl_api.hrl"). -include("ssl_internal.hrl"). -include("ssl_srp.hrl"). --include_lib("public_key/include/public_key.hrl"). +-include_lib("public_key/include/public_key.hrl"). +-include_lib("kernel/include/logger.hrl"). %% Internal application API @@ -49,7 +50,8 @@ handle_protocol_record/3]). %% Handshake handling --export([renegotiation/2, renegotiate/2, send_handshake/2, +-export([renegotiation/2, renegotiate/2, send_handshake/2, + send_handshake_flight/1, queue_handshake/2, queue_change_cipher/2, reinit/1, reinit_handshake_data/1, select_sni_extension/1, empty_connection_state/2]). @@ -66,9 +68,26 @@ -export([init/3, error/3, downgrade/3, %% Initiation and take down states hello/3, user_hello/3, certify/3, cipher/3, abbreviated/3, %% Handshake states connection/3]). +%% TLS 1.3 state functions (server) +-export([start/3, %% common state with client + negotiated/3, + recvd_ch/3, + wait_cert/3, %% common state with client + wait_cv/3, %% common state with client + wait_eoed/3, + wait_finished/3, %% common state with client + wait_flight2/3, + connected/3 %% common state with client + ]). +%% TLS 1.3 state functions (client) +-export([wait_cert_cr/3, + wait_ee/3, + wait_sh/3 + ]). %% gen_statem callbacks -export([callback_mode/0, terminate/3, code_change/4, format_status/2]). +-export([encode_handshake/4]). -define(DIST_CNTRL_SPAWN_OPTS, [{priority, max}]). @@ -175,25 +194,32 @@ next_record(State) -> next_record(State, CipherTexts, ConnectionStates, Check) -> next_record(State, CipherTexts, ConnectionStates, Check, []). %% -next_record(State, [#ssl_tls{type = ?APPLICATION_DATA} = CT|CipherTexts], ConnectionStates0, Check, Acc) -> - case tls_record:decode_cipher_text(CT, ConnectionStates0, Check) of - {#ssl_tls{fragment = Fragment}, ConnectionStates} -> - next_record(State, CipherTexts, ConnectionStates, Check, [Fragment|Acc]); - #alert{} = Alert -> - Alert - end; -next_record(State, [CT|CipherTexts], ConnectionStates0, Check, []) -> - case tls_record:decode_cipher_text(CT, ConnectionStates0, Check) of - {Record, ConnectionStates} -> +next_record(#state{connection_env = #connection_env{negotiated_version = Version}} = State, + [CT|CipherTexts], ConnectionStates0, Check, Acc) -> + case tls_record:decode_cipher_text(Version, CT, ConnectionStates0, Check) of + {#ssl_tls{type = ?APPLICATION_DATA, fragment = Fragment}, ConnectionStates} -> + case CipherTexts of + [] -> + %% End of cipher texts - build and deliver an ?APPLICATION_DATA record + %% from the accumulated fragments + next_record_done(State, [], ConnectionStates, + #ssl_tls{type = ?APPLICATION_DATA, + fragment = iolist_to_binary(lists:reverse(Acc, [Fragment]))}); + [_|_] -> + next_record(State, CipherTexts, ConnectionStates, Check, [Fragment|Acc]) + end; + {Record, ConnectionStates} when Acc =:= [] -> + %% Singelton non-?APPLICATION_DATA record - deliver next_record_done(State, CipherTexts, ConnectionStates, Record); + {_Record, _ConnectionStates_to_forget} -> + %% Not ?APPLICATION_DATA but we have accumulated fragments + %% -> build an ?APPLICATION_DATA record with concatenated fragments + %% and forget about decrypting this record - we'll decrypt it again next time + next_record_done(State, [CT|CipherTexts], ConnectionStates0, + #ssl_tls{type = ?APPLICATION_DATA, fragment = iolist_to_binary(lists:reverse(Acc))}); #alert{} = Alert -> Alert - end; -next_record(State, CipherTexts, ConnectionStates, _Check, Acc) -> - %% Not ?APPLICATION_DATA but we have a nonempty Acc - %% -> build an ?APPLICATION_DATA record with the accumulated fragments - next_record_done(State, CipherTexts, ConnectionStates, - #ssl_tls{type = ?APPLICATION_DATA, fragment = iolist_to_binary(lists:reverse(Acc))}). + end. next_record_done(#state{protocol_buffers = Buffers} = State, CipherTexts, ConnectionStates, Record) -> {Record, @@ -211,7 +237,8 @@ next_event(StateName, no_record, State0, Actions) -> {#ssl_tls{} = Record, State} -> {next_state, StateName, State, [{next_event, internal, {protocol_record, Record}} | Actions]}; #alert{} = Alert -> - {next_state, StateName, State0, [{next_event, internal, Alert} | Actions]} + Version = State0#state.connection_env#connection_env.negotiated_version, + ssl_connection:handle_own_alert(Alert, Version, StateName, State0) end; next_event(StateName, Record, State, Actions) -> case Record of @@ -220,7 +247,8 @@ next_event(StateName, Record, State, Actions) -> #ssl_tls{} = Record -> {next_state, StateName, State, [{next_event, internal, {protocol_record, Record}} | Actions]}; #alert{} = Alert -> - {next_state, StateName, State, [{next_event, internal, Alert} | Actions]} + Version = State#state.connection_env#connection_env.negotiated_version, + ssl_connection:handle_own_alert(Alert, Version, StateName, State) end. @@ -246,8 +274,12 @@ handle_protocol_record(#ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, Stat {stop, _, _} = Stop-> Stop; {Record, State1} -> - {next_state, StateName, State, Actions} = next_event(StateName, Record, State1), - ssl_connection:hibernate_after(StateName, State, Actions) + case next_event(StateName, Record, State1) of + {next_state, StateName, State, Actions} -> + ssl_connection:hibernate_after(StateName, State, Actions); + {stop, _, _} = Stop -> + Stop + end end; %%% TLS record protocol level handshake messages handle_protocol_record(#ssl_tls{type = ?HANDSHAKE, fragment = Data}, @@ -256,7 +288,8 @@ handle_protocol_record(#ssl_tls{type = ?HANDSHAKE, fragment = Data}, connection_env = #connection_env{negotiated_version = Version}, ssl_options = Options} = State0) -> try - {Packets, Buf} = tls_handshake:get_tls_handshake(Version,Data,Buf0, Options), + EffectiveVersion = effective_version(Version, Options), + {Packets, Buf} = tls_handshake:get_tls_handshake(EffectiveVersion,Data,Buf0, Options), State = State0#state{protocol_buffers = Buffers#protocol_buffers{tls_handshake_buffer = Buf}}, @@ -292,9 +325,7 @@ handle_protocol_record(#ssl_tls{type = ?ALERT, fragment = EncAlerts}, StateName, handle_alerts(Alerts, {next_state, StateName, State}); [] -> ssl_connection:handle_own_alert(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, empty_alert), - Version, StateName, State); - #alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, Version, StateName, State) + Version, StateName, State) catch _:_ -> ssl_connection:handle_own_alert(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, alert_decode_error), @@ -337,12 +368,17 @@ renegotiate(#state{static_env = #static_env{role = server, send_handshake(Handshake, State) -> send_handshake_flight(queue_handshake(Handshake, State)). + queue_handshake(Handshake, #state{handshake_env = #handshake_env{tls_handshake_history = Hist0} = HsEnv, connection_env = #connection_env{negotiated_version = Version}, flight_buffer = Flight0, + ssl_options = SslOpts, connection_states = ConnectionStates0} = State0) -> {BinHandshake, ConnectionStates, Hist} = encode_handshake(Handshake, Version, ConnectionStates0, Hist0), + ssl_logger:debug(SslOpts#ssl_options.log_level, outbound, 'handshake', Handshake), + ssl_logger:debug(SslOpts#ssl_options.log_level, outbound, 'record', BinHandshake), + State0#state{connection_states = ConnectionStates, handshake_env = HsEnv#handshake_env{tls_handshake_history = Hist}, flight_buffer = Flight0 ++ [BinHandshake]}. @@ -353,11 +389,14 @@ send_handshake_flight(#state{static_env = #static_env{socket = Socket, tls_socket:send(Transport, Socket, Flight), {State0#state{flight_buffer = []}, []}. + queue_change_cipher(Msg, #state{connection_env = #connection_env{negotiated_version = Version}, flight_buffer = Flight0, + ssl_options = SslOpts, connection_states = ConnectionStates0} = State0) -> {BinChangeCipher, ConnectionStates} = encode_change_cipher(Msg, Version, ConnectionStates0), + ssl_logger:debug(SslOpts#ssl_options.log_level, outbound, 'record', BinChangeCipher), State0#state{connection_states = ConnectionStates, flight_buffer = Flight0 ++ [BinChangeCipher]}. @@ -377,8 +416,8 @@ reinit_handshake_data(#state{handshake_env = HsEnv} =State) -> premaster_secret = undefined} }. -select_sni_extension(#client_hello{extensions = HelloExtensions}) -> - HelloExtensions#hello_extensions.sni; +select_sni_extension(#client_hello{extensions = #{sni := SNI}}) -> + SNI; select_sni_extension(_) -> undefined. @@ -388,6 +427,7 @@ empty_connection_state(ConnectionEnd, BeastMitigation) -> %%==================================================================== %% Alert and close handling %%==================================================================== + %%-------------------------------------------------------------------- -spec encode_alert(#alert{}, ssl_record:ssl_version(), ssl_record:connection_states()) -> {iolist(), ssl_record:connection_states()}. @@ -400,10 +440,12 @@ encode_alert(#alert{} = Alert, Version, ConnectionStates) -> send_alert(Alert, #state{static_env = #static_env{socket = Socket, transport_cb = Transport}, connection_env = #connection_env{negotiated_version = Version}, + ssl_options = SslOpts, connection_states = ConnectionStates0} = StateData0) -> {BinMsg, ConnectionStates} = encode_alert(Alert, Version, ConnectionStates0), tls_socket:send(Transport, Socket, BinMsg), + ssl_logger:debug(SslOpts#ssl_options.log_level, outbound, 'record', BinMsg), StateData0#state{connection_states = ConnectionStates}. %% If an ALERT sent in the connection state, should cause the TLS @@ -490,22 +532,27 @@ init({call, From}, {start, Timeout}, session = #session{own_certificate = Cert} = Session0, connection_states = ConnectionStates0 } = State0) -> + KeyShare = maybe_generate_client_shares(SslOpts), Hello = tls_handshake:client_hello(Host, Port, ConnectionStates0, SslOpts, - Cache, CacheCb, Renegotiation, Cert), - - Version = Hello#client_hello.client_version, - HelloVersion = tls_record:hello_version(Version, SslOpts#ssl_options.versions), + Cache, CacheCb, Renegotiation, Cert, KeyShare), + + HelloVersion = tls_record:hello_version(SslOpts#ssl_options.versions), Handshake0 = ssl_handshake:init_handshake_history(), {BinMsg, ConnectionStates, Handshake} = encode_handshake(Hello, HelloVersion, ConnectionStates0, Handshake0), tls_socket:send(Transport, Socket, BinMsg), + ssl_logger:debug(SslOpts#ssl_options.log_level, outbound, 'handshake', Hello), + ssl_logger:debug(SslOpts#ssl_options.log_level, outbound, 'record', BinMsg), + State = State0#state{connection_states = ConnectionStates, - connection_env = CEnv#connection_env{negotiated_version = Version}, %% Requested version + connection_env = CEnv#connection_env{negotiated_version = HelloVersion}, %% Requested version session = Session0#session{session_id = Hello#client_hello.session_id}, handshake_env = HsEnv#handshake_env{tls_handshake_history = Handshake}, - start_or_recv_from = From}, + start_or_recv_from = From, + key_share = KeyShare}, next_event(hello, no_record, State, [{{timeout, handshake}, Timeout, close}]); + init(Type, Event, State) -> gen_handshake(?FUNCTION_NAME, Type, Event, State). @@ -536,14 +583,15 @@ hello(internal, #client_hello{extensions = Extensions} = Hello, start_or_recv_from = From} = State) -> {next_state, user_hello, State#state{start_or_recv_from = undefined, handshake_env = HsEnv#handshake_env{hello = Hello}}, - [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; + [{reply, From, {ok, Extensions}}]}; hello(internal, #server_hello{extensions = Extensions} = Hello, #state{ssl_options = #ssl_options{handshake = hello}, handshake_env = HsEnv, start_or_recv_from = From} = State) -> {next_state, user_hello, State#state{start_or_recv_from = undefined, handshake_env = HsEnv#handshake_env{hello = Hello}}, - [{reply, From, {ok, ssl_connection:map_extensions(Extensions)}}]}; + [{reply, From, {ok, Extensions}}]}; + hello(internal, #client_hello{client_version = ClientVersion} = Hello, #state{connection_states = ConnectionStates0, static_env = #static_env{ @@ -556,27 +604,40 @@ hello(internal, #client_hello{client_version = ClientVersion} = Hello, connection_env = CEnv, session = #session{own_certificate = Cert} = Session0, ssl_options = SslOpts} = State) -> - case tls_handshake:hello(Hello, SslOpts, {Port, Session0, Cache, CacheCb, - ConnectionStates0, Cert, KeyExAlg}, Renegotiation) of - #alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, ClientVersion, hello, - State#state{connection_env = - CEnv#connection_env{negotiated_version = ClientVersion}}); - {Version, {Type, Session}, - ConnectionStates, Protocol0, ServerHelloExt, HashSign} -> - Protocol = case Protocol0 of - undefined -> CurrentProtocol; - _ -> Protocol0 - end, - gen_handshake(?FUNCTION_NAME, internal, {common_client_hello, Type, ServerHelloExt}, - State#state{connection_states = ConnectionStates, - connection_env = CEnv#connection_env{negotiated_version = Version}, - handshake_env = HsEnv#handshake_env{ - hashsign_algorithm = HashSign, - client_hello_version = ClientVersion, - negotiated_protocol = Protocol}, - session = Session - }) + + case choose_tls_version(SslOpts, Hello) of + 'tls_v1.3' -> + %% Continue in TLS 1.3 'start' state + {next_state, start, State, [{next_event, internal, Hello}]}; + 'tls_v1.2' -> + case tls_handshake:hello(Hello, + SslOpts, + {Port, Session0, Cache, CacheCb, + ConnectionStates0, Cert, KeyExAlg}, + Renegotiation) of + #alert{} = Alert -> + ssl_connection:handle_own_alert(Alert, ClientVersion, hello, + State#state{connection_env = CEnv#connection_env{negotiated_version + = ClientVersion}}); + {Version, {Type, Session}, + ConnectionStates, Protocol0, ServerHelloExt, HashSign} -> + Protocol = case Protocol0 of + undefined -> CurrentProtocol; + _ -> Protocol0 + end, + gen_handshake(?FUNCTION_NAME, + internal, + {common_client_hello, Type, ServerHelloExt}, + State#state{connection_states = ConnectionStates, + connection_env = CEnv#connection_env{negotiated_version = Version}, + handshake_env = HsEnv#handshake_env{ + hashsign_algorithm = HashSign, + client_hello_version = ClientVersion, + negotiated_protocol = Protocol}, + session = Session + }) + end + end; hello(internal, #server_hello{} = Hello, #state{connection_states = ConnectionStates0, @@ -681,7 +742,7 @@ connection(internal, #hello_request{}, try tls_sender:peer_renegotiate(Pid) of {ok, Write} -> Hello = tls_handshake:client_hello(Host, Port, ConnectionStates, SslOpts, - Cache, CacheCb, Renegotiation, Cert), + Cache, CacheCb, Renegotiation, Cert, undefined), {State, Actions} = send_handshake(Hello, State0#state{connection_states = ConnectionStates#{current_write => Write}}), next_event(hello, no_record, State#state{session = Session0#session{session_id = Hello#client_hello.session_id}}, Actions) @@ -700,7 +761,8 @@ connection(internal, #hello_request{}, ssl_options = SslOpts, connection_states = ConnectionStates} = State0) -> Hello = tls_handshake:client_hello(Host, Port, ConnectionStates, SslOpts, - Cache, CacheCb, Renegotiation, Cert), + Cache, CacheCb, Renegotiation, Cert, undefined), + {State, Actions} = send_handshake(Hello, State0), next_event(hello, no_record, State#state{session = Session0#session{session_id = Hello#client_hello.session_id}}, Actions); @@ -755,7 +817,119 @@ downgrade(info, {CloseTag, Socket}, downgrade(info, Info, State) -> handle_info(Info, ?FUNCTION_NAME, State); downgrade(Type, Event, State) -> - ssl_connection:?FUNCTION_NAME(Type, Event, State, ?MODULE). + ssl_connection:?FUNCTION_NAME(Type, Event, State, ?MODULE). + +%%-------------------------------------------------------------------- +%% TLS 1.3 state functions +%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +-spec start(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +start(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +start(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + +%%-------------------------------------------------------------------- +-spec negotiated(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +negotiated(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +negotiated(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + +%%-------------------------------------------------------------------- +-spec recvd_ch(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +recvd_ch(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +recvd_ch(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + +%%-------------------------------------------------------------------- +-spec wait_cert(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +wait_cert(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +wait_cert(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + +%%-------------------------------------------------------------------- +-spec wait_cv(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +wait_cv(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +wait_cv(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + +%%-------------------------------------------------------------------- +-spec wait_eoed(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +wait_eoed(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +wait_eoed(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + +%%-------------------------------------------------------------------- +-spec wait_finished(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +wait_finished(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +wait_finished(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + +%%-------------------------------------------------------------------- +-spec wait_flight2(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +wait_flight2(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +wait_flight2(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + +%%-------------------------------------------------------------------- +-spec connected(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +connected(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +connected(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + +%%-------------------------------------------------------------------- +-spec wait_cert_cr(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +wait_cert_cr(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +wait_cert_cr(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + +%%-------------------------------------------------------------------- +-spec wait_ee(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +wait_ee(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +wait_ee(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + +%%-------------------------------------------------------------------- +-spec wait_sh(gen_statem:event_type(), term(), #state{}) -> + gen_statem:state_function_result(). +%%-------------------------------------------------------------------- +wait_sh(info, Event, State) -> + gen_info_1_3(Event, ?FUNCTION_NAME, State); +wait_sh(Type, Event, State) -> + gen_handshake_1_3(?FUNCTION_NAME, Type, Event, State). + %-------------------------------------------------------------------- %% gen_statem callbacks %%-------------------------------------------------------------------- @@ -786,7 +960,6 @@ initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trac #ssl_options{beast_mitigation = BeastMitigation, erl_dist = IsErlDist} = SSLOptions, ConnectionStates = tls_record:init_connection_states(Role, BeastMitigation), - SessionCacheCb = case application:get_env(ssl, session_cb) of {ok, Cb} when is_atom(Cb) -> Cb; @@ -843,8 +1016,9 @@ initialize_tls_sender(#state{static_env = #static_env{ tracker = Tracker }, connection_env = #connection_env{negotiated_version = Version}, - socket_options = SockOpts, - ssl_options = #ssl_options{renegotiate_at = RenegotiateAt}, + socket_options = SockOpts, + ssl_options = #ssl_options{renegotiate_at = RenegotiateAt, + log_level = LogLevel}, connection_states = #{current_write := ConnectionWriteState}, protocol_specific = #{sender := Sender}}) -> Init = #{current_write => ConnectionWriteState, @@ -854,21 +1028,27 @@ initialize_tls_sender(#state{static_env = #static_env{ tracker => Tracker, transport_cb => Transport, negotiated_version => Version, - renegotiate_at => RenegotiateAt}, + renegotiate_at => RenegotiateAt, + log_level => LogLevel}, tls_sender:initialize(Sender, Init). next_tls_record(Data, StateName, #state{protocol_buffers = #protocol_buffers{tls_record_buffer = Buf0, - tls_cipher_texts = CT0} = Buffers} = State0) -> + tls_cipher_texts = CT0} = Buffers, + ssl_options = SslOpts} = State0) -> Versions = + %% TLS 1.3 Client/Server + %% - Ignore TLSPlaintext.legacy_record_version + %% - Verify that TLSCiphertext.legacy_record_version is set to 0x0303 for all records + %% other than an initial ClientHello, where it MAY also be 0x0301. case StateName of hello -> [tls_record:protocol_version(Vsn) || Vsn <- ?ALL_AVAILABLE_VERSIONS]; _ -> State0#state.connection_env#connection_env.negotiated_version end, - case tls_record:get_tls_records(Data, Versions, Buf0) of + case tls_record:get_tls_records(Data, Versions, Buf0, SslOpts) of {Records, Buf1} -> CT1 = CT0 ++ Records, next_record(State0#state{protocol_buffers = @@ -972,6 +1152,7 @@ encode_handshake(Handshake, Version, ConnectionStates0, Hist0) -> encode_change_cipher(#change_cipher_spec{}, Version, ConnectionStates) -> tls_record:encode_change_cipher_spec(Version, ConnectionStates). +-spec decode_alerts(binary()) -> list(). decode_alerts(Bin) -> ssl_alert:decode(Bin). @@ -987,12 +1168,26 @@ gen_handshake(StateName, Type, Event, Version, StateName, State) end. + +gen_handshake_1_3(StateName, Type, Event, + #state{connection_env = #connection_env{negotiated_version = Version}} = State) -> + try tls_connection_1_3:StateName(Type, Event, State, ?MODULE) of + Result -> + Result + catch + _:_ -> + ssl_connection:handle_own_alert(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, + malformed_handshake_data), + Version, StateName, State) + end. + + gen_info(Event, connection = StateName, #state{connection_env = #connection_env{negotiated_version = Version}} = State) -> try handle_info(Event, StateName, State) of Result -> Result catch - _:_ -> + _:_ -> ssl_connection:handle_own_alert(?ALERT_REC(?FATAL, ?INTERNAL_ERROR, malformed_data), Version, StateName, State) @@ -1008,6 +1203,29 @@ gen_info(Event, StateName, #state{connection_env = #connection_env{negotiated_ve malformed_handshake_data), Version, StateName, State) end. + +gen_info_1_3(Event, connected = StateName, #state{connection_env = #connection_env{negotiated_version = Version}} = State) -> + try handle_info(Event, StateName, State) of + Result -> + Result + catch + _:_ -> + ssl_connection:handle_own_alert(?ALERT_REC(?FATAL, ?INTERNAL_ERROR, + malformed_data), + Version, StateName, State) + end; + +gen_info_1_3(Event, StateName, #state{connection_env = #connection_env{negotiated_version = Version}} = State) -> + try handle_info(Event, StateName, State) of + Result -> + Result + catch + _:_ -> + ssl_connection:handle_own_alert(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, + malformed_handshake_data), + Version, StateName, State) + end. + unprocessed_events(Events) -> %% The first handshake event will be processed immediately @@ -1052,3 +1270,34 @@ ensure_sender_terminate(_, #state{protocol_specific = #{sender := Sender}}) -> end end, spawn(Kill). + +maybe_generate_client_shares(#ssl_options{ + versions = [Version|_], + supported_groups = + #supported_groups{ + supported_groups = Groups}}) + when Version =:= {3,4} -> + ssl_cipher:generate_client_shares(Groups); +maybe_generate_client_shares(_) -> + undefined. + +choose_tls_version(#ssl_options{versions = Versions}, + #client_hello{ + extensions = #{client_hello_versions := + #client_hello_versions{versions = ClientVersions} + } + }) -> + case ssl_handshake:select_supported_version(ClientVersions, Versions) of + {3,4} -> + 'tls_v1.3'; + _Else -> + 'tls_v1.2' + end; +choose_tls_version(_, _) -> + 'tls_v1.2'. + + +effective_version(undefined, #ssl_options{versions = [Version|_]}) -> + Version; +effective_version(Version, _) -> + Version. diff --git a/lib/ssl/src/tls_connection_1_3.erl b/lib/ssl/src/tls_connection_1_3.erl new file mode 100644 index 0000000000..701a5860c2 --- /dev/null +++ b/lib/ssl/src/tls_connection_1_3.erl @@ -0,0 +1,192 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2007-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%%---------------------------------------------------------------------- +%% Purpose: TODO +%%---------------------------------------------------------------------- + +%% RFC 8446 +%% A.1. Client +%% +%% START <----+ +%% Send ClientHello | | Recv HelloRetryRequest +%% [K_send = early data] | | +%% v | +%% / WAIT_SH ----+ +%% | | Recv ServerHello +%% | | K_recv = handshake +%% Can | V +%% send | WAIT_EE +%% early | | Recv EncryptedExtensions +%% data | +--------+--------+ +%% | Using | | Using certificate +%% | PSK | v +%% | | WAIT_CERT_CR +%% | | Recv | | Recv CertificateRequest +%% | | Certificate | v +%% | | | WAIT_CERT +%% | | | | Recv Certificate +%% | | v v +%% | | WAIT_CV +%% | | | Recv CertificateVerify +%% | +> WAIT_FINISHED <+ +%% | | Recv Finished +%% \ | [Send EndOfEarlyData] +%% | K_send = handshake +%% | [Send Certificate [+ CertificateVerify]] +%% Can send | Send Finished +%% app data --> | K_send = K_recv = application +%% after here v +%% CONNECTED +%% +%% A.2. Server +%% +%% START <-----+ +%% Recv ClientHello | | Send HelloRetryRequest +%% v | +%% RECVD_CH ----+ +%% | Select parameters +%% v +%% NEGOTIATED +%% | Send ServerHello +%% | K_send = handshake +%% | Send EncryptedExtensions +%% | [Send CertificateRequest] +%% Can send | [Send Certificate + CertificateVerify] +%% app data | Send Finished +%% after --> | K_send = application +%% here +--------+--------+ +%% No 0-RTT | | 0-RTT +%% | | +%% K_recv = handshake | | K_recv = early data +%% [Skip decrypt errors] | +------> WAIT_EOED -+ +%% | | Recv | | Recv EndOfEarlyData +%% | | early data | | K_recv = handshake +%% | +------------+ | +%% | | +%% +> WAIT_FLIGHT2 <--------+ +%% | +%% +--------+--------+ +%% No auth | | Client auth +%% | | +%% | v +%% | WAIT_CERT +%% | Recv | | Recv Certificate +%% | empty | v +%% | Certificate | WAIT_CV +%% | | | Recv +%% | v | CertificateVerify +%% +-> WAIT_FINISHED <---+ +%% | Recv Finished +%% | K_recv = application +%% v +%% CONNECTED + +-module(tls_connection_1_3). + +-include("ssl_alert.hrl"). +-include("ssl_connection.hrl"). +-include("tls_handshake.hrl"). +-include("tls_handshake_1_3.hrl"). + +%% gen_statem helper functions +-export([start/4, + negotiated/4, + wait_cert/4, + wait_cv/4, + wait_finished/4 + ]). + + +start(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +start(internal, #client_hello{} = Hello, State0, _Module) -> + case tls_handshake_1_3:do_start(Hello, State0) of + #alert{} = Alert -> + ssl_connection:handle_own_alert(Alert, {3,4}, start, State0); + {State, start} -> + {next_state, start, State, []}; + {State, negotiated} -> + {next_state, negotiated, State, [{next_event, internal, start_handshake}]} + end; +start(Type, Msg, State, Connection) -> + ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). + + +negotiated(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +negotiated(internal, Message, State0, _Module) -> + case tls_handshake_1_3:do_negotiated(Message, State0) of + #alert{} = Alert -> + ssl_connection:handle_own_alert(Alert, {3,4}, negotiated, State0); + {State, NextState} -> + {next_state, NextState, State, []} + end. + + +wait_cert(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +wait_cert(internal, + #certificate_1_3{} = Certificate, State0, _Module) -> + case tls_handshake_1_3:do_wait_cert(Certificate, State0) of + {#alert{} = Alert, State} -> + ssl_connection:handle_own_alert(Alert, {3,4}, wait_cert, State); + {State1, NextState} -> + {Record, State} = tls_connection:next_record(State1), + tls_connection:next_event(NextState, Record, State) + end; +wait_cert(Type, Msg, State, Connection) -> + ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). + + +wait_cv(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +wait_cv(internal, + #certificate_verify_1_3{} = CertificateVerify, State0, _Module) -> + case tls_handshake_1_3:do_wait_cv(CertificateVerify, State0) of + {#alert{} = Alert, State} -> + ssl_connection:handle_own_alert(Alert, {3,4}, wait_cv, State); + {State1, NextState} -> + {Record, State} = tls_connection:next_record(State1), + tls_connection:next_event(NextState, Record, State) + end; +wait_cv(Type, Msg, State, Connection) -> + ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). + + +wait_finished(internal, #change_cipher_spec{}, State0, _Module) -> + {Record, State} = tls_connection:next_record(State0), + tls_connection:next_event(?FUNCTION_NAME, Record, State); +wait_finished(internal, + #finished{} = Finished, State0, Module) -> + case tls_handshake_1_3:do_wait_finished(Finished, State0) of + #alert{} = Alert -> + ssl_connection:handle_own_alert(Alert, {3,4}, finished, State0); + State1 -> + {Record, State} = ssl_connection:prepare_connection(State1, Module), + tls_connection:next_event(connection, Record, State) + end; +wait_finished(Type, Msg, State, Connection) -> + ssl_connection:handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection). diff --git a/lib/ssl/src/tls_handshake.erl b/lib/ssl/src/tls_handshake.erl index c2b0d8e039..2480e05097 100644 --- a/lib/ssl/src/tls_handshake.erl +++ b/lib/ssl/src/tls_handshake.erl @@ -26,15 +26,17 @@ -module(tls_handshake). -include("tls_handshake.hrl"). +-include("tls_handshake_1_3.hrl"). -include("tls_record.hrl"). -include("ssl_alert.hrl"). -include("ssl_internal.hrl"). -include("ssl_cipher.hrl"). -include("ssl_api.hrl"). -include_lib("public_key/include/public_key.hrl"). +-include_lib("kernel/include/logger.hrl"). %% Handshake handling --export([client_hello/8, hello/4]). +-export([client_hello/9, hello/4]). %% Handshake encoding -export([encode_handshake/2]). @@ -49,7 +51,8 @@ %%==================================================================== %%-------------------------------------------------------------------- -spec client_hello(ssl:host(), inet:port_number(), ssl_record:connection_states(), - #ssl_options{}, integer(), atom(), boolean(), der_cert()) -> + #ssl_options{}, integer(), atom(), boolean(), der_cert(), + #key_share_client_hello{} | undefined) -> #client_hello{}. %% %% Description: Creates a client hello message. @@ -59,19 +62,32 @@ client_hello(Host, Port, ConnectionStates, ciphers = UserSuites, fallback = Fallback } = SslOpts, - Cache, CacheCb, Renegotiation, OwnCert) -> + Cache, CacheCb, Renegotiation, OwnCert, KeyShare) -> Version = tls_record:highest_protocol_version(Versions), + + %% In TLS 1.3, the client indicates its version preferences in the + %% "supported_versions" extension (Section 4.2.1) and the + %% legacy_version field MUST be set to 0x0303, which is the version + %% number for TLS 1.2. + LegacyVersion = + case tls_record:is_higher(Version, {3,2}) of + true -> + {3,3}; + false -> + Version + end, #{security_parameters := SecParams} = ssl_record:pending_connection_state(ConnectionStates, read), AvailableCipherSuites = ssl_handshake:available_suites(UserSuites, Version), Extensions = ssl_handshake:client_hello_extensions(Version, AvailableCipherSuites, SslOpts, ConnectionStates, - Renegotiation), + Renegotiation, + KeyShare), CipherSuites = ssl_handshake:cipher_suites(AvailableCipherSuites, Renegotiation, Fallback), Id = ssl_session:client_id({Host, Port, SslOpts}, Cache, CacheCb, OwnCert), #client_hello{session_id = Id, - client_version = Version, + client_version = LegacyVersion, cipher_suites = CipherSuites, compression_methods = ssl_record:compressions(), random = SecParams#security_parameters.client_random, @@ -88,11 +104,69 @@ client_hello(Host, Port, ConnectionStates, ssl_record:connection_states(), alpn | npn, binary() | undefined}| {tls_record:tls_version(), {resumed | new, #session{}}, ssl_record:connection_states(), binary() | undefined, - #hello_extensions{}, {ssl:hash(), ssl:sign_algo()} | + HelloExt::map(), {ssl:hash(), ssl:sign_algo()} | undefined} | #alert{}. %% %% Description: Handles a received hello message %%-------------------------------------------------------------------- + + +%% TLS 1.3 - Section 4.1.3 +%% TLS 1.3 clients receiving a ServerHello indicating TLS 1.2 or below +%% MUST check that the last eight bytes are not equal to either of these +%% values. +hello(#server_hello{server_version = {Major, Minor}, + random = <<_:24/binary,Down:8/binary>>}, + #ssl_options{versions = [{M,N}|_]}, _, _) + when (M > 3 orelse M =:= 3 andalso N >= 4) andalso %% TLS 1.3 client + (Major =:= 3 andalso Minor =:= 3 andalso %% Negotiating TLS 1.2 + Down =:= ?RANDOM_OVERRIDE_TLS12) orelse + + (M > 3 orelse M =:= 3 andalso N >= 4) andalso %% TLS 1.3 client + (Major =:= 3 andalso Minor < 3 andalso %% Negotiating TLS 1.1 or prior + Down =:= ?RANDOM_OVERRIDE_TLS11) -> + ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER); + +%% TLS 1.2 clients SHOULD also check that the last eight bytes are not +%% equal to the second value if the ServerHello indicates TLS 1.1 or below. +hello(#server_hello{server_version = {Major, Minor}, + random = <<_:24/binary,Down:8/binary>>}, + #ssl_options{versions = [{M,N}|_]}, _, _) + when (M =:= 3 andalso N =:= 3) andalso %% TLS 1.2 client + (Major =:= 3 andalso Minor < 3 andalso %% Negotiating TLS 1.1 or prior + Down =:= ?RANDOM_OVERRIDE_TLS11) -> + ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER); + + +%% TLS 1.3 - 4.2.1. Supported Versions +%% If the "supported_versions" extension in the ServerHello contains a +%% version not offered by the client or contains a version prior to TLS +%% 1.3, the client MUST abort the handshake with an "illegal_parameter" +%% alert. +%%-------------------------------------------------------------------- +%% TLS 1.2 Client +%% +%% - If "supported_version" is present (ServerHello): +%% - Abort handshake with an "illegal_parameter" alert +hello(#server_hello{server_version = Version, + extensions = #{server_hello_selected_version := + #server_hello_selected_version{selected_version = Version}} + }, + #ssl_options{versions = SupportedVersions}, + _ConnectionStates0, _Renegotiation) -> + case tls_record:is_higher({3,4}, Version) of + true -> + ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER); + false -> + case tls_record:is_acceptable_version(Version, SupportedVersions) of + true -> + %% Implement TLS 1.3 statem ??? + ?ALERT_REC(?FATAL, ?PROTOCOL_VERSION); + false -> + ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER) + end + end; + hello(#server_hello{server_version = Version, random = Random, cipher_suite = CipherSuite, compression_method = Compression, @@ -107,6 +181,36 @@ hello(#server_hello{server_version = Version, random = Random, false -> ?ALERT_REC(?FATAL, ?PROTOCOL_VERSION) end; + + +%% TLS 1.2 Server +%% - If "supported_versions" is present (ClientHello): +%% - Select version from "supported_versions" (ignore ClientHello.legacy_version) +%% - If server only supports versions greater than "supported_versions": +%% - Abort handshake with a "protocol_version" alert (*) +%% - If "supported_versions" is absent (ClientHello): +%% - Negotiate the minimum of ClientHello.legacy_version and TLS 1.2 (**) +%% - If server only supports versions greater than ClientHello.legacy_version: +%% - Abort handshake with a "protocol_version" alert +%% +%% (*) Sends alert even if there is a gap in supported versions +%% e.g. Server 1.0,1.2 Client 1.1,1.3 +%% (**) Current implementation can negotiate a version not supported by the client +%% e.g. Server 1.0,1.2 Client 1.1 -> ServerHello 1.0 +hello(#client_hello{client_version = _ClientVersion, + cipher_suites = CipherSuites, + extensions = #{client_hello_versions := + #client_hello_versions{versions = ClientVersions} + }} = Hello, + #ssl_options{versions = Versions} = SslOpts, + Info, Renegotiation) -> + try + Version = ssl_handshake:select_supported_version(ClientVersions, Versions), + do_hello(Version, Versions, CipherSuites, Hello, SslOpts, Info, Renegotiation) + catch + _:_ -> + ?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, malformed_handshake_data) + end; hello(#client_hello{client_version = ClientVersion, cipher_suites = CipherSuites} = Hello, @@ -114,18 +218,7 @@ hello(#client_hello{client_version = ClientVersion, Info, Renegotiation) -> try Version = ssl_handshake:select_version(tls_record, ClientVersion, Versions), - case ssl_cipher:is_fallback(CipherSuites) of - true -> - Highest = tls_record:highest_protocol_version(Versions), - case tls_record:is_higher(Highest, Version) of - true -> - ?ALERT_REC(?FATAL, ?INAPPROPRIATE_FALLBACK); - false -> - handle_client_hello(Version, Hello, SslOpts, Info, Renegotiation) - end; - false -> - handle_client_hello(Version, Hello, SslOpts, Info, Renegotiation) - end + do_hello(Version, Versions, CipherSuites, Hello, SslOpts, Info, Renegotiation) catch error:{case_clause,{asn1, Asn1Reason}} -> %% ASN-1 decode of certificate somehow failed @@ -140,7 +233,8 @@ hello(#client_hello{client_version = ClientVersion, %%-------------------------------------------------------------------- %%-------------------------------------------------------------------- --spec encode_handshake(tls_handshake(), tls_record:tls_version()) -> iolist(). +-spec encode_handshake(tls_handshake() | tls_handshake_1_3:tls_handshake_1_3(), + tls_record:tls_version()) -> iolist(). %% %% Description: Encode a handshake packet %%-------------------------------------------------------------------- @@ -176,10 +270,7 @@ handle_client_hello(Version, cipher_suites = CipherSuites, compression_methods = Compressions, random = Random, - extensions = - #hello_extensions{elliptic_curves = Curves, - signature_algs = ClientHashSigns} - = HelloExt}, + extensions = HelloExt}, #ssl_options{versions = Versions, signature_algs = SupportedHashSigns, eccs = SupportedECCs, @@ -188,6 +279,9 @@ handle_client_hello(Version, Renegotiation) -> case tls_record:is_acceptable_version(Version, Versions) of true -> + Curves = maps:get(elliptic_curves, HelloExt, undefined), + ClientHashSigns = maps:get(signature_algs, HelloExt, undefined), + ClientSignatureSchemes = maps:get(signature_algs_cert, HelloExt, undefined), AvailableHashSigns = ssl_handshake:available_signature_algs( ClientHashSigns, SupportedHashSigns, Cert, Version), ECCCurve = ssl_handshake:select_curve(Curves, SupportedECCs, ECCOrder), @@ -200,9 +294,11 @@ handle_client_hello(Version, no_suite -> ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_ciphers); _ -> - #{key_exchange := KeyExAlg} = ssl_cipher_format:suite_definition(CipherSuite), - case ssl_handshake:select_hashsign(ClientHashSigns, Cert, KeyExAlg, - SupportedHashSigns, Version) of + #{key_exchange := KeyExAlg} = ssl_cipher_format:suite_bin_to_map(CipherSuite), + case ssl_handshake:select_hashsign({ClientHashSigns, ClientSignatureSchemes}, + Cert, KeyExAlg, + SupportedHashSigns, + Version) of #alert{} = Alert -> Alert; HashSign -> @@ -243,8 +339,26 @@ handle_server_hello_extensions(Version, SessionId, Random, CipherSuite, catch throw:Alert -> Alert end. + + +do_hello(undefined, _Versions, _CipherSuites, _Hello, _SslOpts, _Info, _Renegotiation) -> + ?ALERT_REC(?FATAL, ?PROTOCOL_VERSION); +do_hello(Version, Versions, CipherSuites, Hello, SslOpts, Info, Renegotiation) -> + case ssl_cipher:is_fallback(CipherSuites) of + true -> + Highest = tls_record:highest_protocol_version(Versions), + case tls_record:is_higher(Highest, Version) of + true -> + ?ALERT_REC(?FATAL, ?INAPPROPRIATE_FALLBACK); + false -> + handle_client_hello(Version, Hello, SslOpts, Info, Renegotiation) + end; + false -> + handle_client_hello(Version, Hello, SslOpts, Info, Renegotiation) + end. + %%-------------------------------------------------------------------- -enc_handshake(#hello_request{}, _Version) -> +enc_handshake(#hello_request{}, {3, N}) when N < 4 -> {?HELLO_REQUEST, <<>>}; enc_handshake(#client_hello{client_version = {Major, Minor}, random = Random, @@ -263,7 +377,8 @@ enc_handshake(#client_hello{client_version = {Major, Minor}, ?BYTE(SIDLength), SessionID/binary, ?UINT16(CsLength), BinCipherSuites/binary, ?BYTE(CmLength), BinCompMethods/binary, ExtensionsBin/binary>>}; - +enc_handshake(HandshakeMsg, {3, 4}) -> + tls_handshake_1_3:encode_handshake(HandshakeMsg); enc_handshake(HandshakeMsg, Version) -> ssl_handshake:encode_handshake(HandshakeMsg, Version). @@ -274,6 +389,7 @@ get_tls_handshake_aux(Version, <<?BYTE(Type), ?UINT24(Length), Raw = <<?BYTE(Type), ?UINT24(Length), Body/binary>>, try decode_handshake(Version, Type, Body) of Handshake -> + ssl_logger:debug(Opts#ssl_options.log_level, inbound, 'handshake', Handshake), get_tls_handshake_aux(Version, Rest, Opts, [{Handshake,Raw} | Acc]) catch _:_ -> @@ -282,24 +398,26 @@ get_tls_handshake_aux(Version, <<?BYTE(Type), ?UINT24(Length), get_tls_handshake_aux(_Version, Data, _, Acc) -> {lists:reverse(Acc), Data}. -decode_handshake(_, ?HELLO_REQUEST, <<>>) -> +decode_handshake({3, N}, ?HELLO_REQUEST, <<>>) when N < 4 -> #hello_request{}; -decode_handshake(_Version, ?CLIENT_HELLO, +decode_handshake(Version, ?CLIENT_HELLO, <<?BYTE(Major), ?BYTE(Minor), Random:32/binary, ?BYTE(SID_length), Session_ID:SID_length/binary, ?UINT16(Cs_length), CipherSuites:Cs_length/binary, ?BYTE(Cm_length), Comp_methods:Cm_length/binary, Extensions/binary>>) -> - DecodedExtensions = ssl_handshake:decode_hello_extensions({client, Extensions}), + Exts = ssl_handshake:decode_vector(Extensions), + DecodedExtensions = ssl_handshake:decode_hello_extensions(Exts, Version, {Major, Minor}, + client_hello), #client_hello{ client_version = {Major,Minor}, random = Random, session_id = Session_ID, cipher_suites = ssl_handshake:decode_suites('2_bytes', CipherSuites), - compression_methods = Comp_methods, + compression_methods = erlang:binary_to_list(Comp_methods), extensions = DecodedExtensions }; +decode_handshake({3, 4}, Tag, Msg) -> + tls_handshake_1_3:decode_handshake(Tag, Msg); decode_handshake(Version, Tag, Msg) -> ssl_handshake:decode_handshake(Version, Tag, Msg). - - diff --git a/lib/ssl/src/tls_handshake.hrl b/lib/ssl/src/tls_handshake.hrl index f6644f64af..fc67bb61fd 100644 --- a/lib/ssl/src/tls_handshake.hrl +++ b/lib/ssl/src/tls_handshake.hrl @@ -32,6 +32,7 @@ client_version, random, session_id, % opaque SessionID<0..32> + cookie, % opaque<2..2^16-1> cipher_suites, % cipher_suites<2..2^16-1> compression_methods, % compression_methods<1..2^8-1>, %% Extensions diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl new file mode 100644 index 0000000000..8a4ad922e1 --- /dev/null +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -0,0 +1,1343 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2007-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%%---------------------------------------------------------------------- +%% Purpose: Help funtions for handling the TLS 1.3 (specific parts of) +%%% TLS handshake protocol +%%---------------------------------------------------------------------- + +-module(tls_handshake_1_3). + +-include("tls_handshake_1_3.hrl"). +-include("ssl_alert.hrl"). +-include("ssl_cipher.hrl"). +-include("ssl_connection.hrl"). +-include("ssl_internal.hrl"). +-include("ssl_record.hrl"). +-include_lib("public_key/include/public_key.hrl"). + +%% Encode +-export([encode_handshake/1, decode_handshake/2]). + +%% Create handshake messages +-export([certificate/5, + certificate_verify/4, + encrypted_extensions/0, + server_hello/4]). + +-export([do_start/2, + do_negotiated/2, + do_wait_cert/2, + do_wait_cv/2, + do_wait_finished/2]). + +%%==================================================================== +%% Create handshake messages +%%==================================================================== + +server_hello(MsgType, SessionId, KeyShare, ConnectionStates) -> + #{security_parameters := SecParams} = + ssl_record:pending_connection_state(ConnectionStates, read), + Extensions = server_hello_extensions(MsgType, KeyShare), + #server_hello{server_version = {3,3}, %% legacy_version + cipher_suite = SecParams#security_parameters.cipher_suite, + compression_method = 0, %% legacy attribute + random = server_hello_random(MsgType, SecParams), + session_id = SessionId, + extensions = Extensions + }. + +server_hello_extensions(MsgType, KeyShare) -> + SupportedVersions = #server_hello_selected_version{selected_version = {3,4}}, + Extensions = #{server_hello_selected_version => SupportedVersions}, + ssl_handshake:add_server_share(MsgType, Extensions, KeyShare). + +server_hello_random(server_hello, #security_parameters{server_random = Random}) -> + Random; +%% For reasons of backward compatibility with middleboxes (see +%% Appendix D.4), the HelloRetryRequest message uses the same structure +%% as the ServerHello, but with Random set to the special value of the +%% SHA-256 of "HelloRetryRequest": +%% +%% CF 21 AD 74 E5 9A 61 11 BE 1D 8C 02 1E 65 B8 91 +%% C2 A2 11 16 7A BB 8C 5E 07 9E 09 E2 C8 A8 33 9C +server_hello_random(hello_retry_request, _) -> + crypto:hash(sha256, "HelloRetryRequest"). + + +%% TODO: implement support for encrypted_extensions +encrypted_extensions() -> + #encrypted_extensions{ + extensions = #{} + }. + + +certificate_request(SignAlgs0, SignAlgsCert0) -> + %% Input arguments contain TLS 1.2 algorithms due to backward compatibility + %% reasons. These {Hash, Algo} tuples must be filtered before creating the + %% the extensions. + SignAlgs = filter_tls13_algs(SignAlgs0), + SignAlgsCert = filter_tls13_algs(SignAlgsCert0), + Extensions0 = add_signature_algorithms(#{}, SignAlgs), + Extensions = add_signature_algorithms_cert(Extensions0, SignAlgsCert), + #certificate_request_1_3{ + certificate_request_context = <<>>, + extensions = Extensions}. + + +add_signature_algorithms(Extensions, SignAlgs) -> + Extensions#{signature_algorithms => + #signature_algorithms{signature_scheme_list = SignAlgs}}. + + +add_signature_algorithms_cert(Extensions, undefined) -> + Extensions; +add_signature_algorithms_cert(Extensions, SignAlgsCert) -> + Extensions#{signature_algorithms_cert => + #signature_algorithms{signature_scheme_list = SignAlgsCert}}. + + +filter_tls13_algs(undefined) -> undefined; +filter_tls13_algs(Algo) -> + lists:filter(fun is_atom/1, Algo). + + +%% TODO: use maybe monad for error handling! +%% enum { +%% X509(0), +%% RawPublicKey(2), +%% (255) +%% } CertificateType; +%% +%% struct { +%% select (certificate_type) { +%% case RawPublicKey: +%% /* From RFC 7250 ASN.1_subjectPublicKeyInfo */ +%% opaque ASN1_subjectPublicKeyInfo<1..2^24-1>; +%% +%% case X509: +%% opaque cert_data<1..2^24-1>; +%% }; +%% Extension extensions<0..2^16-1>; +%% } CertificateEntry; +%% +%% struct { +%% opaque certificate_request_context<0..2^8-1>; +%% CertificateEntry certificate_list<0..2^24-1>; +%% } Certificate; +certificate(OwnCert, CertDbHandle, CertDbRef, _CRContext, server) -> + case ssl_certificate:certificate_chain(OwnCert, CertDbHandle, CertDbRef) of + {ok, _, Chain} -> + CertList = chain_to_cert_list(Chain), + %% If this message is in response to a CertificateRequest, the value of + %% certificate_request_context in that message. Otherwise (in the case + %%of server authentication), this field SHALL be zero length. + #certificate_1_3{ + certificate_request_context = <<>>, + certificate_list = CertList}; + {error, Error} -> + ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {server_has_no_suitable_certificates, Error}) + end. + + +certificate_verify(PrivateKey, SignatureScheme, + #state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + tls_handshake_history = {Messages, _}}}, server) -> + #{security_parameters := SecParamsR} = + ssl_record:pending_connection_state(ConnectionStates, write), + #security_parameters{prf_algorithm = HKDFAlgo} = SecParamsR, + + {HashAlgo, _, _} = + ssl_cipher:scheme_to_components(SignatureScheme), + + Context = lists:reverse(Messages), + + %% Transcript-Hash uses the HKDF hash function defined by the cipher suite. + THash = tls_v1:transcript_hash(Context, HKDFAlgo), + + %% Digital signatures use the hash function defined by the selected signature + %% scheme. + case sign(THash, <<"TLS 1.3, server CertificateVerify">>, + HashAlgo, PrivateKey) of + {ok, Signature} -> + {ok, #certificate_verify_1_3{ + algorithm = SignatureScheme, + signature = Signature + }}; + {error, badarg} -> + {error, badarg} + + end. + + +finished(#state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + tls_handshake_history = {Messages, _}}}) -> + #{security_parameters := SecParamsR, + cipher_state := #cipher_state{finished_key = FinishedKey}} = + ssl_record:current_connection_state(ConnectionStates, write), + #security_parameters{prf_algorithm = HKDFAlgo} = SecParamsR, + + VerifyData = tls_v1:finished_verify_data(FinishedKey, HKDFAlgo, Messages), + + #finished{ + verify_data = VerifyData + }. + + +%%==================================================================== +%% Encode handshake +%%==================================================================== + +encode_handshake(#certificate_request_1_3{ + certificate_request_context = Context, + extensions = Exts})-> + EncContext = encode_cert_req_context(Context), + BinExts = encode_extensions(Exts), + {?CERTIFICATE_REQUEST, <<EncContext/binary, BinExts/binary>>}; +encode_handshake(#certificate_1_3{ + certificate_request_context = Context, + certificate_list = Entries}) -> + EncContext = encode_cert_req_context(Context), + EncEntries = encode_cert_entries(Entries), + {?CERTIFICATE, <<EncContext/binary, EncEntries/binary>>}; +encode_handshake(#certificate_verify_1_3{ + algorithm = Algorithm, + signature = Signature}) -> + EncAlgo = encode_algorithm(Algorithm), + EncSign = encode_signature(Signature), + {?CERTIFICATE_VERIFY, <<EncAlgo/binary, EncSign/binary>>}; +encode_handshake(#encrypted_extensions{extensions = Exts})-> + {?ENCRYPTED_EXTENSIONS, encode_extensions(Exts)}; +encode_handshake(#new_session_ticket{ + ticket_lifetime = LifeTime, + ticket_age_add = Age, + ticket_nonce = Nonce, + ticket = Ticket, + extensions = Exts}) -> + TicketSize = byte_size(Ticket), + BinExts = encode_extensions(Exts), + {?NEW_SESSION_TICKET, <<?UINT32(LifeTime), ?UINT32(Age), + ?BYTE(Nonce), ?UINT16(TicketSize), Ticket/binary, + BinExts/binary>>}; +encode_handshake(#end_of_early_data{}) -> + {?END_OF_EARLY_DATA, <<>>}; +encode_handshake(#key_update{request_update = Update}) -> + {?KEY_UPDATE, <<?BYTE(Update)>>}; +encode_handshake(HandshakeMsg) -> + ssl_handshake:encode_handshake(HandshakeMsg, {3,4}). + + +%%==================================================================== +%% Decode handshake +%%==================================================================== + +decode_handshake(?CERTIFICATE_REQUEST, <<?BYTE(0), ?UINT16(Size), EncExts:Size/binary>>) -> + Exts = decode_extensions(EncExts, certificate_request), + #certificate_request_1_3{ + certificate_request_context = <<>>, + extensions = Exts}; +decode_handshake(?CERTIFICATE_REQUEST, <<?BYTE(CSize), Context:CSize/binary, + ?UINT16(Size), EncExts:Size/binary>>) -> + Exts = decode_extensions(EncExts, certificate_request), + #certificate_request_1_3{ + certificate_request_context = Context, + extensions = Exts}; +decode_handshake(?CERTIFICATE, <<?BYTE(0), ?UINT24(Size), Certs:Size/binary>>) -> + CertList = decode_cert_entries(Certs), + #certificate_1_3{ + certificate_request_context = <<>>, + certificate_list = CertList + }; +decode_handshake(?CERTIFICATE, <<?BYTE(CSize), Context:CSize/binary, + ?UINT24(Size), Certs:Size/binary>>) -> + CertList = decode_cert_entries(Certs), + #certificate_1_3{ + certificate_request_context = Context, + certificate_list = CertList + }; +decode_handshake(?CERTIFICATE_VERIFY, <<?UINT16(EncAlgo), ?UINT16(Size), Signature:Size/binary>>) -> + Algorithm = ssl_cipher:signature_scheme(EncAlgo), + #certificate_verify_1_3{ + algorithm = Algorithm, + signature = Signature}; +decode_handshake(?ENCRYPTED_EXTENSIONS, <<?UINT16(Size), EncExts:Size/binary>>) -> + #encrypted_extensions{ + extensions = decode_extensions(EncExts, encrypted_extensions) + }; +decode_handshake(?NEW_SESSION_TICKET, <<?UINT32(LifeTime), ?UINT32(Age), + ?BYTE(Nonce), ?UINT16(TicketSize), Ticket:TicketSize/binary, + BinExts/binary>>) -> + Exts = decode_extensions(BinExts, encrypted_extensions), + #new_session_ticket{ticket_lifetime = LifeTime, + ticket_age_add = Age, + ticket_nonce = Nonce, + ticket = Ticket, + extensions = Exts}; +decode_handshake(?END_OF_EARLY_DATA, _) -> + #end_of_early_data{}; +decode_handshake(?KEY_UPDATE, <<?BYTE(Update)>>) -> + #key_update{request_update = Update}; +decode_handshake(Tag, HandshakeMsg) -> + ssl_handshake:decode_handshake({3,4}, Tag, HandshakeMsg). + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +encode_cert_req_context(<<>>) -> + <<?BYTE(0)>>; +encode_cert_req_context(Bin) -> + Size = byte_size(Bin), + <<?BYTE(Size), Bin/binary>>. + +encode_cert_entries(Entries) -> + CertEntryList = encode_cert_entries(Entries, []), + Size = byte_size(CertEntryList), + <<?UINT24(Size), CertEntryList/binary>>. + +encode_cert_entries([], Acc) -> + iolist_to_binary(lists:reverse(Acc)); +encode_cert_entries([#certificate_entry{data = Data, + extensions = Exts} | Rest], Acc) -> + DSize = byte_size(Data), + BinExts = encode_extensions(Exts), + encode_cert_entries(Rest, + [<<?UINT24(DSize), Data/binary, BinExts/binary>> | Acc]). + +encode_algorithm(Algo) -> + Scheme = ssl_cipher:signature_scheme(Algo), + <<?UINT16(Scheme)>>. + +encode_signature(Signature) -> + Size = byte_size(Signature), + <<?UINT16(Size), Signature/binary>>. + +decode_cert_entries(Entries) -> + decode_cert_entries(Entries, []). + +decode_cert_entries(<<>>, Acc) -> + lists:reverse(Acc); +decode_cert_entries(<<?UINT24(DSize), Data:DSize/binary, ?UINT16(Esize), BinExts:Esize/binary, + Rest/binary>>, Acc) -> + Exts = decode_extensions(BinExts, certificate_request), + decode_cert_entries(Rest, [#certificate_entry{data = Data, + extensions = Exts} | Acc]). + +encode_extensions(Exts)-> + ssl_handshake:encode_extensions(extensions_list(Exts)). +decode_extensions(Exts, MessageType) -> + ssl_handshake:decode_extensions(Exts, {3,4}, MessageType). + +extensions_list(HelloExtensions) -> + [Ext || {_, Ext} <- maps:to_list(HelloExtensions)]. + + +%% TODO: add extensions! +chain_to_cert_list(L) -> + chain_to_cert_list(L, []). +%% +chain_to_cert_list([], Acc) -> + lists:reverse(Acc); +chain_to_cert_list([H|T], Acc) -> + chain_to_cert_list(T, [certificate_entry(H)|Acc]). + + +certificate_entry(DER) -> + #certificate_entry{ + data = DER, + extensions = #{} %% Extensions not supported. + }. + +%% The digital signature is then computed over the concatenation of: +%% - A string that consists of octet 32 (0x20) repeated 64 times +%% - The context string +%% - A single 0 byte which serves as the separator +%% - The content to be signed +%% +%% For example, if the transcript hash was 32 bytes of 01 (this length +%% would make sense for SHA-256), the content covered by the digital +%% signature for a server CertificateVerify would be: +%% +%% 2020202020202020202020202020202020202020202020202020202020202020 +%% 2020202020202020202020202020202020202020202020202020202020202020 +%% 544c5320312e332c207365727665722043657274696669636174655665726966 +%% 79 +%% 00 +%% 0101010101010101010101010101010101010101010101010101010101010101 +sign(THash, Context, HashAlgo, PrivateKey) -> + Content = build_content(Context, THash), + + %% The length of the Salt MUST be equal to the length of the output + %% of the digest algorithm: rsa_pss_saltlen = -1 + try public_key:sign(Content, HashAlgo, PrivateKey, + [{rsa_padding, rsa_pkcs1_pss_padding}, + {rsa_pss_saltlen, -1}, + {rsa_mgf1_md, HashAlgo}]) of + Signature -> + {ok, Signature} + catch + error:badarg -> + {error, badarg} + end. + + +verify(THash, Context, HashAlgo, Signature, PublicKey) -> + Content = build_content(Context, THash), + + %% The length of the Salt MUST be equal to the length of the output + %% of the digest algorithm: rsa_pss_saltlen = -1 + try public_key:verify(Content, HashAlgo, Signature, PublicKey, + [{rsa_padding, rsa_pkcs1_pss_padding}, + {rsa_pss_saltlen, -1}, + {rsa_mgf1_md, HashAlgo}]) of + Result -> + {ok, Result} + catch + error:badarg -> + {error, badarg} + end. + + +build_content(Context, THash) -> + Prefix = binary:copy(<<32>>, 64), + <<Prefix/binary,Context/binary,?BYTE(0),THash/binary>>. + + +%%==================================================================== +%% Handle handshake messages +%%==================================================================== + + +do_start(#client_hello{cipher_suites = ClientCiphers, + session_id = SessionId, + extensions = Extensions} = _Hello, + #state{connection_states = _ConnectionStates0, + ssl_options = #ssl_options{ciphers = ServerCiphers, + signature_algs = ServerSignAlgs, + supported_groups = ServerGroups0}, + session = #session{own_certificate = Cert}} = State0) -> + + ClientGroups0 = maps:get(elliptic_curves, Extensions, undefined), + ClientGroups = get_supported_groups(ClientGroups0), + ServerGroups = get_supported_groups(ServerGroups0), + + ClientShares0 = maps:get(key_share, Extensions, undefined), + ClientShares = get_key_shares(ClientShares0), + + ClientSignAlgs = get_signature_scheme_list( + maps:get(signature_algs, Extensions, undefined)), + ClientSignAlgsCert = get_signature_scheme_list( + maps:get(signature_algs_cert, Extensions, undefined)), + + %% TODO: use library function if it exists + %% Init the maybe "monad" + {Ref,Maybe} = maybe(), + + try + %% If the server does not select a PSK, then the server independently selects a + %% cipher suite, an (EC)DHE group and key share for key establishment, + %% and a signature algorithm/certificate pair to authenticate itself to + %% the client. + Cipher = Maybe(select_cipher_suite(ClientCiphers, ServerCiphers)), + Groups = Maybe(select_common_groups(ServerGroups, ClientGroups)), + Maybe(validate_key_share(ClientGroups, ClientShares)), + + {PublicKeyAlgo, SignAlgo, SignHash} = get_certificate_params(Cert), + + %% Check if client supports signature algorithm of server certificate + Maybe(check_cert_sign_algo(SignAlgo, SignHash, ClientSignAlgs, ClientSignAlgsCert)), + + %% Select signature algorithm (used in CertificateVerify message). + SelectedSignAlg = Maybe(select_sign_algo(PublicKeyAlgo, ClientSignAlgs, ServerSignAlgs)), + + %% Select client public key. If no public key found in ClientShares or + %% ClientShares is empty, trigger HelloRetryRequest as we were able + %% to find an acceptable set of parameters but the ClientHello does not + %% contain sufficient information. + {Group, ClientPubKey} = get_client_public_key(Groups, ClientShares), + + %% Generate server_share + KeyShare = ssl_cipher:generate_server_share(Group), + + State1 = update_start_state(State0, Cipher, KeyShare, SessionId, + Group, SelectedSignAlg, ClientPubKey), + + %% 4.1.4. Hello Retry Request + %% + %% The server will send this message in response to a ClientHello + %% message if it is able to find an acceptable set of parameters but the + %% ClientHello does not contain sufficient information to proceed with + %% the handshake. + Maybe(send_hello_retry_request(State1, ClientPubKey, KeyShare, SessionId)) + + %% TODO: + %% - session handling + %% - handle extensions: ALPN + %% (do not handle: NPN, srp, renegotiation_info, ec_point_formats) + + catch + {Ref, {insufficient_security, no_suitable_groups}} -> + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_groups); + {Ref, illegal_parameter} -> + ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER); + {Ref, no_suitable_cipher} -> + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_cipher); + {Ref, {insufficient_security, no_suitable_signature_algorithm}} -> + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, "No suitable signature algorithm"); + {Ref, {insufficient_security, no_suitable_public_key}} -> + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_public_key) + end. + + +do_negotiated(start_handshake, + #state{connection_states = ConnectionStates0, + session = #session{session_id = SessionId, + own_certificate = OwnCert, + ecc = SelectedGroup, + sign_alg = SignatureScheme, + dh_public_value = ClientKey}, + ssl_options = #ssl_options{} = SslOpts, + key_share = KeyShare, + handshake_env = #handshake_env{tls_handshake_history = _HHistory0}, + connection_env = #connection_env{private_key = CertPrivateKey}, + static_env = #static_env{ + cert_db = CertDbHandle, + cert_db_ref = CertDbRef, + socket = _Socket, + transport_cb = _Transport} + } = State0) -> + {Ref,Maybe} = maybe(), + + try + %% Create server_hello + %% Extensions: supported_versions, key_share, (pre_shared_key) + ServerHello = server_hello(server_hello, SessionId, KeyShare, ConnectionStates0), + + {State1, _} = tls_connection:send_handshake(ServerHello, State0), + + State2 = + calculate_handshake_secrets(ClientKey, SelectedGroup, KeyShare, State1), + + State3 = ssl_record:step_encryption_state(State2), + + %% Create EncryptedExtensions + EncryptedExtensions = encrypted_extensions(), + + %% Encode EncryptedExtensions + State4 = tls_connection:queue_handshake(EncryptedExtensions, State3), + + %% Create and send CertificateRequest ({verify, verify_peer}) + {State5, NextState} = maybe_send_certificate_request(State4, SslOpts), + + %% Create Certificate + Certificate = certificate(OwnCert, CertDbHandle, CertDbRef, <<>>, server), + + %% Encode Certificate + State6 = tls_connection:queue_handshake(Certificate, State5), + + %% Create CertificateVerify + CertificateVerify = Maybe(certificate_verify(CertPrivateKey, SignatureScheme, + State6, server)), + %% Encode CertificateVerify + State7 = tls_connection:queue_handshake(CertificateVerify, State6), + + %% Create Finished + Finished = finished(State7), + + %% Encode Finished + State8 = tls_connection:queue_handshake(Finished, State7), + + %% Send first flight + {State9, _} = tls_connection:send_handshake_flight(State8), + + {State9, NextState} + + catch + {Ref, badarg} -> + ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {digitally_sign, badarg}) + end. + + +do_wait_cert(#certificate_1_3{} = Certificate, State0) -> + {Ref,Maybe} = maybe(), + try + Maybe(process_client_certificate(Certificate, State0)) + catch + {Ref, {certificate_required, State}} -> + {?ALERT_REC(?FATAL, ?CERTIFICATE_REQUIRED, certificate_required), State}; + {Ref, {{certificate_unknown, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?CERTIFICATE_UNKNOWN, Reason), State}; + {Ref, {{internal_error, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?INTERNAL_ERROR, Reason), State}; + {Ref, {{handshake_failure, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, Reason), State}; + {#alert{} = Alert, State} -> + {Alert, State} + end. + + +do_wait_cv(#certificate_verify_1_3{} = CertificateVerify, State0) -> + {Ref,Maybe} = maybe(), + try + Maybe(verify_signature_algorithm(State0, CertificateVerify)), + Maybe(verify_certificate_verify(State0, CertificateVerify)) + catch + {Ref, {{bad_certificate, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?BAD_CERTIFICATE, {bad_certificate, Reason}), State}; + {Ref, {badarg, State}} -> + {?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {verify, badarg}), State}; + {Ref, {{handshake_failure, Reason}, State}} -> + {?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, {handshake_failure, Reason}), State} + end. + + +do_wait_finished(#finished{verify_data = VerifyData}, + #state{connection_states = _ConnectionStates0, + session = #session{session_id = _SessionId, + own_certificate = _OwnCert}, + ssl_options = #ssl_options{} = _SslOpts, + key_share = _KeyShare, + handshake_env = #handshake_env{tls_handshake_history = _HHistory0}, + static_env = #static_env{ + cert_db = _CertDbHandle, + cert_db_ref = _CertDbRef, + socket = _Socket, + transport_cb = _Transport} + } = State0) -> + + {Ref,Maybe} = maybe(), + + try + Maybe(validate_client_finished(State0, VerifyData)), + + State1 = calculate_traffic_secrets(State0), + + %% Configure traffic keys + ssl_record:step_encryption_state(State1) + + + catch + {Ref, decrypt_error} -> + ?ALERT_REC(?FATAL, ?DECRYPT_ERROR, decrypt_error) + end. + + +%% TODO: Remove this function! +%% not_implemented(State, Reason) -> +%% {error, {not_implemented, State, Reason}}. +%% +%% not_implemented(update_secrets, State0, Reason) -> +%% State1 = calculate_traffic_secrets(State0), +%% State = ssl_record:step_encryption_state(State1), +%% {error, {not_implemented, State, Reason}}. + + + +%% Recipients of Finished messages MUST verify that the contents are +%% correct and if incorrect MUST terminate the connection with a +%% "decrypt_error" alert. +validate_client_finished(#state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + tls_handshake_history = {Messages0, _}}}, VerifyData) -> + #{security_parameters := SecParamsR, + cipher_state := #cipher_state{finished_key = FinishedKey}} = + ssl_record:current_connection_state(ConnectionStates, read), + #security_parameters{prf_algorithm = HKDFAlgo} = SecParamsR, + + %% Drop the client's finished message, it is not part of the handshake context + %% when the client calculates its finished message. + [_|Messages] = Messages0, + + ControlData = tls_v1:finished_verify_data(FinishedKey, HKDFAlgo, Messages), + compare_verify_data(ControlData, VerifyData). + + +compare_verify_data(Data, Data) -> + ok; +compare_verify_data(_, _) -> + {error, decrypt_error}. + + +send_hello_retry_request(#state{connection_states = ConnectionStates0} = State0, + no_suitable_key, KeyShare, SessionId) -> + ServerHello = server_hello(hello_retry_request, SessionId, KeyShare, ConnectionStates0), + {State1, _} = tls_connection:send_handshake(ServerHello, State0), + + %% TODO: Fix handshake history! + State2 = replace_ch1_with_message_hash(State1), + + {ok, {State2, start}}; +send_hello_retry_request(State0, _, _, _) -> + %% Suitable key found. + {ok, {State0, negotiated}}. + + +maybe_send_certificate_request(State, #ssl_options{verify = verify_none}) -> + {State, wait_finished}; +maybe_send_certificate_request(State, #ssl_options{ + verify = verify_peer, + signature_algs = SignAlgs, + signature_algs_cert = SignAlgsCert}) -> + CertificateRequest = certificate_request(SignAlgs, SignAlgsCert), + {tls_connection:queue_handshake(CertificateRequest, State), wait_cert}. + + +process_client_certificate(#certificate_1_3{ + certificate_request_context = <<>>, + certificate_list = []}, + #state{ssl_options = + #ssl_options{ + fail_if_no_peer_cert = false}} = State) -> + {ok, {State, wait_finished}}; +process_client_certificate(#certificate_1_3{ + certificate_request_context = <<>>, + certificate_list = []}, + #state{ssl_options = + #ssl_options{ + fail_if_no_peer_cert = true}} = State0) -> + + %% At this point the client believes that the connection is up and starts using + %% its traffic secrets. In order to be able send an proper Alert to the client + %% the server should also change its connection state and use the traffic + %% secrets. + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {certificate_required, State}}; +process_client_certificate(#certificate_1_3{certificate_list = Certs0}, + #state{ssl_options = + #ssl_options{signature_algs = SignAlgs, + signature_algs_cert = SignAlgsCert} = SslOptions, + static_env = + #static_env{ + role = Role, + host = Host, + cert_db = CertDbHandle, + cert_db_ref = CertDbRef, + crl_db = CRLDbHandle}} = State0) -> + %% TODO: handle extensions! + + %% Remove extensions from list of certificates! + Certs = convert_certificate_chain(Certs0), + case is_supported_signature_algorithm(Certs, SignAlgs, SignAlgsCert) of + true -> + case validate_certificate_chain(Certs, CertDbHandle, CertDbRef, + SslOptions, CRLDbHandle, Role, Host) of + {ok, {PeerCert, PublicKeyInfo}} -> + State = store_peer_cert(State0, PeerCert, PublicKeyInfo), + {ok, {State, wait_cv}}; + {error, Reason} -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {Reason, State}}; + #alert{} = Alert -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {Alert, State} + end; + false -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {{handshake_failure, + "Client certificate uses unsupported signature algorithm"}, State}} + end. + + +%% TODO: check whole chain! +is_supported_signature_algorithm(Certs, SignAlgs, undefined) -> + is_supported_signature_algorithm(Certs, SignAlgs); +is_supported_signature_algorithm(Certs, _, SignAlgsCert) -> + is_supported_signature_algorithm(Certs, SignAlgsCert). +%% +is_supported_signature_algorithm([BinCert|_], SignAlgs0) -> + #'OTPCertificate'{signatureAlgorithm = SignAlg} = + public_key:pkix_decode_cert(BinCert, otp), + SignAlgs = filter_tls13_algs(SignAlgs0), + Scheme = ssl_cipher:signature_algorithm_to_scheme(SignAlg), + lists:member(Scheme, SignAlgs). + + +validate_certificate_chain(Certs, CertDbHandle, CertDbRef, SslOptions, CRLDbHandle, Role, Host) -> + ServerName = ssl_handshake:server_name(SslOptions#ssl_options.server_name_indication, Host, Role), + [PeerCert | ChainCerts ] = Certs, + try + {TrustedCert, CertPath} = + ssl_certificate:trusted_cert_and_path(Certs, CertDbHandle, CertDbRef, + SslOptions#ssl_options.partial_chain), + ValidationFunAndState = + ssl_handshake:validation_fun_and_state(SslOptions#ssl_options.verify_fun, Role, + CertDbHandle, CertDbRef, ServerName, + SslOptions#ssl_options.customize_hostname_check, + SslOptions#ssl_options.crl_check, CRLDbHandle, CertPath), + Options = [{max_path_length, SslOptions#ssl_options.depth}, + {verify_fun, ValidationFunAndState}], + %% TODO: Validate if Certificate is using a supported signature algorithm + %% (signature_algs_cert)! + case public_key:pkix_path_validation(TrustedCert, CertPath, Options) of + {ok, {PublicKeyInfo,_}} -> + {ok, {PeerCert, PublicKeyInfo}}; + {error, Reason} -> + ssl_handshake:handle_path_validation_error(Reason, PeerCert, ChainCerts, + SslOptions, Options, + CertDbHandle, CertDbRef) + end + catch + error:{badmatch,{asn1, Asn1Reason}} -> + %% ASN-1 decode of certificate somehow failed + {error, {certificate_unknown, {failed_to_decode_certificate, Asn1Reason}}}; + error:OtherReason -> + {error, {internal_error, {unexpected_error, OtherReason}}} + end. + + +store_peer_cert(#state{session = Session, + handshake_env = HsEnv} = State, PeerCert, PublicKeyInfo) -> + State#state{session = Session#session{peer_certificate = PeerCert}, + handshake_env = HsEnv#handshake_env{public_key_info = PublicKeyInfo}}. + + +convert_certificate_chain(Certs) -> + Fun = fun(#certificate_entry{data = Data}) -> + {true, Data}; + (_) -> + false + end, + lists:filtermap(Fun, Certs). + + +%% 4.4.1. The Transcript Hash +%% +%% As an exception to this general rule, when the server responds to a +%% ClientHello with a HelloRetryRequest, the value of ClientHello1 is +%% replaced with a special synthetic handshake message of handshake type +%% "message_hash" containing Hash(ClientHello1). I.e., +%% +%% Transcript-Hash(ClientHello1, HelloRetryRequest, ... Mn) = +%% Hash(message_hash || /* Handshake type */ +%% 00 00 Hash.length || /* Handshake message length (bytes) */ +%% Hash(ClientHello1) || /* Hash of ClientHello1 */ +%% HelloRetryRequest || ... || Mn) +%% +%% NOTE: Hash.length is used in practice (openssl) and not message length! +%% It is most probably a fault in the RFC. +replace_ch1_with_message_hash(#state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + tls_handshake_history = + {[HRR,CH1|HHistory], LM}} = HSEnv} = State0) -> + #{security_parameters := SecParamsR} = + ssl_record:pending_connection_state(ConnectionStates, read), + #security_parameters{prf_algorithm = HKDFAlgo} = SecParamsR, + MessageHash = message_hash(CH1, HKDFAlgo), + State0#state{handshake_env = + HSEnv#handshake_env{ + tls_handshake_history = + {[HRR,MessageHash|HHistory], LM}}}. + + +message_hash(ClientHello1, HKDFAlgo) -> + [?MESSAGE_HASH, + 0,0,ssl_cipher:hash_size(HKDFAlgo), + crypto:hash(HKDFAlgo, ClientHello1)]. + + +calculate_handshake_secrets(ClientKey, SelectedGroup, KeyShare, + #state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + tls_handshake_history = HHistory}} = State0) -> + #{security_parameters := SecParamsR} = + ssl_record:pending_connection_state(ConnectionStates, read), + #security_parameters{prf_algorithm = HKDFAlgo, + cipher_suite = CipherSuite} = SecParamsR, + + %% Calculate handshake_secret + PSK = binary:copy(<<0>>, ssl_cipher:hash_size(HKDFAlgo)), + EarlySecret = tls_v1:key_schedule(early_secret, HKDFAlgo , {psk, PSK}), + PrivateKey = get_server_private_key(KeyShare), %% #'ECPrivateKey'{} + + IKM = calculate_shared_secret(ClientKey, PrivateKey, SelectedGroup), + HandshakeSecret = tls_v1:key_schedule(handshake_secret, HKDFAlgo, IKM, EarlySecret), + + %% Calculate [sender]_handshake_traffic_secret + {Messages, _} = HHistory, + ClientHSTrafficSecret = + tls_v1:client_handshake_traffic_secret(HKDFAlgo, HandshakeSecret, lists:reverse(Messages)), + ServerHSTrafficSecret = + tls_v1:server_handshake_traffic_secret(HKDFAlgo, HandshakeSecret, lists:reverse(Messages)), + + %% Calculate traffic keys + #{cipher := Cipher} = ssl_cipher_format:suite_bin_to_map(CipherSuite), + {ReadKey, ReadIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, ClientHSTrafficSecret), + {WriteKey, WriteIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, ServerHSTrafficSecret), + + %% Calculate Finished Keys + ReadFinishedKey = tls_v1:finished_key(ClientHSTrafficSecret, HKDFAlgo), + WriteFinishedKey = tls_v1:finished_key(ServerHSTrafficSecret, HKDFAlgo), + + update_pending_connection_states(State0, HandshakeSecret, + ReadKey, ReadIV, ReadFinishedKey, + WriteKey, WriteIV, WriteFinishedKey). + +calculate_traffic_secrets(#state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + tls_handshake_history = HHistory}} = State0) -> + #{security_parameters := SecParamsR} = + ssl_record:pending_connection_state(ConnectionStates, read), + #security_parameters{prf_algorithm = HKDFAlgo, + cipher_suite = CipherSuite, + master_secret = HandshakeSecret} = SecParamsR, + + MasterSecret = + tls_v1:key_schedule(master_secret, HKDFAlgo, HandshakeSecret), + + %% Get the correct list messages for the handshake context. + Messages = get_handshake_context(HHistory), + + %% Calculate [sender]_application_traffic_secret_0 + ClientAppTrafficSecret0 = + tls_v1:client_application_traffic_secret_0(HKDFAlgo, MasterSecret, lists:reverse(Messages)), + ServerAppTrafficSecret0 = + tls_v1:server_application_traffic_secret_0(HKDFAlgo, MasterSecret, lists:reverse(Messages)), + + %% Calculate traffic keys + #{cipher := Cipher} = ssl_cipher_format:suite_bin_to_map(CipherSuite), + {ReadKey, ReadIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, ClientAppTrafficSecret0), + {WriteKey, WriteIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, ServerAppTrafficSecret0), + + update_pending_connection_states(State0, MasterSecret, + ReadKey, ReadIV, undefined, + WriteKey, WriteIV, undefined). + + +get_server_private_key(#key_share_server_hello{server_share = ServerShare}) -> + get_private_key(ServerShare). + +get_private_key(#key_share_entry{ + key_exchange = #'ECPrivateKey'{} = PrivateKey}) -> + PrivateKey; +get_private_key(#key_share_entry{ + key_exchange = + {_, PrivateKey}}) -> + PrivateKey. + +%% TODO: implement EC keys +get_public_key({?'rsaEncryption', PublicKey, _}) -> + PublicKey. + + +%% X25519, X448 +calculate_shared_secret(OthersKey, MyKey, Group) + when is_binary(OthersKey) andalso is_binary(MyKey) andalso + (Group =:= x25519 orelse Group =:= x448)-> + crypto:compute_key(ecdh, OthersKey, MyKey, Group); +%% FFDHE +calculate_shared_secret(OthersKey, MyKey, Group) + when is_binary(OthersKey) andalso is_binary(MyKey) -> + Params = #'DHParameter'{prime = P} = ssl_dh_groups:dh_params(Group), + S = public_key:compute_key(OthersKey, MyKey, Params), + Size = byte_size(binary:encode_unsigned(P)), + ssl_cipher:add_zero_padding(S, Size); +%% ECDHE +calculate_shared_secret(OthersKey, MyKey = #'ECPrivateKey'{}, _Group) + when is_binary(OthersKey) -> + Point = #'ECPoint'{point = OthersKey}, + public_key:compute_key(Point, MyKey). + + +update_pending_connection_states(#state{connection_states = + CS = #{pending_read := PendingRead0, + pending_write := PendingWrite0}} = State, + HandshakeSecret, + ReadKey, ReadIV, ReadFinishedKey, + WriteKey, WriteIV, WriteFinishedKey) -> + PendingRead = update_connection_state(PendingRead0, HandshakeSecret, + ReadKey, ReadIV, ReadFinishedKey), + PendingWrite = update_connection_state(PendingWrite0, HandshakeSecret, + WriteKey, WriteIV, WriteFinishedKey), + State#state{connection_states = CS#{pending_read => PendingRead, + pending_write => PendingWrite}}. + +update_connection_state(ConnectionState = #{security_parameters := SecurityParameters0}, + HandshakeSecret, Key, IV, FinishedKey) -> + %% Store secret + SecurityParameters = SecurityParameters0#security_parameters{ + master_secret = HandshakeSecret}, + ConnectionState#{security_parameters => SecurityParameters, + cipher_state => cipher_init(Key, IV, FinishedKey)}. + + +update_start_state(#state{connection_states = ConnectionStates0, + connection_env = CEnv, + session = Session} = State, + Cipher, KeyShare, SessionId, + Group, SelectedSignAlg, ClientPubKey) -> + #{security_parameters := SecParamsR0} = PendingRead = + maps:get(pending_read, ConnectionStates0), + #{security_parameters := SecParamsW0} = PendingWrite = + maps:get(pending_write, ConnectionStates0), + SecParamsR = ssl_cipher:security_parameters_1_3(SecParamsR0, Cipher), + SecParamsW = ssl_cipher:security_parameters_1_3(SecParamsW0, Cipher), + ConnectionStates = + ConnectionStates0#{pending_read => PendingRead#{security_parameters => SecParamsR}, + pending_write => PendingWrite#{security_parameters => SecParamsW}}, + State#state{connection_states = ConnectionStates, + key_share = KeyShare, + session = Session#session{session_id = SessionId, + ecc = Group, + sign_alg = SelectedSignAlg, + dh_public_value = ClientPubKey, + cipher_suite = Cipher}, + connection_env = CEnv#connection_env{negotiated_version = {3,4}}}. + + +cipher_init(Key, IV, FinishedKey) -> + #cipher_state{key = Key, + iv = IV, + finished_key = FinishedKey, + tag_len = 16}. + + +%% Get handshake context for verification of CertificateVerify. +%% +%% Verify CertificateVerify: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% CertificateRequest (server) (13) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Certificate (client) (11) +%% CertificateVerify (client) (15) - Drop! Not included in calculations! +get_handshake_context_cv({[<<15,_/binary>>|Messages], _}) -> + Messages. + + +%% Get handshake context for traffic key calculation. +%% +%% Client is authenticated with certificate: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% CertificateRequest (server) (13) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Certificate (client) (11) - Drop! Not included in calculations! +%% CertificateVerify (client) (15) - Drop! Not included in calculations! +%% Finished (client) (20) - Drop! Not included in calculations! +%% +%% Client is authenticated but sends empty certificate: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% CertificateRequest (server) (13) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Certificate (client) (11) - Drop! Not included in calculations! +%% Finished (client) (20) - Drop! Not included in calculations! +%% +%% Client is not authenticated: +%% ClientHello (client) (1) +%% ServerHello (server) (2) +%% EncryptedExtensions (server) (8) +%% Certificate (server) (11) +%% CertificateVerify (server) (15) +%% Finished (server) (20) +%% Finished (client) (20) - Drop! Not included in calculations! +%% +%% Drop all client messages from the front of the iolist using the property that +%% incoming messages are binaries. +get_handshake_context({Messages, _}) -> + get_handshake_context(Messages); +get_handshake_context([H|T]) when is_binary(H) -> + get_handshake_context(T); +get_handshake_context(L) -> + L. + + +%% If sent by a client, the signature algorithm used in the signature +%% MUST be one of those present in the supported_signature_algorithms +%% field of the "signature_algorithms" extension in the +%% CertificateRequest message. +verify_signature_algorithm(#state{ssl_options = + #ssl_options{ + signature_algs = ServerSignAlgs}} = State0, + #certificate_verify_1_3{algorithm = ClientSignAlg}) -> + case lists:member(ClientSignAlg, ServerSignAlgs) of + true -> + ok; + false -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {{handshake_failure, + "CertificateVerify uses unsupported signature algorithm"}, State}} + end. + + +verify_certificate_verify(#state{connection_states = ConnectionStates, + handshake_env = + #handshake_env{ + public_key_info = PublicKeyInfo, + tls_handshake_history = HHistory}} = State0, + #certificate_verify_1_3{ + algorithm = SignatureScheme, + signature = Signature}) -> + #{security_parameters := SecParamsR} = + ssl_record:pending_connection_state(ConnectionStates, write), + #security_parameters{prf_algorithm = HKDFAlgo} = SecParamsR, + + {HashAlgo, _, _} = + ssl_cipher:scheme_to_components(SignatureScheme), + + Messages = get_handshake_context_cv(HHistory), + + Context = lists:reverse(Messages), + + %% Transcript-Hash uses the HKDF hash function defined by the cipher suite. + THash = tls_v1:transcript_hash(Context, HKDFAlgo), + + PublicKey = get_public_key(PublicKeyInfo), + + %% Digital signatures use the hash function defined by the selected signature + %% scheme. + case verify(THash, <<"TLS 1.3, client CertificateVerify">>, + HashAlgo, Signature, PublicKey) of + {ok, true} -> + {ok, {State0, wait_finished}}; + {ok, false} -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {{handshake_failure, "Failed to verify CertificateVerify"}, State}}; + {error, badarg} -> + State1 = calculate_traffic_secrets(State0), + State = ssl_record:step_encryption_state(State1), + {error, {badarg, State}} + end. + + +%% If there is no overlap between the received +%% "supported_groups" and the groups supported by the server, then the +%% server MUST abort the handshake with a "handshake_failure" or an +%% "insufficient_security" alert. +select_common_groups(_, []) -> + {error, {insufficient_security, no_suitable_groups}}; +select_common_groups(ServerGroups, ClientGroups) -> + Fun = fun(E) -> lists:member(E, ClientGroups) end, + case lists:filter(Fun, ServerGroups) of + [] -> + {error, {insufficient_security, no_suitable_groups}}; + L -> + {ok, L} + end. + + +%% RFC 8446 - 4.2.8. Key Share +%% This vector MAY be empty if the client is requesting a +%% HelloRetryRequest. Each KeyShareEntry value MUST correspond to a +%% group offered in the "supported_groups" extension and MUST appear in +%% the same order. However, the values MAY be a non-contiguous subset +%% of the "supported_groups" extension and MAY omit the most preferred +%% groups. +%% +%% Clients can offer as many KeyShareEntry values as the number of +%% supported groups it is offering, each representing a single set of +%% key exchange parameters. +%% +%% Clients MUST NOT offer multiple KeyShareEntry values +%% for the same group. Clients MUST NOT offer any KeyShareEntry values +%% for groups not listed in the client's "supported_groups" extension. +%% Servers MAY check for violations of these rules and abort the +%% handshake with an "illegal_parameter" alert if one is violated. +validate_key_share(_ ,[]) -> + ok; +validate_key_share([], _) -> + {error, illegal_parameter}; +validate_key_share([G|ClientGroups], [{_, G, _}|ClientShares]) -> + validate_key_share(ClientGroups, ClientShares); +validate_key_share([_|ClientGroups], [_|_] = ClientShares) -> + validate_key_share(ClientGroups, ClientShares). + + +get_client_public_key([Group|_] = Groups, ClientShares) -> + get_client_public_key(Groups, ClientShares, Group). +%% +get_client_public_key(_, [], PreferredGroup) -> + {PreferredGroup, no_suitable_key}; +get_client_public_key([], _, PreferredGroup) -> + {PreferredGroup, no_suitable_key}; +get_client_public_key([Group|Groups], ClientShares, PreferredGroup) -> + case lists:keysearch(Group, 2, ClientShares) of + {value, {_, _, ClientPublicKey}} -> + {Group, ClientPublicKey}; + false -> + get_client_public_key(Groups, ClientShares, PreferredGroup) + end. + + +%% get_client_public_key(Group, ClientShares) -> +%% case lists:keysearch(Group, 2, ClientShares) of +%% {value, {_, _, ClientPublicKey}} -> +%% ClientPublicKey; +%% false -> +%% %% 4.1.4. Hello Retry Request +%% %% +%% %% The server will send this message in response to a ClientHello +%% %% message if it is able to find an acceptable set of parameters but the +%% %% ClientHello does not contain sufficient information to proceed with +%% %% the handshake. +%% no_suitable_key +%% end. + +select_cipher_suite([], _) -> + {error, no_suitable_cipher}; +select_cipher_suite([Cipher|ClientCiphers], ServerCiphers) -> + case lists:member(Cipher, tls_v1:suites('TLS_v1.3')) andalso + lists:member(Cipher, ServerCiphers) of + true -> + {ok, Cipher}; + false -> + select_cipher_suite(ClientCiphers, ServerCiphers) + end. + +%% RFC 8446 (TLS 1.3) +%% TLS 1.3 provides two extensions for indicating which signature +%% algorithms may be used in digital signatures. The +%% "signature_algorithms_cert" extension applies to signatures in +%% certificates and the "signature_algorithms" extension, which +%% originally appeared in TLS 1.2, applies to signatures in +%% CertificateVerify messages. +%% +%% If no "signature_algorithms_cert" extension is +%% present, then the "signature_algorithms" extension also applies to +%% signatures appearing in certificates. + +%% Check if the signature algorithm of the server certificate is supported +%% by the client. +check_cert_sign_algo(SignAlgo, SignHash, ClientSignAlgs, undefined) -> + do_check_cert_sign_algo(SignAlgo, SignHash, ClientSignAlgs); +check_cert_sign_algo(SignAlgo, SignHash, _, ClientSignAlgsCert) -> + do_check_cert_sign_algo(SignAlgo, SignHash, ClientSignAlgsCert). + + +%% DSA keys are not supported by TLS 1.3 +select_sign_algo(dsa, _ClientSignAlgs, _ServerSignAlgs) -> + {error, {insufficient_security, no_suitable_public_key}}; +%% TODO: Implement support for ECDSA keys! +select_sign_algo(_, [], _) -> + {error, {insufficient_security, no_suitable_signature_algorithm}}; +select_sign_algo(PublicKeyAlgo, [C|ClientSignAlgs], ServerSignAlgs) -> + {_, S, _} = ssl_cipher:scheme_to_components(C), + %% RSASSA-PKCS1-v1_5 and Legacy algorithms are not defined for use in signed + %% TLS handshake messages: filter sha-1 and rsa_pkcs1. + case ((PublicKeyAlgo =:= rsa andalso S =:= rsa_pss_rsae) + orelse (PublicKeyAlgo =:= rsa_pss andalso S =:= rsa_pss_rsae)) + andalso + lists:member(C, ServerSignAlgs) of + true -> + {ok, C}; + false -> + select_sign_algo(PublicKeyAlgo, ClientSignAlgs, ServerSignAlgs) + end. + + +do_check_cert_sign_algo(_, _, []) -> + {error, {insufficient_security, no_suitable_signature_algorithm}}; +do_check_cert_sign_algo(SignAlgo, SignHash, [Scheme|T]) -> + {Hash, Sign, _Curve} = ssl_cipher:scheme_to_components(Scheme), + case compare_sign_algos(SignAlgo, SignHash, Sign, Hash) of + true -> + ok; + _Else -> + do_check_cert_sign_algo(SignAlgo, SignHash, T) + end. + + +%% id-RSASSA-PSS (rsa_pss) indicates that the key may only be used for PSS signatures. +%% TODO: Uncomment when rsa_pss signatures are supported in certificates +%% compare_sign_algos(rsa_pss, Hash, Algo, Hash) +%% when Algo =:= rsa_pss_pss -> +%% true; +%% rsaEncryption (rsa) allows the key to be used for any of the standard encryption or +%% signature schemes. +compare_sign_algos(rsa, Hash, Algo, Hash) + when Algo =:= rsa_pss_rsae orelse + Algo =:= rsa_pkcs1 -> + true; +compare_sign_algos(Algo, Hash, Algo, Hash) -> + true; +compare_sign_algos(_, _, _, _) -> + false. + + +get_certificate_params(Cert) -> + {SignAlgo0, _Param, PublicKeyAlgo0} = ssl_handshake:get_cert_params(Cert), + {SignHash0, SignAlgo} = public_key:pkix_sign_types(SignAlgo0), + %% Convert hash to new format + SignHash = case SignHash0 of + sha -> + sha1; + H -> H + end, + PublicKeyAlgo = public_key_algo(PublicKeyAlgo0), + {PublicKeyAlgo, SignAlgo, SignHash}. + + +%% Note: copied from ssl_handshake +public_key_algo(?'id-RSASSA-PSS') -> + rsa_pss; +public_key_algo(?rsaEncryption) -> + rsa; +public_key_algo(?'id-ecPublicKey') -> + ecdsa; +public_key_algo(?'id-dsa') -> + dsa. + +get_signature_scheme_list(undefined) -> + undefined; +get_signature_scheme_list(#signature_algorithms_cert{ + signature_scheme_list = ClientSignatureSchemes}) -> + ClientSignatureSchemes; +get_signature_scheme_list(#signature_algorithms{ + signature_scheme_list = ClientSignatureSchemes}) -> + %% Filter unassigned and legacy elements + lists:filter(fun (E) -> is_atom(E) andalso E =/= unassigned end, + ClientSignatureSchemes). + +get_supported_groups(#supported_groups{supported_groups = Groups}) -> + Groups. + +get_key_shares(#key_share_client_hello{client_shares = ClientShares}) -> + ClientShares. + +maybe() -> + Ref = erlang:make_ref(), + Ok = fun(ok) -> ok; + ({ok,R}) -> R; + ({error,Reason}) -> + throw({Ref,Reason}) + end, + {Ref,Ok}. diff --git a/lib/ssl/src/tls_handshake_1_3.hrl b/lib/ssl/src/tls_handshake_1_3.hrl new file mode 100644 index 0000000000..7ae1b93e1c --- /dev/null +++ b/lib/ssl/src/tls_handshake_1_3.hrl @@ -0,0 +1,238 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% +%%---------------------------------------------------------------------- +%% Purpose: Record and constant defenitions for the TLS-handshake protocol +%% see RFC 8446. Also includes supported hello extensions. +%%---------------------------------------------------------------------- + +-ifndef(tls_handshake_1_3). +-define(tls_handshake_1_3, true). + +%% Common to TLS-1.3 and previous TLS versions +%% Some defenitions may not exist in TLS-1.3 this is +%% handled elsewhere +-include("tls_handshake.hrl"). + +%% New handshake types in TLS-1.3 RFC 8446 B.3 +-define(NEW_SESSION_TICKET, 4). +-define(END_OF_EARLY_DATA, 5). +-define(ENCRYPTED_EXTENSIONS, 8). +-define(KEY_UPDATE, 24). +%% %% Not really a message but special way to handle handshake hashes +%% %% when a "hello-retry-request" (special server_hello) is sent +-define(MESSAGE_HASH, 254). + +%% %% RFC 8446 B.3.1. +%% %% New extension types in TLS-1.3 +-define(PRE_SHARED_KEY_EXT, 41). +-define(EARLY_DATA_EXT, 42). +%%-define(SUPPORTED_VERSIONS_EXT, 43). %% Updates TLS 1.2 so defined in ssl_handshake.hrl +-define(COOKIE_EXT, 44). +-define(PSK_KEY_EXCHANGE_MODES_EXT, 45). +-define(CERTIFICATE_AUTHORITIES_EXT, 47). +-define(OID_FILTERS_EXT, 48). +-define(POST_HANDSHAKE_AUTH_EXT, 49). +%% -define(SIGNATURE_ALGORITHMS_CERT_EXT, 50). %% Updates TLS 1.2 so defined in ssl_handshake.hrl +-define(KEY_SHARE_EXT, 51). + +%% RFC 8446 B.3.1 +-record(key_share_entry, { + group, %NamedGroup + key_exchange %key_exchange<1..2^16-1>; + }). +-record(key_share_client_hello, { + client_shares %% KeyShareEntry client_shares<0..2^16-1>; + }). +-record(key_share_hello_retry_request, { + selected_group %% NamedGroup + }). +-record(key_share_server_hello, { + server_share %% KeyShareEntry server_share; + }). + +-record(uncompressed_point_representation, { + legacy_form = 4, % uint8 legacy_form = 4; + x, % opaque X[coordinate_length]; + y % opaque Y[coordinate_length]; + }). + +-define(PSK_KE, 0). +-define(PSK_DHE_KE, 1). + +-record(psk_keyexchange_modes, { + ke_modes % ke_modes<1..255> + }). +-record(empty, { + }). +-record(early_data_indication, { + indication % uint32 max_early_data_size (new_session_ticket) | + %% #empty{} (client_hello, encrypted_extensions) + }). +-record(psk_identity, { + identity, % opaque identity<1..2^16-1> + obfuscated_ticket_age % uint32 + }). +-record(offered_psks, { + psk_identity, %identities<7..2^16-1>; + psk_binder_entry %binders<33..2^16-1>, opaque PskBinderEntry<32..255> + }). +-record(pre_shared_keyextension,{ + extension %OfferedPsks (client_hello) | uint16 selected_identity (server_hello) + }). + +%% RFC 8446 B.3.1.2. +-record(cookie, { + cookie %cookie<1..2^16-1>; + }). + +%%% RFC 8446 B.3.1.3. Signature Algorithm Extension +%% Signature Schemes +%% RSASSA-PKCS1-v1_5 algorithms +-define(RSA_PKCS1_SHA256, 16#0401). +-define(RSA_PKCS1_SHA384, 16#0501). +-define(RSA_PKCS1_SHA512, 16#0601). + +%% ECDSA algorithms +-define(ECDSA_SECP256R1_SHA256, 16#0403). +-define(ECDSA_SECP384R1_SHA384, 16#0503). +-define(ECDSA_SECP521R1_SHA512, 16#0603). + +%% RSASSA-PSS algorithms with public key OID rsaEncryption +-define(RSA_PSS_RSAE_SHA256, 16#0804). +-define(RSA_PSS_RSAE_SHA384, 16#0805). +-define(RSA_PSS_RSAE_SHA512, 16#0806). + +%% EdDSA algorithms +-define(ED25519, 16#0807). +-define(ED448, 16#0808). + +%% RSASSA-PSS algorithms with public key OID RSASSA-PSS +-define(RSA_PSS_PSS_SHA256, 16#0809). +-define(RSA_PSS_PSS_SHA384, 16#080a). +-define(RSA_PSS_PSS_SHA512, 16#080b). + +%% Legacy algorithms +-define(RSA_PKCS1_SHA1, 16#201). +-define(ECDSA_SHA1, 16#0203). + +%% RFC 8446 B.3.1.4. Supported Groups Extension +%% Elliptic Curve Groups (ECDHE) +-define(SECP256R1, 16#0017). +-define(SECP384R1, 16#0018). +-define(SECP521R1, 16#0019). +-define(X25519, 16#001D). +-define(X448, 16#001E). + +%% RFC 8446 Finite Field Groups (DHE) +-define(FFDHE2048, 16#0100). +-define(FFDHE3072, 16#0101). +-define(FFDHE4096, 16#0102). +-define(FFDHE6144, 16#0103). +-define(FFDHE8192 ,16#0104). + +-record(named_group_list, { + named_group_list %named_group_list<2..2^16-1>; + }). + +%% RFC 8446 B.3.2 Server Parameters Messages +%% opaque DistinguishedName<1..2^16-1>;XS +-record(certificate_authoritie_sextension, { + authorities %DistinguishedName authorities<3..2^16-1>; + }). + +-record(oid_filter, { + certificate_extension_oid, % opaque certificate_extension_oid<1..2^8-1>; + certificate_extension_values % opaque certificate_extension_values<0..2^16-1>; + }). + +-record(oid_filter_extension, { + filters %OIDFilter filters<0..2^16-1>; + }). +-record(post_handshake_auth, { + }). + +-record(encrypted_extensions, { + extensions %extensions<0..2^16-1>; + }). + +-record(certificate_request_1_3, { + certificate_request_context, % opaque certificate_request_context<0..2^8-1>; + extensions %Extension extensions<2..2^16-1>; + }). + +%% RFC 8446 B.3.3 Authentication Messages + +%% Certificate Type +-define(X509, 0). +-define(OpenPGP_RESERVED, 1). +-define(RawPublicKey, 2). + +-record(certificate_entry, { + data, + %% select (certificate_type) { + %% case RawPublicKey: + %% /* From RFC 7250 ASN.1_subjectPublicKeyInfo */ + %% opaque ASN1_subjectPublicKeyInfo<1..2^24-1>; + %% + %% case X509: + %% opaque cert_data<1..2^24-1>; + %% }; + extensions %% Extension extensions<0..2^16-1>; + }). + +-record(certificate_1_3, { + certificate_request_context, % opaque certificate_request_context<0..2^8-1>; + certificate_list % CertificateEntry certificate_list<0..2^24-1>; + }). + +-record(certificate_verify_1_3, { + algorithm, % SignatureScheme + signature % signature<0..2^16-1> + }). + +%% RFC 8446 B.3.4. Ticket Establishment +-record(new_session_ticket, { + ticket_lifetime, %unit32 + ticket_age_add, %unit32 + ticket_nonce, %opaque ticket_nonce<0..255>; + ticket, %opaque ticket<1..2^16-1> + extensions %extensions<0..2^16-2> + }). + +%% RFC 8446 B.3.5. Updating Keys +-record(end_of_early_data, { + }). + +-define(UPDATE_NOT_REQUESTED, 0). +-define(UPDATE_REQUESTED, 1). + +-record(key_update, { + request_update + }). + +-type tls_handshake_1_3() :: #encrypted_extensions{} | + #certificate_request_1_3{} | + #certificate_1_3{} | + #certificate_verify_1_3{}. + +-export_type([tls_handshake_1_3/0]). + +-endif. % -ifdef(tls_handshake_1_3). diff --git a/lib/ssl/src/tls_record.erl b/lib/ssl/src/tls_record.erl index 38022030ee..a5c550a429 100644 --- a/lib/ssl/src/tls_record.erl +++ b/lib/ssl/src/tls_record.erl @@ -30,9 +30,10 @@ -include("ssl_alert.hrl"). -include("tls_handshake.hrl"). -include("ssl_cipher.hrl"). +-include_lib("kernel/include/logger.hrl"). %% Handling of incoming data --export([get_tls_records/3, init_connection_states/2]). +-export([get_tls_records/4, init_connection_states/2]). %% Encoding TLS records -export([encode_handshake/3, encode_alert_record/3, @@ -40,13 +41,16 @@ -export([encode_plain_text/4]). %% Decoding --export([decode_cipher_text/3]). +-export([decode_cipher_text/4]). + +%% Logging helper +-export([build_tls_record/1]). %% Protocol version handling -export([protocol_version/1, lowest_protocol_version/1, lowest_protocol_version/2, highest_protocol_version/1, highest_protocol_version/2, is_higher/2, supported_protocol_versions/0, - is_acceptable_version/1, is_acceptable_version/2, hello_version/2]). + is_acceptable_version/1, is_acceptable_version/2, hello_version/1]). -export_type([tls_version/0, tls_atom_version/0]). @@ -76,8 +80,10 @@ init_connection_states(Role, BeastMitigation) -> %%-------------------------------------------------------------------- -spec get_tls_records( - binary(), [tls_version()] | tls_version(), - Buffer0 :: binary() | {'undefined' | #ssl_tls{}, {[binary()],non_neg_integer(),[binary()]}}) -> + binary(), + [tls_version()] | tls_version(), + Buffer0 :: binary() | {'undefined' | #ssl_tls{}, {[binary()],non_neg_integer(),[binary()]}}, + #ssl_options{}) -> {Records :: [#ssl_tls{}], Buffer :: {'undefined' | #ssl_tls{}, {[binary()],non_neg_integer(),[binary()]}}} | #alert{}. @@ -86,11 +92,10 @@ init_connection_states(Role, BeastMitigation) -> %% Description: Given old buffer and new data from TCP, packs up a records %% data %%-------------------------------------------------------------------- - -get_tls_records(Data, Versions, Buffer) when is_binary(Buffer) -> - parse_tls_records(Versions, {[Data],byte_size(Data),[]}, undefined); -get_tls_records(Data, Versions, {Hdr, {Front,Size,Rear}}) -> - parse_tls_records(Versions, {Front,Size + byte_size(Data),[Data|Rear]}, Hdr). +get_tls_records(Data, Versions, Buffer, SslOpts) when is_binary(Buffer) -> + parse_tls_records(Versions, {[Data],byte_size(Data),[]}, SslOpts, undefined); +get_tls_records(Data, Versions, {Hdr, {Front,Size,Rear}}, SslOpts) -> + parse_tls_records(Versions, {Front,Size + byte_size(Data),[Data|Rear]}, SslOpts, Hdr). %%==================================================================== %% Encoding @@ -102,6 +107,8 @@ get_tls_records(Data, Versions, {Hdr, {Front,Size,Rear}}) -> % %% Description: Encodes a handshake message to send on the ssl-socket. %%-------------------------------------------------------------------- +encode_handshake(Frag, {3, 4}, ConnectionStates) -> + tls_record_1_3:encode_handshake(Frag, ConnectionStates); encode_handshake(Frag, Version, #{current_write := #{beast_mitigation := BeastMitigation, @@ -122,6 +129,8 @@ encode_handshake(Frag, Version, %% %% Description: Encodes an alert message to send on the ssl-socket. %%-------------------------------------------------------------------- +encode_alert_record(Alert, {3, 4}, ConnectionStates) -> + tls_record_1_3:encode_alert_record(Alert, ConnectionStates); encode_alert_record(#alert{level = Level, description = Description}, Version, ConnectionStates) -> encode_plain_text(?ALERT, Version, <<?BYTE(Level), ?BYTE(Description)>>, @@ -142,6 +151,8 @@ encode_change_cipher_spec(Version, ConnectionStates) -> %% %% Description: Encodes data to send on the ssl-socket. %%-------------------------------------------------------------------- +encode_data(Data, {3, 4}, ConnectionStates) -> + tls_record_1_3:encode_data(Data, ConnectionStates); encode_data(Data, Version, #{current_write := #{beast_mitigation := BeastMitigation, security_parameters := @@ -155,12 +166,14 @@ encode_data(Data, Version, %%==================================================================== %%-------------------------------------------------------------------- --spec decode_cipher_text(#ssl_tls{}, ssl_record:connection_states(), boolean()) -> +-spec decode_cipher_text(tls_version(), #ssl_tls{}, ssl_record:connection_states(), boolean()) -> {#ssl_tls{}, ssl_record:connection_states()}| #alert{}. %% %% Description: Decode cipher text %%-------------------------------------------------------------------- -decode_cipher_text(CipherText, +decode_cipher_text({3,4}, CipherTextRecord, ConnectionStates, _) -> + tls_record_1_3:decode_cipher_text(CipherTextRecord, ConnectionStates); +decode_cipher_text(_, CipherTextRecord, #{current_read := #{sequence_number := Seq, security_parameters := @@ -170,7 +183,7 @@ decode_cipher_text(CipherText, } } = ConnectionStates0, _) -> SeqBin = <<?UINT64(Seq)>>, - #ssl_tls{type = Type, version = {MajVer,MinVer} = Version, fragment = Fragment} = CipherText, + #ssl_tls{type = Type, version = {MajVer,MinVer} = Version, fragment = Fragment} = CipherTextRecord, StartAdditionalData = <<SeqBin/binary, ?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer)>>, CipherS = ssl_record:nonce_seed(BulkCipherAlgo, SeqBin, CipherS0), case ssl_record:decipher_aead( @@ -187,17 +200,17 @@ decode_cipher_text(CipherText, cipher_state => CipherS, sequence_number => Seq + 1, compression_state => CompressionS}}, - {CipherText#ssl_tls{fragment = Plain}, ConnectionStates}; + {CipherTextRecord#ssl_tls{fragment = Plain}, ConnectionStates}; #alert{} = Alert -> Alert end; -decode_cipher_text(#ssl_tls{version = Version, - fragment = CipherFragment} = CipherText, +decode_cipher_text(_, #ssl_tls{version = Version, + fragment = CipherFragment} = CipherTextRecord, #{current_read := ReadState0} = ConnnectionStates0, PaddingCheck) -> case ssl_record:decipher(Version, CipherFragment, ReadState0, PaddingCheck) of {PlainFragment, Mac, ReadState1} -> - MacHash = ssl_cipher:calc_mac_hash(CipherText#ssl_tls.type, Version, PlainFragment, ReadState1), + MacHash = ssl_cipher:calc_mac_hash(CipherTextRecord#ssl_tls.type, Version, PlainFragment, ReadState1), case ssl_record:is_correct_mac(Mac, MacHash) of true -> #{sequence_number := Seq, @@ -210,7 +223,7 @@ decode_cipher_text(#ssl_tls{version = Version, ConnnectionStates0#{current_read => ReadState1#{sequence_number => Seq + 1, compression_state => CompressionS1}}, - {CipherText#ssl_tls{fragment = Plain}, ConnnectionStates}; + {CipherTextRecord#ssl_tls{fragment = Plain}, ConnnectionStates}; false -> ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC) end; @@ -229,6 +242,8 @@ decode_cipher_text(#ssl_tls{version = Version, %% Description: Creates a protocol version record from a version atom %% or vice versa. %%-------------------------------------------------------------------- +protocol_version('tlsv1.3') -> + {3, 4}; protocol_version('tlsv1.2') -> {3, 3}; protocol_version('tlsv1.1') -> @@ -239,6 +254,8 @@ protocol_version(sslv3) -> {3, 0}; protocol_version(sslv2) -> %% Backwards compatibility {2, 0}; +protocol_version({3, 4}) -> + 'tlsv1.3'; protocol_version({3, 3}) -> 'tlsv1.2'; protocol_version({3, 2}) -> @@ -372,10 +389,10 @@ is_acceptable_version({N,_} = Version, Versions) is_acceptable_version(_,_) -> false. --spec hello_version(tls_version(), [tls_version()]) -> tls_version(). -hello_version(Version, _) when Version >= {3, 3} -> - Version; -hello_version(_, Versions) -> +-spec hello_version([tls_version()]) -> tls_version(). +hello_version([Highest|_]) when Highest >= {3,3} -> + Highest; +hello_version(Versions) -> lowest_protocol_version(Versions). %%-------------------------------------------------------------------- @@ -394,84 +411,91 @@ initial_connection_state(ConnectionEnd, BeastMitigation) -> server_verify_data => undefined }. +%% Used by logging to recreate the received bytes +build_tls_record(#ssl_tls{type = Type, version = {MajVer, MinVer}, fragment = Fragment}) -> + Length = byte_size(Fragment), + <<?BYTE(Type),?BYTE(MajVer),?BYTE(MinVer),?UINT16(Length), Fragment/binary>>. -parse_tls_records(Versions, Q, undefined) -> - decode_tls_records(Versions, Q, [], undefined, undefined, undefined); -parse_tls_records(Versions, Q, #ssl_tls{type = Type, version = Version, fragment = Length}) -> - decode_tls_records(Versions, Q, [], Type, Version, Length). + +parse_tls_records(Versions, Q, SslOpts, undefined) -> + decode_tls_records(Versions, Q, SslOpts, [], undefined, undefined, undefined); +parse_tls_records(Versions, Q, SslOpts, #ssl_tls{type = Type, version = Version, fragment = Length}) -> + decode_tls_records(Versions, Q, SslOpts, [], Type, Version, Length). %% Generic code path -decode_tls_records(Versions, {_,Size,_} = Q0, Acc, undefined, _Version, _Length) -> +decode_tls_records(Versions, {_,Size,_} = Q0, SslOpts, Acc, undefined, _Version, _Length) -> if 5 =< Size -> {<<?BYTE(Type),?BYTE(MajVer),?BYTE(MinVer), ?UINT16(Length)>>, Q} = binary_from_front(5, Q0), - validate_tls_records_type(Versions, Q, Acc, Type, {MajVer,MinVer}, Length); + validate_tls_records_type(Versions, Q, SslOpts, Acc, Type, {MajVer,MinVer}, Length); 3 =< Size -> {<<?BYTE(Type),?BYTE(MajVer),?BYTE(MinVer)>>, Q} = binary_from_front(3, Q0), - validate_tls_records_type(Versions, Q, Acc, Type, {MajVer,MinVer}, undefined); + validate_tls_records_type(Versions, Q, SslOpts, Acc, Type, {MajVer,MinVer}, undefined); 1 =< Size -> {<<?BYTE(Type)>>, Q} = binary_from_front(1, Q0), - validate_tls_records_type(Versions, Q, Acc, Type, undefined, undefined); + validate_tls_records_type(Versions, Q, SslOpts, Acc, Type, undefined, undefined); true -> - validate_tls_records_type(Versions, Q0, Acc, undefined, undefined, undefined) + validate_tls_records_type(Versions, Q0, SslOpts, Acc, undefined, undefined, undefined) end; -decode_tls_records(Versions, {_,Size,_} = Q0, Acc, Type, undefined, _Length) -> +decode_tls_records(Versions, {_,Size,_} = Q0, SslOpts, Acc, Type, undefined, _Length) -> if 4 =< Size -> {<<?BYTE(MajVer),?BYTE(MinVer), ?UINT16(Length)>>, Q} = binary_from_front(4, Q0), - validate_tls_record_version(Versions, Q, Acc, Type, {MajVer,MinVer}, Length); + validate_tls_record_version(Versions, Q, SslOpts, Acc, Type, {MajVer,MinVer}, Length); 2 =< Size -> {<<?BYTE(MajVer),?BYTE(MinVer)>>, Q} = binary_from_front(2, Q0), - validate_tls_record_version(Versions, Q, Acc, Type, {MajVer,MinVer}, undefined); + validate_tls_record_version(Versions, Q, SslOpts, Acc, Type, {MajVer,MinVer}, undefined); true -> - validate_tls_record_version(Versions, Q0, Acc, Type, undefined, undefined) + validate_tls_record_version(Versions, Q0, SslOpts, Acc, Type, undefined, undefined) end; -decode_tls_records(Versions, {_,Size,_} = Q0, Acc, Type, Version, undefined) -> +decode_tls_records(Versions, {_,Size,_} = Q0, SslOpts, Acc, Type, Version, undefined) -> if 2 =< Size -> {<<?UINT16(Length)>>, Q} = binary_from_front(2, Q0), - validate_tls_record_length(Versions, Q, Acc, Type, Version, Length); + validate_tls_record_length(Versions, Q, SslOpts, Acc, Type, Version, Length); true -> - validate_tls_record_length(Versions, Q0, Acc, Type, Version, undefined) + validate_tls_record_length(Versions, Q0, SslOpts, Acc, Type, Version, undefined) end; -decode_tls_records(Versions, Q, Acc, Type, Version, Length) -> - validate_tls_record_length(Versions, Q, Acc, Type, Version, Length). +decode_tls_records(Versions, Q, SslOpts, Acc, Type, Version, Length) -> + validate_tls_record_length(Versions, Q, SslOpts, Acc, Type, Version, Length). -validate_tls_records_type(_Versions, Q, Acc, undefined, _Version, _Length) -> +validate_tls_records_type(_Versions, Q, _SslOpts, Acc, undefined, _Version, _Length) -> {lists:reverse(Acc), {undefined, Q}}; -validate_tls_records_type(Versions, Q, Acc, Type, Version, Length) -> +validate_tls_records_type(Versions, Q, SslOpts, Acc, Type, Version, Length) -> if ?KNOWN_RECORD_TYPE(Type) -> - validate_tls_record_version(Versions, Q, Acc, Type, Version, Length); + validate_tls_record_version(Versions, Q, SslOpts, Acc, Type, Version, Length); true -> %% Not ?KNOWN_RECORD_TYPE(Type) ?ALERT_REC(?FATAL, ?UNEXPECTED_MESSAGE) end. -validate_tls_record_version(_Versions, Q, Acc, Type, undefined, _Length) -> +validate_tls_record_version(_Versions, Q, _SslOpts, Acc, Type, undefined, _Length) -> {lists:reverse(Acc), {#ssl_tls{type = Type, version = undefined, fragment = undefined}, Q}}; -validate_tls_record_version(Versions, Q, Acc, Type, Version, Length) -> - if - is_list(Versions) -> +validate_tls_record_version(Versions, Q, SslOpts, Acc, Type, Version, Length) -> + case Versions of + _ when is_list(Versions) -> case is_acceptable_version(Version, Versions) of true -> - validate_tls_record_length(Versions, Q, Acc, Type, Version, Length); + validate_tls_record_length(Versions, Q, SslOpts, Acc, Type, Version, Length); false -> ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC) end; - Version =:= Versions -> + {3, 4} when Version =:= {3, 3} -> + validate_tls_record_length(Versions, Q, SslOpts, Acc, Type, Version, Length); + Version -> %% Exact version match - validate_tls_record_length(Versions, Q, Acc, Type, Version, Length); - true -> + validate_tls_record_length(Versions, Q, SslOpts, Acc, Type, Version, Length); + _ -> ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC) end. -validate_tls_record_length(_Versions, Q, Acc, Type, Version, undefined) -> +validate_tls_record_length(_Versions, Q, _SslOpts, Acc, Type, Version, undefined) -> {lists:reverse(Acc), {#ssl_tls{type = Type, version = Version, fragment = undefined}, Q}}; -validate_tls_record_length(Versions, {_,Size0,_} = Q0, Acc, Type, Version, Length) -> +validate_tls_record_length(Versions, {_,Size0,_} = Q0, SslOpts, Acc, Type, Version, Length) -> if Length =< ?MAX_CIPHER_TEXT_LENGTH -> if @@ -479,7 +503,8 @@ validate_tls_record_length(Versions, {_,Size0,_} = Q0, Acc, Type, Version, Lengt %% Complete record {Fragment, Q} = binary_from_front(Length, Q0), Record = #ssl_tls{type = Type, version = Version, fragment = Fragment}, - decode_tls_records(Versions, Q, [Record|Acc], undefined, undefined, undefined); + ssl_logger:debug(SslOpts#ssl_options.log_level, inbound, 'record', Record), + decode_tls_records(Versions, Q, SslOpts, [Record|Acc], undefined, undefined, undefined); true -> {lists:reverse(Acc), {#ssl_tls{type = Type, version = Version, fragment = Length}, Q0}} diff --git a/lib/ssl/src/tls_record_1_3.erl b/lib/ssl/src/tls_record_1_3.erl new file mode 100644 index 0000000000..74321a1ae2 --- /dev/null +++ b/lib/ssl/src/tls_record_1_3.erl @@ -0,0 +1,311 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2007-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% + +-module(tls_record_1_3). + +-include("tls_record.hrl"). +-include("tls_record_1_3.hrl"). +-include("ssl_internal.hrl"). +-include("ssl_alert.hrl"). +-include("ssl_cipher.hrl"). + +%% Encoding +-export([encode_handshake/2, encode_alert_record/2, + encode_data/2]). +-export([encode_plain_text/3]). + +%% Decoding +-export([decode_cipher_text/2]). + +%%==================================================================== +%% Encoding +%%==================================================================== + +%%-------------------------------------------------------------------- +-spec encode_handshake(iolist(), ssl_record:connection_states()) -> + {iolist(), ssl_record:connection_states()}. +% +%% Description: Encodes a handshake message to send on the tls-1.3-socket. +%%-------------------------------------------------------------------- +encode_handshake(Frag, ConnectionStates) -> + case iolist_size(Frag) of + N when N > ?MAX_PLAIN_TEXT_LENGTH -> + %% TODO: Consider padding here + Data = split_bin(iolist_to_binary(Frag), ?MAX_PLAIN_TEXT_LENGTH), + encode_iolist(?HANDSHAKE, Data, ConnectionStates); + _ -> + encode_plain_text(?HANDSHAKE, Frag, ConnectionStates) + end. + +%%-------------------------------------------------------------------- +-spec encode_alert_record(#alert{}, ssl_record:connection_states()) -> + {iolist(), ssl_record:connection_states()}. +%% +%% Description: Encodes an alert message to send on the ssl-socket. +%%-------------------------------------------------------------------- +encode_alert_record(#alert{level = Level, description = Description}, + ConnectionStates) -> + encode_plain_text(?ALERT, <<?BYTE(Level), ?BYTE(Description)>>, + ConnectionStates). +%%-------------------------------------------------------------------- +-spec encode_data(binary(), ssl_record:connection_states()) -> + {iolist(), ssl_record:connection_states()}. +%% +%% Description: Encodes data to send on the ssl-socket. +%%-------------------------------------------------------------------- +encode_data(Frag, ConnectionStates) -> + Data = split_bin(Frag, ?MAX_PLAIN_TEXT_LENGTH, {3,4}), + encode_iolist(?APPLICATION_DATA, Data, ConnectionStates). + +encode_plain_text(Type, Data0, #{current_write := Write0} = ConnectionStates) -> + PadLen = 0, %% TODO where to specify PadLen? + Data = inner_plaintext(Type, Data0, PadLen), + CipherFragment = encode_plain_text(Data, Write0), + {CipherText, Write} = encode_tls_cipher_text(CipherFragment, Write0), + {CipherText, ConnectionStates#{current_write => Write}}. + +encode_iolist(Type, Data, ConnectionStates0) -> + {ConnectionStates, EncodedMsg} = + lists:foldl(fun(Text, {CS0, Encoded}) -> + {Enc, CS1} = + encode_plain_text(Type, Text, CS0), + {CS1, [Enc | Encoded]} + end, {ConnectionStates0, []}, Data), + {lists:reverse(EncodedMsg), ConnectionStates}. + +%%==================================================================== +%% Decoding +%%==================================================================== + +%%-------------------------------------------------------------------- +-spec decode_cipher_text(#ssl_tls{}, ssl_record:connection_states()) -> + {#ssl_tls{}, ssl_record:connection_states()}| #alert{}. +%% +%% Description: Decode cipher text, use legacy type ssl_tls instead of tls_cipher_text +%% in decoding context so that we can reuse the code from erlier versions. +%%-------------------------------------------------------------------- +decode_cipher_text(#ssl_tls{type = ?OPAQUE_TYPE, + version = ?LEGACY_VERSION, + fragment = CipherFragment}, + #{current_read := + #{sequence_number := Seq, + cipher_state := #cipher_state{key = Key, + iv = IV, + tag_len = TagLen}, + security_parameters := + #security_parameters{ + cipher_type = ?AEAD, + bulk_cipher_algorithm = + BulkCipherAlgo} + } = ReadState0} = ConnectionStates0) -> + case decipher_aead(CipherFragment, BulkCipherAlgo, Key, Seq, IV, TagLen) of + #alert{} = Alert -> + Alert; + PlainFragment -> + ConnectionStates = + ConnectionStates0#{current_read => + ReadState0#{sequence_number => Seq + 1}}, + {decode_inner_plaintext(PlainFragment), ConnectionStates} + end; + + +%% RFC8446 - TLS 1.3 (OpenSSL compatibility) +%% Handle unencrypted Alerts from openssl s_client when server's +%% connection states are already stepped into traffic encryption. +%% (E.g. openssl s_client receives a CertificateRequest with +%% a signature_algorithms_cert extension that does not contain +%% the signature algorithm of the client's certificate.) +decode_cipher_text(#ssl_tls{type = ?ALERT, + version = ?LEGACY_VERSION, + fragment = <<2,47>>}, + ConnectionStates0) -> + {#ssl_tls{type = ?ALERT, + version = {3,4}, %% Internally use real version + fragment = <<2,47>>}, ConnectionStates0}; +%% RFC8446 - TLS 1.3 +%% D.4. Middlebox Compatibility Mode +%% - If not offering early data, the client sends a dummy +%% change_cipher_spec record (see the third paragraph of Section 5) +%% immediately before its second flight. This may either be before +%% its second ClientHello or before its encrypted handshake flight. +%% If offering early data, the record is placed immediately after the +%% first ClientHello. +decode_cipher_text(#ssl_tls{type = ?CHANGE_CIPHER_SPEC, + version = ?LEGACY_VERSION, + fragment = <<1>>}, + ConnectionStates0) -> + {#ssl_tls{type = ?CHANGE_CIPHER_SPEC, + version = {3,4}, %% Internally use real version + fragment = <<1>>}, ConnectionStates0}; +decode_cipher_text(#ssl_tls{type = Type, + version = ?LEGACY_VERSION, + fragment = CipherFragment}, + #{current_read := + #{security_parameters := + #security_parameters{ + cipher_suite = ?TLS_NULL_WITH_NULL_NULL} + }} = ConnnectionStates0) -> + {#ssl_tls{type = Type, + version = {3,4}, %% Internally use real version + fragment = CipherFragment}, ConnnectionStates0}; +decode_cipher_text(#ssl_tls{type = Type}, _) -> + %% Version mismatch is already asserted + ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, {record_type_mismatch, Type}). + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +split_bin(Bin, ChunkSize) -> + split_bin(Bin, ChunkSize, []). +split_bin(Bin, ChunkSize, _) -> + do_split_bin(Bin, ChunkSize, []). + +do_split_bin(<<>>, _, Acc) -> + lists:reverse(Acc); +do_split_bin(Bin, ChunkSize, Acc) -> + case Bin of + <<Chunk:ChunkSize/binary, Rest/binary>> -> + do_split_bin(Rest, ChunkSize, [Chunk | Acc]); + _ -> + lists:reverse(Acc, [Bin]) + end. + +inner_plaintext(Type, Data, Length) -> + #inner_plaintext{ + content = Data, + type = Type, + zeros = zero_padding(Length) + }. +zero_padding(Length)-> + binary:copy(<<?BYTE(0)>>, Length). + +encode_plain_text(#inner_plaintext{ + content = Data, + type = Type, + zeros = Zeros + }, #{cipher_state := #cipher_state{key= Key, + iv = IV, + tag_len = TagLen}, + sequence_number := Seq, + security_parameters := + #security_parameters{ + cipher_type = ?AEAD, + bulk_cipher_algorithm = BulkCipherAlgo} + }) -> + PlainText = [Data, Type, Zeros], + Encoded = cipher_aead(PlainText, BulkCipherAlgo, Key, Seq, IV, TagLen), + #tls_cipher_text{opaque_type = 23, %% 23 (application_data) for outward compatibility + legacy_version = {3,3}, + encoded_record = Encoded}; +encode_plain_text(#inner_plaintext{ + content = Data, + type = Type + }, #{security_parameters := + #security_parameters{ + cipher_suite = ?TLS_NULL_WITH_NULL_NULL} + }) -> + %% RFC8446 - 5.1. Record Layer + %% When record protection has not yet been engaged, TLSPlaintext + %% structures are written directly onto the wire. + #tls_cipher_text{opaque_type = Type, + legacy_version = {3,3}, + encoded_record = Data}; + +encode_plain_text(_, CS) -> + exit({cs, CS}). + +additional_data(Length) -> + <<?BYTE(?OPAQUE_TYPE), ?BYTE(3), ?BYTE(3),?UINT16(Length)>>. + +%% The per-record nonce for the AEAD construction is formed as +%% follows: +%% +%% 1. The 64-bit record sequence number is encoded in network byte +%% order and padded to the left with zeros to iv_length. +%% +%% 2. The padded sequence number is XORed with either the static +%% client_write_iv or server_write_iv (depending on the role). +%% +%% The resulting quantity (of length iv_length) is used as the +%% per-record nonce. +nonce(Seq, IV) -> + Padding = binary:copy(<<0>>, byte_size(IV) - 8), + crypto:exor(<<Padding/binary,?UINT64(Seq)>>, IV). + +cipher_aead(Fragment, BulkCipherAlgo, Key, Seq, IV, TagLen) -> + AAD = additional_data(erlang:iolist_size(Fragment) + TagLen), + Nonce = nonce(Seq, IV), + {Content, CipherTag} = + ssl_cipher:aead_encrypt(BulkCipherAlgo, Key, Nonce, Fragment, AAD, TagLen), + <<Content/binary, CipherTag/binary>>. + +encode_tls_cipher_text(#tls_cipher_text{opaque_type = Type, + legacy_version = {MajVer, MinVer}, + encoded_record = Encoded}, #{sequence_number := Seq} = Write) -> + Length = erlang:iolist_size(Encoded), + {[<<?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer), ?UINT16(Length)>>, Encoded], + Write#{sequence_number => Seq +1}}. + +decipher_aead(CipherFragment, BulkCipherAlgo, Key, Seq, IV, TagLen) -> + try + AAD = additional_data(erlang:iolist_size(CipherFragment)), + Nonce = nonce(Seq, IV), + {CipherText, CipherTag} = aead_ciphertext_split(CipherFragment, TagLen), + case ssl_cipher:aead_decrypt(BulkCipherAlgo, Key, Nonce, CipherText, CipherTag, AAD) of + Content when is_binary(Content) -> + Content; + _ -> + ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, decryption_failed) + end + catch + _:_ -> + ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, decryption_failed) + end. + + +aead_ciphertext_split(CipherTextFragment, TagLen) + when is_binary(CipherTextFragment) -> + CipherLen = erlang:byte_size(CipherTextFragment) - TagLen, + <<CipherText:CipherLen/bytes, CipherTag:TagLen/bytes>> = CipherTextFragment, + {CipherText, CipherTag}; +aead_ciphertext_split(CipherTextFragment, TagLen) + when is_list(CipherTextFragment) -> + CipherLen = erlang:iolist_size(CipherTextFragment) - TagLen, + <<CipherText:CipherLen/bytes, CipherTag:TagLen/bytes>> = + erlang:iolist_to_binary(CipherTextFragment), + {CipherText, CipherTag}. + +decode_inner_plaintext(PlainText) -> + case binary:last(PlainText) of + 0 -> + decode_inner_plaintext(init_binary(PlainText)); + Type when Type =:= ?APPLICATION_DATA orelse + Type =:= ?HANDSHAKE orelse + Type =:= ?ALERT -> + #ssl_tls{type = Type, + version = {3,4}, %% Internally use real version + fragment = init_binary(PlainText)}; + _Else -> + ?ALERT_REC(?FATAL, ?UNEXPECTED_MESSAGE, empty_alert) + end. + +init_binary(B) -> + {Init, _} = + split_binary(B, byte_size(B) - 1), + Init. diff --git a/lib/ssl/src/tls_record_1_3.hrl b/lib/ssl/src/tls_record_1_3.hrl new file mode 100644 index 0000000000..273427a34e --- /dev/null +++ b/lib/ssl/src/tls_record_1_3.hrl @@ -0,0 +1,58 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%%---------------------------------------------------------------------- +%% Purpose: Record and constant defenitions for the TLS-1.3-record protocol +%% see RFC 8446 not present in earlier versions +%%---------------------------------------------------------------------- + +-ifndef(tls_record_1_3). +-define(tls_record_1_3, true). + +%% enum { +%% invalid(0), +%% %% defined in ssl_record.hrl +%% change_cipher_spec(20), +%% alert(21), +%% handshake(22), +%% application_data(23), +%% heartbeat(24), /* RFC 6520 */ +%% (255) +%% } ContentType; + +-define(INVALID, 0). +-define(LEGACY_VERSION, {3,3}). +-define(OPAQUE_TYPE, 23). + +-record(inner_plaintext, { + content, %% data + type, %% Contentype + zeros %% padding "uint8 zeros[length_of_padding]" + }). +-record(tls_cipher_text, { %% Equivalent of encrypted version of #ssl_tls from previous versions + %% decrypted version will still use #ssl_tls for code reuse purposes + %% with real values for content type and version + opaque_type = ?OPAQUE_TYPE, + legacy_version = ?LEGACY_VERSION, + encoded_record + }). + +-endif. % -ifdef(tls_record_1_3). diff --git a/lib/ssl/src/tls_sender.erl b/lib/ssl/src/tls_sender.erl index c07b7f49cd..d0604565e3 100644 --- a/lib/ssl/src/tls_sender.erl +++ b/lib/ssl/src/tls_sender.erl @@ -48,7 +48,8 @@ negotiated_version, renegotiate_at, connection_monitor, - dist_handle + dist_handle, + log_level }). -record(data, @@ -202,7 +203,8 @@ init({call, From}, {Pid, #{current_write := WriteState, tracker := Tracker, transport_cb := Transport, negotiated_version := Version, - renegotiate_at := RenegotiateAt}}, + renegotiate_at := RenegotiateAt, + log_level := LogLevel}}, #data{connection_states = ConnectionStates, static = Static0} = StateData0) -> Monitor = erlang:monitor(process, Pid), StateData = @@ -215,7 +217,8 @@ init({call, From}, {Pid, #{current_write := WriteState, tracker = Tracker, transport_cb = Transport, negotiated_version = Version, - renegotiate_at = RenegotiateAt}}, + renegotiate_at = RenegotiateAt, + log_level = LogLevel}}, {next_state, handshake, StateData, [{reply, From, ok}]}; init(_, _, _) -> %% Just in case anything else sneeks through @@ -406,11 +409,13 @@ handle_common(Type, Msg, _) -> send_tls_alert(#alert{} = Alert, #data{static = #static{negotiated_version = Version, socket = Socket, - transport_cb = Transport}, + transport_cb = Transport, + log_level = LogLevel}, connection_states = ConnectionStates0} = StateData0) -> {BinMsg, ConnectionStates} = tls_record:encode_alert_record(Alert, Version, ConnectionStates0), tls_socket:send(Transport, Socket, BinMsg), + ssl_logger:debug(LogLevel, outbound, 'record', BinMsg), StateData0#data{connection_states = ConnectionStates}. send_application_data(Data, From, StateName, @@ -419,7 +424,8 @@ send_application_data(Data, From, StateName, dist_handle = DistHandle, negotiated_version = Version, transport_cb = Transport, - renegotiate_at = RenegotiateAt}, + renegotiate_at = RenegotiateAt, + log_level = LogLevel}, connection_states = ConnectionStates0} = StateData0) -> case time_to_renegotiate(Data, ConnectionStates0, RenegotiateAt) of true -> @@ -431,10 +437,12 @@ send_application_data(Data, From, StateName, StateData = StateData0#data{connection_states = ConnectionStates}, case tls_socket:send(Transport, Socket, Msgs) of ok when DistHandle =/= undefined -> + ssl_logger:debug(LogLevel, outbound, 'record', Msgs), {next_state, StateName, StateData, []}; Reason when DistHandle =/= undefined -> {next_state, death_row, StateData, [{state_timeout, 5000, Reason}]}; ok -> + ssl_logger:debug(LogLevel, outbound, 'record', Msgs), {next_state, StateName, StateData, [{reply, From, ok}]}; Result -> {next_state, StateName, StateData, [{reply, From, Result}]} diff --git a/lib/ssl/src/tls_v1.erl b/lib/ssl/src/tls_v1.erl index 1bfd9a8b6d..27cd5765e5 100644 --- a/lib/ssl/src/tls_v1.erl +++ b/lib/ssl/src/tls_v1.erl @@ -32,7 +32,19 @@ -export([master_secret/4, finished/5, certificate_verify/3, mac_hash/7, hmac_hash/3, setup_keys/8, suites/1, prf/5, ecc_curves/1, ecc_curves/2, oid_to_enum/1, enum_to_oid/1, - default_signature_algs/1, signature_algs/2]). + default_signature_algs/1, signature_algs/2, + default_signature_schemes/1, signature_schemes/2, + groups/1, groups/2, group_to_enum/1, enum_to_group/1, default_groups/1]). + +-export([derive_secret/4, hkdf_expand_label/5, hkdf_extract/3, hkdf_expand/4, + key_schedule/3, key_schedule/4, create_info/3, + external_binder_key/2, resumption_binder_key/2, + client_early_traffic_secret/3, early_exporter_master_secret/3, + client_handshake_traffic_secret/3, server_handshake_traffic_secret/3, + client_application_traffic_secret_0/3, server_application_traffic_secret_0/3, + exporter_master_secret/3, resumption_master_secret/3, + update_traffic_secret/2, calculate_traffic_keys/3, + transcript_hash/2, finished_key/2, finished_verify_data/3]). -type named_curve() :: sect571r1 | sect571k1 | secp521r1 | brainpoolP512r1 | sect409k1 | sect409r1 | brainpoolP384r1 | secp384r1 | @@ -41,12 +53,68 @@ sect193r1 | sect193r2 | secp192k1 | secp192r1 | sect163k1 | sect163r1 | sect163r2 | secp160k1 | secp160r1 | secp160r2. -type curves() :: [named_curve()]. --export_type([curves/0, named_curve/0]). +-type group() :: secp256r1 | secp384r1 | secp521r1 | ffdhe2048 | + ffdhe3072 | ffdhe4096 | ffdhe6144 | ffdhe8192. +-type supported_groups() :: [group()]. +-export_type([curves/0, named_curve/0, group/0, supported_groups/0]). %%==================================================================== %% Internal application API %%==================================================================== +%% TLS 1.3 --------------------------------------------------- +-spec derive_secret(Secret::binary(), Label::binary(), + Messages::iodata(), Algo::ssl:hash()) -> Key::binary(). +derive_secret(Secret, Label, Messages, Algo) -> + Hash = crypto:hash(mac_algo(Algo), Messages), + hkdf_expand_label(Secret, Label, + Hash, ssl_cipher:hash_size(Algo), Algo). + +-spec hkdf_expand_label(Secret::binary(), Label0::binary(), + Context::binary(), Length::integer(), + Algo::ssl:hash()) -> KeyingMaterial::binary(). +hkdf_expand_label(Secret, Label0, Context, Length, Algo) -> + HkdfLabel = create_info(Label0, Context, Length), + hkdf_expand(Secret, HkdfLabel, Length, Algo). + +%% Create info parameter for HKDF-Expand: +%% HKDF-Expand(PRK, info, L) -> OKM +create_info(Label0, Context0, Length) -> + %% struct { + %% uint16 length = Length; + %% opaque label<7..255> = "tls13 " + Label; + %% opaque context<0..255> = Context; + %% } HkdfLabel; + Label1 = << <<"tls13 ">>/binary, Label0/binary>>, + LabelLen = size(Label1), + Label = <<?BYTE(LabelLen), Label1/binary>>, + ContextLen = size(Context0), + Context = <<?BYTE(ContextLen),Context0/binary>>, + Content = <<Label/binary, Context/binary>>, + <<?UINT16(Length), Content/binary>>. + +-spec hkdf_extract(MacAlg::ssl:hash(), Salt::binary(), + KeyingMaterial::binary()) -> PseudoRandKey::binary(). + +hkdf_extract(MacAlg, Salt, KeyingMaterial) -> + hmac_hash(MacAlg, Salt, KeyingMaterial). + + +-spec hkdf_expand(PseudoRandKey::binary(), ContextInfo::binary(), + Length::integer(), Algo::ssl:hash()) -> KeyingMaterial::binary(). + +hkdf_expand(PseudoRandKey, ContextInfo, Length, Algo) -> + Iterations = erlang:ceil(Length / ssl_cipher:hash_size(Algo)), + hkdf_expand(Algo, PseudoRandKey, ContextInfo, Length, 1, Iterations, <<>>, <<>>). + + +-spec transcript_hash(Messages::iodata(), Algo::ssl:hash()) -> Hash::binary(). + +transcript_hash(Messages, Algo) -> + crypto:hash(mac_algo(Algo), Messages). +%% TLS 1.3 --------------------------------------------------- + +%% TLS 1.0 -1.2 --------------------------------------------------- -spec master_secret(integer(), binary(), binary(), binary()) -> binary(). master_secret(PrfAlgo, PreMasterSecret, ClientRandom, ServerRandom) -> @@ -56,9 +124,10 @@ master_secret(PrfAlgo, PreMasterSecret, ClientRandom, ServerRandom) -> prf(PrfAlgo, PreMasterSecret, <<"master secret">>, [ClientRandom, ServerRandom], 48). +%% TLS 1.0 -1.2 --------------------------------------------------- -spec finished(client | server, integer(), integer(), binary(), [binary()]) -> binary(). - +%% TLS 1.0 -1.1 --------------------------------------------------- finished(Role, Version, PrfAlgo, MasterSecret, Handshake) when Version == 1; Version == 2; PrfAlgo == ?MD5SHA -> %% RFC 2246 & 4346 - 7.4.9. Finished @@ -72,7 +141,9 @@ finished(Role, Version, PrfAlgo, MasterSecret, Handshake) MD5 = crypto:hash(md5, Handshake), SHA = crypto:hash(sha, Handshake), prf(?MD5SHA, MasterSecret, finished_label(Role), [MD5, SHA], 12); +%% TLS 1.0 -1.1 --------------------------------------------------- +%% TLS 1.2 --------------------------------------------------- finished(Role, Version, PrfAlgo, MasterSecret, Handshake) when Version == 3 -> %% RFC 5246 - 7.4.9. Finished @@ -84,21 +155,28 @@ finished(Role, Version, PrfAlgo, MasterSecret, Handshake) %% PRF(master_secret, finished_label, Hash(handshake_messages)) [0..11]; Hash = crypto:hash(mac_algo(PrfAlgo), Handshake), prf(PrfAlgo, MasterSecret, finished_label(Role), Hash, 12). +%% TLS 1.2 --------------------------------------------------- + +%% TODO 1.3 finished -spec certificate_verify(md5sha | sha, integer(), [binary()]) -> binary(). +%% TLS 1.0 -1.1 --------------------------------------------------- certificate_verify(md5sha, _Version, Handshake) -> MD5 = crypto:hash(md5, Handshake), SHA = crypto:hash(sha, Handshake), <<MD5/binary, SHA/binary>>; +%% TLS 1.0 -1.1 --------------------------------------------------- +%% TLS 1.2 --------------------------------------------------- certificate_verify(HashAlgo, _Version, Handshake) -> crypto:hash(HashAlgo, Handshake). +%% TLS 1.2 --------------------------------------------------- -spec setup_keys(integer(), integer(), binary(), binary(), binary(), integer(), integer(), integer()) -> {binary(), binary(), binary(), binary(), binary(), binary()}. - +%% TLS v1.0 --------------------------------------------------- setup_keys(Version, _PrfAlgo, MasterSecret, ServerRandom, ClientRandom, HashSize, KeyMatLen, IVSize) when Version == 1 -> @@ -123,8 +201,9 @@ setup_keys(Version, _PrfAlgo, MasterSecret, ServerRandom, ClientRandom, HashSize ClientIV:IVSize/binary, ServerIV:IVSize/binary>> = KeyBlock, {ClientWriteMacSecret, ServerWriteMacSecret, ClientWriteKey, ServerWriteKey, ClientIV, ServerIV}; +%% TLS v1.0 --------------------------------------------------- -%% TLS v1.1 +%% TLS v1.1 --------------------------------------------------- setup_keys(Version, _PrfAlgo, MasterSecret, ServerRandom, ClientRandom, HashSize, KeyMatLen, IVSize) when Version == 2 -> @@ -150,11 +229,12 @@ setup_keys(Version, _PrfAlgo, MasterSecret, ServerRandom, ClientRandom, HashSize ClientIV:IVSize/binary, ServerIV:IVSize/binary>> = KeyBlock, {ClientWriteMacSecret, ServerWriteMacSecret, ClientWriteKey, ServerWriteKey, ClientIV, ServerIV}; +%% TLS v1.1 --------------------------------------------------- -%% TLS v1.2 +%% TLS v1.2 --------------------------------------------------- setup_keys(Version, PrfAlgo, MasterSecret, ServerRandom, ClientRandom, HashSize, KeyMatLen, IVSize) - when Version == 3 -> + when Version == 3; Version == 4 -> %% RFC 5246 - 6.3. Key calculation %% key_block = PRF(SecurityParameters.master_secret, %% "key expansion", @@ -176,8 +256,177 @@ setup_keys(Version, PrfAlgo, MasterSecret, ServerRandom, ClientRandom, HashSize, ClientIV:IVSize/binary, ServerIV:IVSize/binary>> = KeyBlock, {ClientWriteMacSecret, ServerWriteMacSecret, ClientWriteKey, ServerWriteKey, ClientIV, ServerIV}. +%% TLS v1.2 --------------------------------------------------- --spec mac_hash(integer(), binary(), integer(), integer(), tls_record:tls_version(), +%% TLS v1.3 --------------------------------------------------- +%% RFC 8446 - 7.1. Key Schedule +%% +%% 0 +%% | +%% v +%% PSK -> HKDF-Extract = Early Secret +%% | +%% +-----> Derive-Secret(., "ext binder" | "res binder", "") +%% | = binder_key +%% | +%% +-----> Derive-Secret(., "c e traffic", ClientHello) +%% | = client_early_traffic_secret +%% | +%% +-----> Derive-Secret(., "e exp master", ClientHello) +%% | = early_exporter_master_secret +%% v +%% Derive-Secret(., "derived", "") +%% | +%% v +%% (EC)DHE -> HKDF-Extract = Handshake Secret +%% | +%% +-----> Derive-Secret(., "c hs traffic", +%% | ClientHello...ServerHello) +%% | = client_handshake_traffic_secret +%% | +%% +-----> Derive-Secret(., "s hs traffic", +%% | ClientHello...ServerHello) +%% | = server_handshake_traffic_secret +%% v +%% Derive-Secret(., "derived", "") +%% | +%% v +%% 0 -> HKDF-Extract = Master Secret +%% | +%% +-----> Derive-Secret(., "c ap traffic", +%% | ClientHello...server Finished) +%% | = client_application_traffic_secret_0 +%% | +%% +-----> Derive-Secret(., "s ap traffic", +%% | ClientHello...server Finished) +%% | = server_application_traffic_secret_0 +%% | +%% +-----> Derive-Secret(., "exp master", +%% | ClientHello...server Finished) +%% | = exporter_master_secret +%% | +%% +-----> Derive-Secret(., "res master", +%% ClientHello...client Finished) +%% = resumption_master_secret +-spec key_schedule(early_secret | handshake_secret | master_secret, + atom(), {psk | early_secret | handshake_secret, binary()}) -> + {early_secret | handshake_secret | master_secret, binary()}. + +key_schedule(early_secret, Algo, {psk, PSK}) -> + Len = ssl_cipher:hash_size(Algo), + Salt = binary:copy(<<?BYTE(0)>>, Len), + {early_secret, hkdf_extract(Algo, Salt, PSK)}; +key_schedule(master_secret, Algo, {handshake_secret, Secret}) -> + Len = ssl_cipher:hash_size(Algo), + IKM = binary:copy(<<?BYTE(0)>>, Len), + Salt = derive_secret(Secret, <<"derived">>, <<>>, Algo), + {master_secret, hkdf_extract(Algo, Salt, IKM)}. +%% +key_schedule(handshake_secret, Algo, IKM, {early_secret, Secret}) -> + Salt = derive_secret(Secret, <<"derived">>, <<>>, Algo), + {handshake_secret, hkdf_extract(Algo, Salt, IKM)}. + +-spec external_binder_key(atom(), {early_secret, binary()}) -> binary(). +external_binder_key(Algo, {early_secret, Secret}) -> + derive_secret(Secret, <<"ext binder">>, <<>>, Algo). + +-spec resumption_binder_key(atom(), {early_secret, binary()}) -> binary(). +resumption_binder_key(Algo, {early_secret, Secret}) -> + derive_secret(Secret, <<"res binder">>, <<>>, Algo). + +-spec client_early_traffic_secret(atom(), {early_secret, binary()}, iodata()) -> binary(). +%% M = ClientHello +client_early_traffic_secret(Algo, {early_secret, Secret}, M) -> + derive_secret(Secret, <<"c e traffic">>, M, Algo). + +-spec early_exporter_master_secret(atom(), {early_secret, binary()}, iodata()) -> binary(). +%% M = ClientHello +early_exporter_master_secret(Algo, {early_secret, Secret}, M) -> + derive_secret(Secret, <<"e exp master">>, M, Algo). + +-spec client_handshake_traffic_secret(atom(), {handshake_secret, binary()}, iodata()) -> binary(). +%% M = ClientHello...ServerHello +client_handshake_traffic_secret(Algo, {handshake_secret, Secret}, M) -> + derive_secret(Secret, <<"c hs traffic">>, M, Algo). + +-spec server_handshake_traffic_secret(atom(), {handshake_secret, binary()}, iodata()) -> binary(). +%% M = ClientHello...ServerHello +server_handshake_traffic_secret(Algo, {handshake_secret, Secret}, M) -> + derive_secret(Secret, <<"s hs traffic">>, M, Algo). + +-spec client_application_traffic_secret_0(atom(), {master_secret, binary()}, iodata()) -> binary(). +%% M = ClientHello...server Finished +client_application_traffic_secret_0(Algo, {master_secret, Secret}, M) -> + derive_secret(Secret, <<"c ap traffic">>, M, Algo). + +-spec server_application_traffic_secret_0(atom(), {master_secret, binary()}, iodata()) -> binary(). +%% M = ClientHello...server Finished +server_application_traffic_secret_0(Algo, {master_secret, Secret}, M) -> + derive_secret(Secret, <<"s ap traffic">>, M, Algo). + +-spec exporter_master_secret(atom(), {master_secret, binary()}, iodata()) -> binary(). +%% M = ClientHello...server Finished +exporter_master_secret(Algo, {master_secret, Secret}, M) -> + derive_secret(Secret, <<"exp master">>, M, Algo). + +-spec resumption_master_secret(atom(), {master_secret, binary()}, iodata()) -> binary(). +%% M = ClientHello...client Finished +resumption_master_secret(Algo, {master_secret, Secret}, M) -> + derive_secret(Secret, <<"res master">>, M, Algo). + +-spec finished_key(binary(), atom()) -> binary(). +finished_key(BaseKey, Algo) -> + %% finished_key = + %% HKDF-Expand-Label(BaseKey, "finished", "", Hash.length) + ssl_cipher:hash_size(Algo), + hkdf_expand_label(BaseKey, <<"finished">>, <<>>, ssl_cipher:hash_size(Algo), Algo). + +-spec finished_verify_data(binary(), atom(), iodata()) -> binary(). +finished_verify_data(FinishedKey, HKDFAlgo, Messages) -> + %% The verify_data value is computed as follows: + %% + %% verify_data = + %% HMAC(finished_key, + %% Transcript-Hash(Handshake Context, + %% Certificate*, CertificateVerify*)) + Context = lists:reverse(Messages), + THash = tls_v1:transcript_hash(Context, HKDFAlgo), + tls_v1:hmac_hash(HKDFAlgo, FinishedKey, THash). + +%% The next-generation application_traffic_secret is computed as: +%% +%% application_traffic_secret_N+1 = +%% HKDF-Expand-Label(application_traffic_secret_N, +%% "traffic upd", "", Hash.length) +-spec update_traffic_secret(atom(), binary()) -> binary(). +update_traffic_secret(Algo, Secret) -> + hkdf_expand_label(Secret, <<"traffic upd">>, <<>>, ssl_cipher:hash_size(Algo), Algo). + +%% The traffic keying material is generated from the following input +%% values: +%% +%% - A secret value +%% +%% - A purpose value indicating the specific value being generated +%% +%% - The length of the key being generated +%% +%% The traffic keying material is generated from an input traffic secret +%% value using: +%% +%% [sender]_write_key = HKDF-Expand-Label(Secret, "key", "", key_length) +%% [sender]_write_iv = HKDF-Expand-Label(Secret, "iv", "", iv_length) +-spec calculate_traffic_keys(atom(), atom(), binary()) -> {binary(), binary()}. +calculate_traffic_keys(HKDFAlgo, Cipher, Secret) -> + Key = hkdf_expand_label(Secret, <<"key">>, <<>>, ssl_cipher:key_material(Cipher), HKDFAlgo), + %% TODO: remove hard coded IV size + IV = hkdf_expand_label(Secret, <<"iv">>, <<>>, 12, HKDFAlgo), + {Key, IV}. + +%% TLS v1.3 --------------------------------------------------- + +%% TLS 1.0 -1.2 --------------------------------------------------- +-spec mac_hash(integer() | atom(), binary(), integer(), integer(), tls_record:tls_version(), integer(), binary()) -> binary(). mac_hash(Method, Mac_write_secret, Seq_num, Type, {Major, Minor}, @@ -191,8 +440,11 @@ mac_hash(Method, Mac_write_secret, Seq_num, Type, {Major, Minor}, ?BYTE(Major), ?BYTE(Minor), ?UINT16(Length)>>, Fragment]), Mac. +%% TLS 1.0 -1.2 --------------------------------------------------- + +%% TODO 1.3 same as above? --spec suites(1|2|3) -> [ssl_cipher_format:cipher_suite()]. +-spec suites(1|2|3|4|'TLS_v1.3') -> [ssl_cipher_format:cipher_suite()]. suites(Minor) when Minor == 1; Minor == 2 -> [ @@ -244,8 +496,29 @@ suites(3) -> %% ?TLS_DH_DSS_WITH_AES_256_GCM_SHA384, %% ?TLS_DH_RSA_WITH_AES_128_GCM_SHA256, %% ?TLS_DH_DSS_WITH_AES_128_GCM_SHA256 - ] ++ suites(2). - + ] ++ suites(2); + +suites(4) -> + [?TLS_AES_256_GCM_SHA384, + ?TLS_AES_128_GCM_SHA256, + ?TLS_CHACHA20_POLY1305_SHA256, + ?TLS_AES_128_CCM_SHA256 + %% Not supported + %% ?TLS_AES_128_CCM_8_SHA256 + ] ++ suites(3); + +suites('TLS_v1.3') -> + [?TLS_AES_256_GCM_SHA384, + ?TLS_AES_128_GCM_SHA256, + ?TLS_CHACHA20_POLY1305_SHA256, + ?TLS_AES_128_CCM_SHA256 + %% Not supported + %% ?TLS_AES_128_CCM_8_SHA256 + ]. + + +signature_algs({3, 4}, HashSigns) -> + signature_algs({3, 3}, HashSigns); signature_algs({3, 3}, HashSigns) -> CryptoSupports = crypto:supports(), Hashes = proplists:get_value(hashs, CryptoSupports), @@ -273,6 +546,10 @@ signature_algs({3, 3}, HashSigns) -> end, [], HashSigns), lists:reverse(Supported). +default_signature_algs({3, 4} = Version) -> + %% TLS 1.3 servers shall be prepared to process TLS 1.2 ClientHellos + %% containing legacy hash-sign tuples. + default_signature_schemes(Version) ++ default_signature_algs({3,3}); default_signature_algs({3, 3} = Version) -> Default = [%% SHA2 {sha512, ecdsa}, @@ -291,15 +568,99 @@ default_signature_algs({3, 3} = Version) -> default_signature_algs(_) -> undefined. + +signature_schemes(Version, SignatureSchemes) when is_tuple(Version) + andalso Version >= {3, 3} -> + CryptoSupports = crypto:supports(), + Hashes = proplists:get_value(hashs, CryptoSupports), + PubKeys = proplists:get_value(public_keys, CryptoSupports), + Curves = proplists:get_value(curves, CryptoSupports), + RSAPSSSupported = lists:member(rsa_pkcs1_pss_padding, + proplists:get_value(rsa_opts, CryptoSupports)), + Fun = fun (Scheme, Acc) when is_atom(Scheme) -> + {Hash0, Sign0, Curve} = + ssl_cipher:scheme_to_components(Scheme), + Sign = case Sign0 of + rsa_pkcs1 -> + rsa; + rsa_pss_rsae when RSAPSSSupported -> + rsa; + rsa_pss_pss when RSAPSSSupported -> + rsa; + S -> S + end, + Hash = case Hash0 of + sha1 -> + sha; + H -> H + end, + case proplists:get_bool(Sign, PubKeys) + andalso proplists:get_bool(Hash, Hashes) + andalso (Curve =:= undefined orelse + proplists:get_bool(Curve, Curves)) + andalso is_pair(Hash, Sign, Hashes) + of + true -> + [Scheme | Acc]; + false -> + Acc + end; + %% Special clause for filtering out the legacy hash-sign tuples. + (_ , Acc) -> + Acc + end, + Supported = lists:foldl(Fun, [], SignatureSchemes), + lists:reverse(Supported); +signature_schemes(_, _) -> + []. + +default_signature_schemes(Version) -> + Default = [ + ecdsa_secp521r1_sha512, + ecdsa_secp384r1_sha384, + ecdsa_secp256r1_sha256, + rsa_pss_pss_sha512, + rsa_pss_pss_sha384, + rsa_pss_pss_sha256, + rsa_pss_rsae_sha512, + rsa_pss_rsae_sha384, + rsa_pss_rsae_sha256, + %% ed25519, + %% ed448, + + %% These values refer solely to signatures + %% which appear in certificates (see Section 4.4.2.2) and are not + %% defined for use in signed TLS handshake messages, although they + %% MAY appear in "signature_algorithms" and + %% "signature_algorithms_cert" for backward compatibility with + %% TLS 1.2. + rsa_pkcs1_sha512, + rsa_pkcs1_sha384, + rsa_pkcs1_sha256, + ecdsa_sha1, + rsa_pkcs1_sha1 + ], + signature_schemes(Version, Default). + + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- +hkdf_expand(Algo, PseudoRandKey, ContextInfo, Length, N, N, Prev, Acc) -> + Keyingmaterial = hmac_hash(Algo, PseudoRandKey, <<Prev/binary, ContextInfo/binary, ?BYTE(N)>>), + binary:part(<<Acc/binary, Keyingmaterial/binary>>, {0, Length}); +hkdf_expand(Algo, PseudoRandKey, ContextInfo, Length, M, N, Prev, Acc) -> + Keyingmaterial = hmac_hash(Algo, PseudoRandKey, <<Prev/binary, ContextInfo/binary, ?BYTE(M)>>), + hkdf_expand(Algo, PseudoRandKey, ContextInfo, Length, M + 1, N, Keyingmaterial, <<Acc/binary, Keyingmaterial/binary>>). + %%%% HMAC and the Pseudorandom Functions RFC 2246 & 4346 - 5.%%%% hmac_hash(?NULL, _, _) -> <<>>; hmac_hash(Alg, Key, Value) -> crypto:hmac(mac_algo(Alg), Key, Value). +mac_algo(Alg) when is_atom(Alg) -> + Alg; mac_algo(?MD5) -> md5; mac_algo(?SHA) -> sha; mac_algo(?SHA256) -> sha256; @@ -395,6 +756,7 @@ ecc_curves(all) -> sect239k1,sect233k1,sect233r1,secp224k1,secp224r1, sect193r1,sect193r2,secp192k1,secp192r1,sect163k1, sect163r1,sect163r2,secp160k1,secp160r1,secp160r2]; + ecc_curves(Minor) -> TLSCurves = ecc_curves(all), ecc_curves(Minor, TLSCurves). @@ -409,6 +771,63 @@ ecc_curves(_Minor, TLSCurves) -> end end, [], TLSCurves). +-spec groups(4 | all | default) -> [group()]. +groups(all) -> + [x25519, + x448, + secp256r1, + secp384r1, + secp521r1, + ffdhe2048, + ffdhe3072, + ffdhe4096, + ffdhe6144, + ffdhe8192]; +groups(default) -> + [x25519, + x448, + secp256r1, + secp384r1]; +groups(Minor) -> + TLSGroups = groups(all), + groups(Minor, TLSGroups). +%% +-spec groups(4, [group()]) -> [group()]. +groups(_Minor, TLSGroups) -> + CryptoGroups = supported_groups(), + lists:filter(fun(Group) -> proplists:get_bool(Group, CryptoGroups) end, TLSGroups). + +default_groups(Minor) -> + TLSGroups = groups(default), + groups(Minor, TLSGroups). + +supported_groups() -> + %% TODO: Add new function to crypto? + proplists:get_value(curves, crypto:supports()) ++ + [ffdhe2048,ffdhe3072,ffdhe4096,ffdhe6144,ffdhe8192]. + +group_to_enum(secp256r1) -> 23; +group_to_enum(secp384r1) -> 24; +group_to_enum(secp521r1) -> 25; +group_to_enum(x25519) -> 29; +group_to_enum(x448) -> 30; +group_to_enum(ffdhe2048) -> 256; +group_to_enum(ffdhe3072) -> 257; +group_to_enum(ffdhe4096) -> 258; +group_to_enum(ffdhe6144) -> 259; +group_to_enum(ffdhe8192) -> 260. + +enum_to_group(23) -> secp256r1; +enum_to_group(24) -> secp384r1; +enum_to_group(25) -> secp521r1; +enum_to_group(29) -> x25519; +enum_to_group(30) -> x448; +enum_to_group(256) -> ffdhe2048; +enum_to_group(257) -> ffdhe3072; +enum_to_group(258) -> ffdhe4096; +enum_to_group(259) -> ffdhe6144; +enum_to_group(260) -> ffdhe8192; +enum_to_group(_) -> undefined. %% ECC curves from draft-ietf-tls-ecc-12.txt (Oct. 17, 2005) oid_to_enum(?sect163k1) -> 1; diff --git a/lib/ssl/test/Makefile b/lib/ssl/test/Makefile index a10f71a3de..dba90aaff0 100644 --- a/lib/ssl/test/Makefile +++ b/lib/ssl/test/Makefile @@ -43,6 +43,8 @@ MODULES = \ ssl_basic_SUITE \ ssl_bench_SUITE \ ssl_cipher_SUITE \ + ssl_cipher_suite_SUITE \ + openssl_server_cipher_suite_SUITE\ ssl_certificate_verify_SUITE\ ssl_crl_SUITE\ ssl_dist_SUITE \ @@ -61,8 +63,11 @@ MODULES = \ ssl_ECC\ ssl_upgrade_SUITE\ ssl_sni_SUITE \ - make_certs\ - x509_test + ssl_eqc_SUITE \ + ssl_rfc_5869_SUITE \ + make_certs \ + x509_test \ + inet_crypto_dist ERL_FILES = $(MODULES:%=%.erl) @@ -144,7 +149,7 @@ release_tests_spec: opt $(INSTALL_DATA) $(ERL_FILES) $(HRL_FILES) $(HRL_FILES_NEEDED_IN_TEST) $(COVER_FILE) "$(RELSYSDIR)" $(INSTALL_DATA) ssl.spec ssl_bench.spec ssl.cover "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" - @tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) + @tar cf - *_SUITE_data property_test | (cd "$(RELSYSDIR)"; tar xf -) release_docs_spec: diff --git a/lib/ssl/test/inet_crypto_dist.erl b/lib/ssl/test/inet_crypto_dist.erl new file mode 100644 index 0000000000..5aafaac983 --- /dev/null +++ b/lib/ssl/test/inet_crypto_dist.erl @@ -0,0 +1,1323 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% ------------------------------------------------------------------------- +%% +%% Module for encrypted Erlang protocol - a minimal encrypted +%% distribution protocol based on only a shared secret +%% and the crypto application +%% +-module(inet_crypto_dist). +-define(DIST_NAME, inet_crypto). +-define(DIST_PROTO, crypto). +-define(DRIVER, inet_tcp). +-define(FAMILY, inet). + +-define(PROTOCOL, inet_crypto_dist_v1). +-define(DEFAULT_BLOCK_CRYPTO, aes_128_gcm). +-define(DEFAULT_HASH_ALGORITHM, sha256). +-define(DEFAULT_REKEY_INTERVAL, 32768). + +-export([listen/1, accept/1, accept_connection/5, + setup/5, close/1, select/1, is_node_name/1]). +-export([is_supported/0]). + +%% Generalized dist API, for sibling IPv6 module inet6_crypto_dist +-export([gen_listen/2, gen_accept/2, gen_accept_connection/6, + gen_setup/6, gen_close/2, gen_select/2]). + +-export([nodelay/0]). + +%% Debug +%%%-compile(export_all). +-export([dbg/0, test_server/0, test_client/1]). + +-include_lib("kernel/include/net_address.hrl"). +-include_lib("kernel/include/dist.hrl"). +-include_lib("kernel/include/dist_util.hrl"). + +%% Test if crypto has got enough capabilities for this module to run +%% +is_supported() -> + try {crypto:cipher_info(?DEFAULT_BLOCK_CRYPTO), + crypto:hash_info(?DEFAULT_HASH_ALGORITHM)} + of + {#{block_size := _, iv_length := _, key_length := _}, + #{size := _}} -> + true + catch + error:undef -> + false + end. + +%% ------------------------------------------------------------------------- +%% Erlang distribution plugin structure explained to myself +%% ------- +%% These are the processes involved in the distribution: +%% * net_kernel +%% * The Acceptor +%% * The Controller | Handshaker | Ticker +%% * The DistCtrl process that may be split into: +%% + The Output controller +%% + The Input controller +%% For the regular inet_tcp_dist distribution module, DistCtrl +%% is not one or two processes, but one port - a gen_tcp socket +%% +%% When the VM is started with the argument "-proto_dist inet_crypto" +%% net_kernel registers the module inet_crypto_dist as distribution +%% module. net_kernel calls listen/1 to create a listen socket +%% and then accept/1 with the listen socket as argument to spawn +%% the Acceptor process, which is linked to net_kernel. Apparently +%% the listen socket is owned by net_kernel - I wonder if it could +%% be owned by the Acceptor process instead... +%% +%% The Acceptor process calls blocking accept on the listen socket +%% and when an incoming socket is returned it spawns the DistCtrl +%% process a linked to the Acceptor. The ownership of the accepted +%% socket is transferred to the DistCtrl process. +%% A message is sent to net_kernel to inform it that an incoming +%% connection has appeared and the Acceptor awaits a reply from net_kernel. +%% +%% net_kernel then calls accept_connection/5 to spawn the Controller | +%% Handshaker | Ticker process that is linked to net_kernel. +%% The Controller then awaits a message from the Acceptor process. +%% +%% When net_kernel has spawned the Controller it replies with a message +%% to the Acceptor that then calls DistCtrl to changes its links +%% so DistCtrl ends up linked to the Controller and not to the Acceptor. +%% The Acceptor then sends a message to the Controller. The Controller +%% then changes role into the Handshaker creates a #hs_data{} record +%% and calls dist_util:handshake_other_started/1. After this +%% the Acceptor goes back into a blocking accept on the listen socket. +%% +%% For the regular distribution inet_tcp_dist DistCtrl is a gen_tcp socket +%% and when it is a process it also acts as a socket. The #hs_data{} +%% record used by dist_util presents a set of funs that are used +%% by dist_util to perform the distribution handshake. These funs +%% make sure to transfer the handshake messages through the DistCtrl +%% "socket". +%% +%% When the handshake is finished a fun for this purpose in #hs_data{} +%% is called, which tells DistCtrl that it does not need to be prepared +%% for any more #hs_data{} handshake calls. The DistCtrl process in this +%% module then spawns the Input controller process that gets ownership +%% of the connection's gen_tcp socket and changes into {active, N} mode +%% so now it gets all incoming traffic and delivers that to the VM. +%% The original DistCtrl process changes role into the Output controller +%% process and starts asking the VM for outbound messages and transfers +%% them on the connection socket. +%% +%% The Handshaker now changes into the Ticker role, and uses only two +%% functions in the #hs_data{} record; one to get socket statistics +%% and one to send a tick. None of these may block for any reason +%% in particular not for a congested socket since that would destroy +%% connection supervision. +%% +%% +%% For an connection net_kernel calls setup/5 which spawns the +%% Controller process as linked to net_kernel. This Controller process +%% connects to the other node's listen socket and when that is succesful +%% spawns the DistCtrl process as linked to the controller and transfers +%% socket ownership to it. +%% +%% Then the Controller creates the #hs_data{} record and calls +%% dist_util:handshake_we_started/1 which changes the process role +%% into Handshaker. +%% +%% When the distribution handshake is finished the procedure is just +%% as for an incoming connection above. +%% +%% +%% To sum it up. +%% +%% There is an Acceptor process that is linked to net_kernel and +%% informs it when new connections arrive. +%% +%% net_kernel spawns Controllers for incoming and for outgoing connections. +%% these Controllers use the DistCtrl processes to do distribution +%% handshake and after that becomes Tickers that supervise the connection. +%% +%% The Controller | Handshaker | Ticker is linked to net_kernel, and to +%% DistCtrl, one or both. If any of these connection processes would die +%% all others should be killed by the links. Therefore none of them may +%% terminate with reason 'normal'. +%% ------------------------------------------------------------------------- + +%% ------------------------------------------------------------------------- +%% select/1 is called by net_kernel to ask if this distribution protocol +%% is willing to handle Node +%% + +select(Node) -> + gen_select(Node, ?DRIVER). + +gen_select(Node, Driver) -> + case dist_util:split_node(Node) of + {node, _, Host} -> + case Driver:getaddr(Host) of + {ok, _} -> true; + _ -> false + end; + _ -> + false + end. + +%% ------------------------------------------------------------------------- + +is_node_name(Node) -> + dist_util:is_node_name(Node). + +%% ------------------------------------------------------------------------- +%% Called by net_kernel to create a listen socket for this +%% distribution protocol. This listen socket is used by +%% the Acceptor process. +%% + +listen(Name) -> + gen_listen(Name, ?DRIVER). + +gen_listen(Name, Driver) -> + case inet_tcp_dist:gen_listen(Driver, Name) of + {ok, {Socket, Address, Creation}} -> + inet:setopts(Socket, [binary, {nodelay, true}]), + {ok, + {Socket, Address#net_address{protocol = ?DIST_PROTO}, Creation}}; + Other -> + Other + end. + +%% ------------------------------------------------------------------------- +%% Called by net_kernel to spawn the Acceptor process that awaits +%% new connection in a blocking accept and informs net_kernel +%% when a new connection has appeared, and starts the DistCtrl +%% "socket" process for the connection. +%% + +accept(Listen) -> + gen_accept(Listen, ?DRIVER). + +gen_accept(Listen, Driver) -> + NetKernel = self(), + %% + %% Spawn Acceptor process + %% + Config = config(), + monitor_dist_proc( + spawn_opt( + fun () -> + accept_loop(Listen, Driver, NetKernel, Config) + end, + [link, {priority, max}])). + +accept_loop(Listen, Driver, NetKernel, Config) -> + case Driver:accept(Listen) of + {ok, Socket} -> + wait_for_code_server(), + Timeout = net_kernel:connecttime(), + DistCtrl = start_dist_ctrl(Socket, Config, Timeout), + %% DistCtrl is a "socket" + NetKernel ! + {accept, + self(), DistCtrl, Driver:family(), ?DIST_PROTO}, + receive + {NetKernel, controller, Controller} -> + call_dist_ctrl(DistCtrl, {controller, Controller, self()}), + Controller ! {self(), controller, Socket}; + {NetKernel, unsupported_protocol} -> + exit(unsupported_protocol) + end, + accept_loop(Listen, Driver, NetKernel, Config); + AcceptError -> + exit({accept, AcceptError}) + end. + +wait_for_code_server() -> + %% This is an ugly hack. Starting encryption on a connection + %% requires the crypto module to be loaded. Loading the crypto + %% module triggers its on_load function, which calls + %% code:priv_dir/1 to find the directory where its NIF library is. + %% However, distribution is started earlier than the code server, + %% so the code server is not necessarily started yet, and + %% code:priv_dir/1 might fail because of that, if we receive + %% an incoming connection on the distribution port early enough. + %% + %% If the on_load function of a module fails, the module is + %% unloaded, and the function call that triggered loading it fails + %% with 'undef', which is rather confusing. + %% + %% So let's avoid that by waiting for the code server to start. + %% + case whereis(code_server) of + undefined -> + timer:sleep(10), + wait_for_code_server(); + Pid when is_pid(Pid) -> + ok + end. + +%% ------------------------------------------------------------------------- +%% Called by net_kernel when a new connection has appeared, to spawn +%% a Controller process that performs the handshake with the new node, +%% and then becomes the Ticker connection supervisor. +%% ------------------------------------------------------------------------- + +accept_connection(Acceptor, DistCtrl, MyNode, Allowed, SetupTime) -> + gen_accept_connection( + Acceptor, DistCtrl, MyNode, Allowed, SetupTime, ?DRIVER). + +gen_accept_connection( + Acceptor, DistCtrl, MyNode, Allowed, SetupTime, Driver) -> + NetKernel = self(), + %% + %% Spawn Controller/handshaker/ticker process + %% + monitor_dist_proc( + spawn_opt( + fun() -> + do_accept( + Acceptor, DistCtrl, + MyNode, Allowed, SetupTime, Driver, NetKernel) + end, + [link, {priority, max}])). + +do_accept( + Acceptor, DistCtrl, MyNode, Allowed, SetupTime, Driver, NetKernel) -> + receive + {Acceptor, controller, Socket} -> + Timer = dist_util:start_timer(SetupTime), + HSData = + hs_data_common( + NetKernel, MyNode, DistCtrl, Timer, + Socket, Driver:family()), + HSData_1 = + HSData#hs_data{ + this_node = MyNode, + this_flags = 0, + allowed = Allowed}, + dist_util:handshake_other_started(trace(HSData_1)) + end. + +%% ------------------------------------------------------------------------- +%% Called by net_kernel to spawn a Controller process that sets up +%% a new connection to another Erlang node, performs the handshake +%% with the other it, and then becomes the Ticker process +%% that supervises the connection. +%% ------------------------------------------------------------------------- + +setup(Node, Type, MyNode, LongOrShortNames, SetupTime) -> + gen_setup(Node, Type, MyNode, LongOrShortNames, SetupTime, ?DRIVER). + +gen_setup(Node, Type, MyNode, LongOrShortNames, SetupTime, Driver) -> + NetKernel = self(), + %% + %% Spawn Controller/handshaker/ticker process + %% + monitor_dist_proc( + spawn_opt( + setup_fun( + Node, Type, MyNode, LongOrShortNames, SetupTime, Driver, NetKernel), + [link, {priority, max}])). + +-spec setup_fun(_,_,_,_,_,_,_) -> fun(() -> no_return()). +setup_fun( + Node, Type, MyNode, LongOrShortNames, SetupTime, Driver, NetKernel) -> + fun() -> + do_setup( + Node, Type, MyNode, LongOrShortNames, SetupTime, + Driver, NetKernel) + end. + +-spec do_setup(_,_,_,_,_,_,_) -> no_return(). +do_setup( + Node, Type, MyNode, LongOrShortNames, SetupTime, Driver, NetKernel) -> + {Name, Address} = split_node(Driver, Node, LongOrShortNames), + ErlEpmd = net_kernel:epmd_module(), + {ARMod, ARFun} = get_address_resolver(ErlEpmd, Driver), + Timer = trace(dist_util:start_timer(SetupTime)), + case ARMod:ARFun(Name, Address, Driver:family()) of + {ok, Ip, TcpPort, Version} -> + do_setup_connect( + Node, Type, MyNode, Timer, Driver, NetKernel, + Ip, TcpPort, Version); + {ok, Ip} -> + case ErlEpmd:port_please(Name, Ip) of + {port, TcpPort, Version} -> + do_setup_connect( + Node, Type, MyNode, Timer, Driver, NetKernel, + Ip, TcpPort, Version); + Other -> + ?shutdown2( + Node, + trace( + {port_please_failed, ErlEpmd, Name, Ip, Other})) + end; + Other -> + ?shutdown2( + Node, + trace({getaddr_failed, Driver, Address, Other})) + end. + +-spec do_setup_connect(_,_,_,_,_,_,_,_,_) -> no_return(). + +do_setup_connect( + Node, Type, MyNode, Timer, Driver, NetKernel, + Ip, TcpPort, Version) -> + dist_util:reset_timer(Timer), + ConnectOpts = + trace( + connect_options( + [binary, {active, false}, {packet, 2}, {nodelay, true}])), + case Driver:connect(Ip, TcpPort, ConnectOpts) of + {ok, Socket} -> + Config = config(), + DistCtrl = + start_dist_ctrl(Socket, Config, net_kernel:connecttime()), + %% DistCtrl is a "socket" + HSData = + hs_data_common( + NetKernel, MyNode, DistCtrl, Timer, + Socket, Driver:family()), + HSData_1 = + HSData#hs_data{ + other_node = Node, + this_flags = 0, + other_version = Version, + request_type = Type}, + dist_util:handshake_we_started(trace(HSData_1)); + ConnectError -> + ?shutdown2(Node, + trace({connect_failed, Ip, TcpPort, ConnectError})) + end. + +%% ------------------------------------------------------------------------- +%% close/1 is only called by net_kernel on the socket returned by listen/1. + +close(Socket) -> + gen_close(Socket, ?DRIVER). + +gen_close(Socket, Driver) -> + trace(Driver:close(Socket)). + +%% ------------------------------------------------------------------------- + + +hs_data_common(NetKernel, MyNode, DistCtrl, Timer, Socket, Family) -> + %% Field 'socket' below is set to DistCtrl, which makes + %% the distribution handshake process (ticker) call + %% the funs below with DistCtrl as the S argument. + %% So, S =:= DistCtrl below... + #hs_data{ + kernel_pid = NetKernel, + this_node = MyNode, + socket = DistCtrl, + timer = Timer, + %% + f_send = + fun (S, Packet) when S =:= DistCtrl -> + call_dist_ctrl(S, {send, Packet}) + end, + f_recv = + fun (S, 0, infinity) when S =:= DistCtrl -> + case call_dist_ctrl(S, recv) of + {ok, Bin} when is_binary(Bin) -> + {ok, binary_to_list(Bin)}; + Error -> + Error + end + end, + f_setopts_pre_nodeup = + fun (S) when S =:= DistCtrl -> + ok + end, + f_setopts_post_nodeup = + fun (S) when S =:= DistCtrl -> + ok + end, + f_getll = + fun (S) when S =:= DistCtrl -> + {ok, S} %% DistCtrl is the distribution port + end, + f_address = + fun (S, Node) when S =:= DistCtrl -> + case call_dist_ctrl(S, peername) of + {ok, Address} -> + case dist_util:split_node(Node) of + {node, _, Host} -> + #net_address{ + address = Address, + host = Host, + protocol = ?DIST_PROTO, + family = Family}; + _ -> + {error, no_node} + end + end + end, + f_handshake_complete = + fun (S, _Node, DistHandle) when S =:= DistCtrl -> + call_dist_ctrl(S, {handshake_complete, DistHandle}) + end, + %% + %% mf_tick/1, mf_getstat/1, mf_setopts/2 and mf_getopts/2 + %% are called by the ticker any time after f_handshake_complete/3 + %% so they may not block the caller even for congested socket + mf_tick = + fun (S) when S =:= DistCtrl -> + S ! dist_tick + end, + mf_getstat = + fun (S) when S =:= DistCtrl -> + case + inet:getstat(Socket, [recv_cnt, send_cnt, send_pend]) + of + {ok, Stat} -> + split_stat(Stat, 0, 0, 0); + Error -> + Error + end + end, + mf_setopts = + fun (S, Opts) when S =:= DistCtrl -> + inet:setopts(Socket, setopts_filter(Opts)) + end, + mf_getopts = + fun (S, Opts) when S =:= DistCtrl -> + inet:getopts(Socket, Opts) + end}. + +setopts_filter(Opts) -> + [Opt || + Opt <- Opts, + case Opt of + {K, _} when K =:= active; K =:= deliver; K =:= packet -> false; + K when K =:= list; K =:= binary -> false; + K when K =:= inet; K =:= inet6 -> false; + _ -> true + end]. + +split_stat([{recv_cnt, R}|Stat], _, W, P) -> + split_stat(Stat, R, W, P); +split_stat([{send_cnt, W}|Stat], R, _, P) -> + split_stat(Stat, R, W, P); +split_stat([{send_pend, P}|Stat], R, W, _) -> + split_stat(Stat, R, W, P); +split_stat([], R, W, P) -> + {ok, R, W, P}. + +%% ------------------------------------------------------------ +%% Determine if EPMD module supports address resolving. Default +%% is to use inet_tcp:getaddr/2. +%% ------------------------------------------------------------ +get_address_resolver(EpmdModule, _Driver) -> + case erlang:function_exported(EpmdModule, address_please, 3) of + true -> {EpmdModule, address_please}; + _ -> {erl_epmd, address_please} + end. + + +%% If Node is illegal terminate the connection setup!! +split_node(Driver, Node, LongOrShortNames) -> + case dist_util:split_node(Node) of + {node, Name, Host} -> + check_node(Driver, Node, Name, Host, LongOrShortNames); + {host, _} -> + error_logger:error_msg( + "** Nodename ~p illegal, no '@' character **~n", + [Node]), + ?shutdown2(Node, trace({illegal_node_n@me, Node})); + _ -> + error_logger:error_msg( + "** Nodename ~p illegal **~n", [Node]), + ?shutdown2(Node, trace({illegal_node_name, Node})) + end. + +check_node(Driver, Node, Name, Host, LongOrShortNames) -> + case string:split(Host, ".", all) of + [_] when LongOrShortNames =:= longnames -> + case Driver:parse_address(Host) of + {ok, _} -> + {Name, Host}; + _ -> + error_logger:error_msg( + "** System running to use " + "fully qualified hostnames **~n" + "** Hostname ~s is illegal **~n", + [Host]), + ?shutdown2(Node, trace({not_longnames, Host})) + end; + [_, _|_] when LongOrShortNames =:= shortnames -> + error_logger:error_msg( + "** System NOT running to use " + "fully qualified hostnames **~n" + "** Hostname ~s is illegal **~n", + [Host]), + ?shutdown2(Node, trace({not_shortnames, Host})); + _ -> + {Name, Host} + end. + +%% ------------------------------------------------------------------------- + +connect_options(Opts) -> + case application:get_env(kernel, inet_dist_connect_options) of + {ok, ConnectOpts} -> + Opts ++ setopts_filter(ConnectOpts); + _ -> + Opts + end. + +%% we may not always want the nodelay behaviour +%% for performance reasons +nodelay() -> + case application:get_env(kernel, dist_nodelay) of + undefined -> + {nodelay, true}; + {ok, true} -> + {nodelay, true}; + {ok, false} -> + {nodelay, false}; + _ -> + {nodelay, true} + end. + +config() -> + case init:get_argument(?DIST_NAME) of + error -> + error({missing_argument, ?DIST_NAME}); + {ok, [[String]]} -> + {ok, Tokens, _} = erl_scan:string(String ++ "."), + case erl_parse:parse_term(Tokens) of + {ok, #{secret := Secret} = Config} + when is_binary(Secret); is_list(Secret) -> + Config; + {ok, #{} = Config} -> + error({missing_secret, [{?DIST_NAME,Config}]}); + _ -> + error({bad_argument_value, [{?DIST_NAME,String}]}) + end; + {ok, Value} -> + error({malformed_argument, [{?DIST_NAME,Value}]}) + end. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% +%% The DistCtrl process(es). +%% +%% At net_kernel handshake_complete spawns off the input controller that +%% takes over the socket ownership, and itself becomes the output controller +%% +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%%% XXX Missing to "productified": +%%% * Cryptoanalysis by experts +%%% * Proof of usefulness +%%% * Unifying exit reasons using a death_row() function +%%% * Verification (and rejection) of other end's crypto parameters +%%% * OTP:ification (proc_lib?) +%%% * An application to belong to (crypto|kernel?) +%%% * Secret on file (cookie as default?), parameter handling +%%% * Restart and/or code reload policy + +%% Debug client and server + +test_config() -> + #{secret => <<"Secret Cluster Password 123456">>}. + +test_server() -> + {ok, Listen} = gen_tcp:listen(0, [{packet, 2}, {active, false}, binary]), + {ok, Port} = inet:port(Listen), + io:format(?MODULE_STRING":test_client(~w).~n", [Port]), + {ok, Socket} = gen_tcp:accept(Listen), + test(Socket). + +test_client(Port) -> + {ok, Socket} = + gen_tcp:connect( + localhost, Port, [{packet, 2}, {active, false}, binary]), + test(Socket). + +test(Socket) -> + start_dist_ctrl(Socket, test_config(), 10000). + +%% ------------------------------------------------------------------------- + +start_dist_ctrl(Socket, Config, Timeout) -> + Protocol = ?PROTOCOL, + Controller = self(), + Server = + monitor_dist_proc( + spawn_opt( + fun () -> + receive + {?MODULE, From, start} -> + {SendParams, RecvParams} = + init(Socket, Config, Protocol), + reply(From, self()), + handshake(SendParams, 0, RecvParams, 0, Controller) + end + end, + [link, + {priority, max}, + {message_queue_data, off_heap}, + {fullsweep_after, 0}])), + ok = gen_tcp:controlling_process(Socket, Server), + call_dist_ctrl(Server, start, Timeout). + + +call_dist_ctrl(Server, Msg) -> + call_dist_ctrl(Server, Msg, infinity). +%% +call_dist_ctrl(Server, Msg, Timeout) -> + Ref = erlang:monitor(process, Server), + Server ! {?MODULE, {Ref, self()}, Msg}, + receive + {Ref, Res} -> + erlang:demonitor(Ref, [flush]), + Res; + {'DOWN', Ref, process, Server, Reason} -> + exit({?PROTOCOL, Reason}) + after Timeout -> + exit(Server, timeout), + receive + {'DOWN', Ref, process, Server, _} -> + exit({?PROTOCOL, timeout}) + end + end. + +reply({Ref, Pid}, Msg) -> + Pid ! {Ref, Msg}, + ok. + +%% ------------------------------------------------------------------------- + +-record(params, + {protocol, % Encryption protocol tag + socket, + dist_handle, + hash_algorithm, + block_crypto, + rekey_interval, + iv, + key, + tag_len}). + +-define(TCP_ACTIVE, 64). +-define(CHUNK_SIZE, (65536 - 512)). +%% The start chunk starts with zeros, so it seems logical to not have +%% a chunk type with value 0 +-define(HANDSHAKE_CHUNK, 1). +-define(DATA_CHUNK, 2). +-define(TICK_CHUNK, 3). +-define(REKEY_CHUNK, 4). + +%% ------------------------------------------------------------------------- +%% Crypto strategy +%% ------- +%% The crypto strategy is as simple as possible to get an encrypted +%% connection as benchmark reference. It is geared around AEAD +%% ciphers in particular AES-GCM. +%% +%% The init message and the start message must fit in the TCP buffers +%% since both sides start with sending the init message, waits +%% for the other end's init message, sends the start message +%% and waits for the other end's start message. So if the send +%% blocks we have a deadlock. +%% +%% The init message is unencrypted and contains the block cipher and hash +%% algorithms the sender will use, the IV and a key salt. Both sides' +%% key salt is used with the mutual secret as input to the hash algorithm +%% to create different encryption/decryption keys for both directions. +%% +%% The start message is the first encrypted message and contains just +%% encrypted zeros the width of the key, with the header of the init +%% message as AAD data. Successfully decrypting this message +%% verifies that we have an encrypted channel. +%% +%% Subsequent encrypted messages has the sequence number and the length +%% of the message as AAD data. These messages has got a message type +%% differentiating data from ticks. Ticks have a random size in an +%% attempt to make them less obvious to spot. +%% +%% The only reaction to errors is to crash noisily wich will bring +%% down the connection and hopefully produce something useful +%% in the local log, but all the other end sees is a closed connection. +%% ------------------------------------------------------------------------- + +init(Socket, Config, Protocol) -> + Secret = maps:get(secret, Config), + HashAlgorithm = + maps:get(hash_algorithm, Config, ?DEFAULT_HASH_ALGORITHM), + BlockCrypto = + maps:get(block_crypto, Config, ?DEFAULT_BLOCK_CRYPTO), + RekeyInterval = + maps:get(rekey_interval, Config, ?DEFAULT_REKEY_INTERVAL), + %% + SendParams = + init_params( + Socket, Protocol, HashAlgorithm, BlockCrypto, RekeyInterval), + send_init(SendParams, Secret). + +send_init( + #params{ + protocol = Protocol, + socket = Socket, + block_crypto = BlockCrypto, + iv = IVLen, + key = KeyLen, + hash_algorithm = HashAlgorithm} = SendParams, + Secret) -> + %% + ProtocolString = atom_to_binary(Protocol, utf8), + BlockCryptoString = atom_to_binary(BlockCrypto, utf8), + HashAlgorithmString = atom_to_binary(HashAlgorithm, utf8), + SendHeader = + <<ProtocolString/binary, 0, + HashAlgorithmString/binary, 0, + BlockCryptoString/binary, 0>>, + <<IV:IVLen/binary, KeySalt:KeyLen/binary>> = IV_KeySalt = + crypto:strong_rand_bytes(IVLen + KeyLen), + InitPacket = [SendHeader, IV_KeySalt], + ok = gen_tcp:send(Socket, InitPacket), + recv_init(SendParams#params{iv = IV, key = KeySalt}, Secret, SendHeader). + +recv_init( + #params{ + socket = Socket, + hash_algorithm = SendHashAlgorithm, + key = SendKeySalt} = SendParams, Secret, SendHeader) -> + %% + {ok, InitPacket} = gen_tcp:recv(Socket, 0), + [ProtocolString, Rest_1] = binary:split(InitPacket, <<0>>), + Protocol = binary_to_existing_atom(ProtocolString, utf8), + case Protocol of + ?PROTOCOL -> + [HashAlgorithmString, Rest_2] = binary:split(Rest_1, <<0>>), + HashAlgorithm = binary_to_existing_atom(HashAlgorithmString, utf8), + [BlockCryptoString, Rest_3] = binary:split(Rest_2, <<0>>), + BlockCrypto = binary_to_existing_atom(BlockCryptoString, utf8), + #params{ + hash_algorithm = RecvHashAlgorithm, + iv = RecvIVLen, + key = RecvKeyLen} = RecvParams = + init_params( + Socket, Protocol, HashAlgorithm, BlockCrypto, undefined), + <<RecvIV:RecvIVLen/binary, + RecvKeySalt:RecvKeyLen/binary>> = Rest_3, + SendKey = + hash_key(SendHashAlgorithm, SendKeySalt, [RecvKeySalt, Secret]), + RecvKey = + hash_key(RecvHashAlgorithm, RecvKeySalt, [SendKeySalt, Secret]), + SendParams_1 = SendParams#params{key = SendKey}, + RecvParams_1 = RecvParams#params{iv = RecvIV, key = RecvKey}, + RecvHeaderLen = byte_size(InitPacket) - RecvIVLen - RecvKeyLen, + <<RecvHeader:RecvHeaderLen/binary, _/binary>> = InitPacket, + send_start(SendParams_1, SendHeader), + RecvRekeyInterval = recv_start(RecvParams_1, RecvHeader), + {SendParams_1, + RecvParams_1#params{rekey_interval = RecvRekeyInterval}} + end. + +send_start( + #params{ + socket = Socket, + block_crypto = BlockCrypto, + rekey_interval= RekeyInterval, + iv = IV, + key = Key, + tag_len = TagLen}, AAD) -> + %% + KeyLen = byte_size(Key), + Zeros = binary:copy(<<0>>, KeyLen), + {Ciphertext, CipherTag} = + crypto:block_encrypt( + crypto_cipher_name(BlockCrypto), + Key, IV, {AAD, [Zeros, <<RekeyInterval:32>>], TagLen}), + ok = gen_tcp:send(Socket, [Ciphertext, CipherTag]). + +recv_start( + #params{ + socket = Socket, + block_crypto = BlockCrypto, + iv = IV, + key = Key, + tag_len = TagLen}, AAD) -> + {ok, Packet} = gen_tcp:recv(Socket, 0), + KeyLen = byte_size(Key), + PacketLen = KeyLen + 4, + <<Ciphertext:PacketLen/binary, CipherTag:TagLen/binary>> = Packet, + Zeros = binary:copy(<<0>>, KeyLen), + case + crypto:block_decrypt( + crypto_cipher_name(BlockCrypto), + Key, IV, {AAD, Ciphertext, CipherTag}) + of + <<Zeros:KeyLen/binary, RekeyInterval:32>> + when 1 =< RekeyInterval -> + RekeyInterval; + _ -> + error(decrypt_error) + end. + +init_params(Socket, Protocol, HashAlgorithm, BlockCrypto, RekeyInterval) -> + #{block_size := 1, + iv_length := IVLen, + key_length := KeyLen} = crypto:cipher_info(BlockCrypto), + case crypto:hash_info(HashAlgorithm) of + #{size := HashSize} when HashSize >= KeyLen -> + #params{ + socket = Socket, + protocol = Protocol, + hash_algorithm = HashAlgorithm, + block_crypto = BlockCrypto, + rekey_interval = RekeyInterval, + iv = IVLen, + key = KeyLen, + tag_len = 16} + end. + +crypto_cipher_name(BlockCrypto) -> + case BlockCrypto of + aes_128_gcm -> aes_gcm; + aes_192_gcm -> aes_gcm; + aes_256_gcm -> aes_gcm + end. + +hash_key(HashAlgorithm, KeySalt, OtherSalt) -> + KeyLen = byte_size(KeySalt), + <<Key:KeyLen/binary, _/binary>> = + crypto:hash(HashAlgorithm, [KeySalt, OtherSalt]), + Key. + +%% ------------------------------------------------------------------------- +%% net_kernel distribution handshake in progress +%% + +handshake( + SendParams, SendSeq, + #params{socket = Socket} = RecvParams, RecvSeq, Controller) -> + receive + {?MODULE, From, {controller, Controller_1, Parent}} -> + Result = link(Controller_1), + true = unlink(Parent), + reply(From, Result), + handshake(SendParams, SendSeq, RecvParams, RecvSeq, Controller_1); + {?MODULE, From, {handshake_complete, DistHandle}} -> + reply(From, ok), + InputHandler = + monitor_dist_proc( + spawn_opt( + fun () -> + link(Controller), + receive + DistHandle -> + ok = + inet:setopts( + Socket, + [{active, ?TCP_ACTIVE}, + nodelay()]), + input_handler( + RecvParams#params{ + dist_handle = DistHandle}, + RecvSeq, empty_q(), infinity) + end + end, + [link, + {priority, normal}, + {message_queue_data, off_heap}, + {fullsweep_after, 0}])), + _ = monitor(process, InputHandler), % For the benchmark test + ok = gen_tcp:controlling_process(Socket, InputHandler), + ok = erlang:dist_ctrl_input_handler(DistHandle, InputHandler), + InputHandler ! DistHandle, + process_flag(priority, normal), + erlang:dist_ctrl_get_data_notification(DistHandle), + crypto:rand_seed_alg(crypto_cache), + output_handler( + SendParams#params{dist_handle = DistHandle}, SendSeq); + %% + {?MODULE, From, {send, Data}} -> + {SendParams_1, SendSeq_1} = + encrypt_and_send_chunk( + SendParams, SendSeq, [?HANDSHAKE_CHUNK, Data]), + reply(From, ok), + handshake( + SendParams_1, SendSeq_1, RecvParams, RecvSeq, Controller); + {?MODULE, From, recv} -> + {RecvParams_1, RecvSeq_1, Reply} = + recv_and_decrypt_chunk(RecvParams, RecvSeq), + reply(From, Reply), + handshake( + SendParams, SendSeq, RecvParams_1, RecvSeq_1, Controller); + {?MODULE, From, peername} -> + reply(From, inet:peername(Socket)), + handshake(SendParams, SendSeq, RecvParams, RecvSeq, Controller); + %% + _Alien -> + handshake(SendParams, SendSeq, RecvParams, RecvSeq, Controller) + end. + +recv_and_decrypt_chunk(#params{socket = Socket} = RecvParams, RecvSeq) -> + case gen_tcp:recv(Socket, 0) of + {ok, Chunk} -> + case decrypt_chunk(RecvParams, RecvSeq, Chunk) of + <<?HANDSHAKE_CHUNK, Cleartext/binary>> -> + {RecvParams, RecvSeq + 1, {ok, Cleartext}}; + #params{} = RecvParams_1 -> + recv_and_decrypt_chunk(RecvParams_1, 0); + _ -> + error(decrypt_error) + end; + Error -> + {RecvParams, RecvSeq, Error} + end. + +%% ------------------------------------------------------------------------- +%% Output handler process +%% +%% The game here is to flush all dist_data and dist_tick messages, +%% prioritize dist_data over dist_tick, and to not use selective receive + +output_handler(Params, Seq) -> + receive + Msg -> + case Msg of + dist_data -> + output_handler_data(Params, Seq); + dist_tick -> + output_handler_tick(Params, Seq); + _Other -> + %% Ignore + output_handler(Params, Seq) + end + end. + +output_handler_data(Params, Seq) -> + receive + Msg -> + case Msg of + dist_data -> + output_handler_data(Params, Seq); + dist_tick -> + output_handler_data(Params, Seq); + _Other -> + %% Ignore + output_handler_data(Params, Seq) + end + after 0 -> + DistHandle = Params#params.dist_handle, + Q = get_data(DistHandle, empty_q()), + {Params_1, Seq_1} = output_handler_send(Params, Seq, Q, true), + erlang:dist_ctrl_get_data_notification(DistHandle), + output_handler(Params_1, Seq_1) + end. + +output_handler_tick(Params, Seq) -> + receive + Msg -> + case Msg of + dist_data -> + output_handler_data(Params, Seq); + dist_tick -> + output_handler_tick(Params, Seq); + _Other -> + %% Ignore + output_handler_tick(Params, Seq) + end + after 0 -> + TickSize = 8 + rand:uniform(56), + TickData = binary:copy(<<0>>, TickSize), + {Params_1, Seq_1} = + encrypt_and_send_chunk(Params, Seq, [?TICK_CHUNK, TickData]), + output_handler(Params_1, Seq_1) + end. + +output_handler_send( + #params{dist_handle = DistHandle} = Params, Seq, {_, Size, _} = Q, Retry) -> + %% + if + ?CHUNK_SIZE < Size -> + {Cleartext, Q_1} = deq_iovec(?CHUNK_SIZE, Q), + {Params_1, Seq_1} = + encrypt_and_send_chunk(Params, Seq, [?DATA_CHUNK, Cleartext]), + output_handler_send(Params_1, Seq_1, Q_1, Retry); + Retry -> + Q_1 = get_data(DistHandle, Q), + output_handler_send(Params, Seq, Q_1, false); + true -> + {Cleartext, _} = deq_iovec(Size, Q), + encrypt_and_send_chunk(Params, Seq, [?DATA_CHUNK, Cleartext]) + end. + +%% ------------------------------------------------------------------------- +%% Input handler process +%% +%% Here is T 0 or infinity to steer if we should try to receive +%% more data or not; start with infinity, and when we get some +%% data try with 0 to see if more is waiting + +input_handler(#params{socket = Socket} = Params, Seq, Q, T) -> + receive + Msg -> + case Msg of + {tcp_passive, Socket} -> + ok = inet:setopts(Socket, [{active, ?TCP_ACTIVE}]), + Q_1 = + case T of + 0 -> + deliver_data(Params#params.dist_handle, Q); + infinity -> + Q + end, + input_handler(Params, Seq, Q_1, infinity); + {tcp, Socket, Chunk} -> + input_chunk(Params, Seq, Q, Chunk); + {tcp_closed, Socket} -> + error(connection_closed); + _Other -> + %% Ignore... + input_handler(Params, Seq, Q, T) + end + after T -> + Q_1 = deliver_data(Params#params.dist_handle, Q), + input_handler(Params, Seq, Q_1, infinity) + end. + +input_chunk(Params, Seq, Q, Chunk) -> + case decrypt_chunk(Params, Seq, Chunk) of + <<?DATA_CHUNK, Cleartext/binary>> -> + input_handler(Params, Seq + 1, enq_binary(Cleartext, Q), 0); + <<?TICK_CHUNK, _/binary>> -> + input_handler(Params, Seq + 1, Q, 0); + #params{} = Params_1 -> + input_handler(Params_1, 0, Q, 0); + _ -> + error(decrypt_error) + end. + +%% ------------------------------------------------------------------------- +%% erlang:dist_ctrl_* helpers + +%% Get data for sending from the VM and place it in a queue +%% +get_data(DistHandle, {Front, Size, Rear}) -> + get_data(DistHandle, Front, Size, Rear). +%% +get_data(DistHandle, Front, Size, Rear) -> + case erlang:dist_ctrl_get_data(DistHandle) of + none -> + {Front, Size, Rear}; + Bin when is_binary(Bin) -> + Len = byte_size(Bin), + get_data( + DistHandle, Front, Size + 4 + Len, + [Bin, <<Len:32>>|Rear]); + [Bin1, Bin2] -> + Len = byte_size(Bin1) + byte_size(Bin2), + get_data( + DistHandle, Front, Size + 4 + Len, + [Bin2, Bin1, <<Len:32>>|Rear]); + Iovec -> + Len = iolist_size(Iovec), + get_data( + DistHandle, Front, Size + 4 + Len, + lists:reverse(Iovec, [<<Len:32>>|Rear])) + end. + +%% De-packet and deliver received data to the VM from a queue +%% +deliver_data(DistHandle, Q) -> + case Q of + {[], Size, []} -> + Size = 0, % Assert + Q; + {[], Size, Rear} -> + [Bin|Front] = lists:reverse(Rear), + deliver_data(DistHandle, Front, Size, [], Bin); + {[Bin|Front], Size, Rear} -> + deliver_data(DistHandle, Front, Size, Rear, Bin) + end. +%% +deliver_data(DistHandle, Front, Size, Rear, Bin) -> + case Bin of + <<DataSizeA:32, DataA:DataSizeA/binary, + DataSizeB:32, DataB:DataSizeB/binary, Rest/binary>> -> + erlang:dist_ctrl_put_data(DistHandle, DataA), + erlang:dist_ctrl_put_data(DistHandle, DataB), + deliver_data( + DistHandle, + Front, Size - (4 + DataSizeA + 4 + DataSizeB), Rear, + Rest); + <<DataSize:32, Data:DataSize/binary, Rest/binary>> -> + erlang:dist_ctrl_put_data(DistHandle, Data), + deliver_data(DistHandle, Front, Size - (4 + DataSize), Rear, Rest); + <<DataSize:32, FirstData/binary>> -> + TotalSize = 4 + DataSize, + if + TotalSize =< Size -> + BinSize = byte_size(Bin), + {MoreData, Q} = + deq_iovec( + TotalSize - BinSize, + Front, Size - BinSize, Rear), + erlang:dist_ctrl_put_data(DistHandle, [FirstData|MoreData]), + deliver_data(DistHandle, Q); + true -> % Incomplete data + {[Bin|Front], Size, Rear} + end; + <<_/binary>> -> + BinSize = byte_size(Bin), + if + 4 =< Size -> % Fragmented header - extract a header bin + {RestHeader, {Front_1, _Size_1, Rear_1}} = + deq_iovec(4 - BinSize, Front, Size - BinSize, Rear), + Header = iolist_to_binary([Bin|RestHeader]), + deliver_data(DistHandle, Front_1, Size, Rear_1, Header); + true -> % Incomplete header + {[Bin|Front], Size, Rear} + end + end. + +%% ------------------------------------------------------------------------- +%% Encryption and decryption helpers + +encrypt_and_send_chunk( + #params{ + socket = Socket, rekey_interval = Seq, + key = Key, iv = IV, hash_algorithm = HashAlgorithm} = Params, + Seq, Cleartext) -> + %% + KeyLen = byte_size(Key), + IVLen = byte_size(IV), + Chunk = <<IV_1:IVLen/binary, KeySalt:KeyLen/binary>> = + crypto:strong_rand_bytes(IVLen + KeyLen), + ok = gen_tcp:send(Socket, encrypt_chunk(Params, Seq, [?REKEY_CHUNK, Chunk])), + Key_1 = hash_key(HashAlgorithm, Key, KeySalt), + Params_1 = Params#params{key = Key_1, iv = IV_1}, + ok = gen_tcp:send(Socket, encrypt_chunk(Params_1, 0, Cleartext)), + {Params_1, 1}; +encrypt_and_send_chunk(#params{socket = Socket} = Params, Seq, Cleartext) -> + ok = gen_tcp:send(Socket, encrypt_chunk(Params, Seq, Cleartext)), + {Params, Seq + 1}. + +encrypt_chunk( + #params{ + block_crypto = BlockCrypto, + iv = IV, key = Key, tag_len = TagLen}, Seq, Cleartext) -> + %% + ChunkLen = iolist_size(Cleartext) + TagLen, + AAD = <<Seq:32, ChunkLen:32>>, + {Ciphertext, CipherTag} = + crypto:block_encrypt( + crypto_cipher_name(BlockCrypto), Key, IV, {AAD, Cleartext, TagLen}), + Chunk = [Ciphertext,CipherTag], + Chunk. + +decrypt_chunk( + #params{ + block_crypto = BlockCrypto, + iv = IV, key = Key, tag_len = TagLen} = Params, Seq, Chunk) -> + %% + ChunkLen = byte_size(Chunk), + true = TagLen =< ChunkLen, % Assert + AAD = <<Seq:32, ChunkLen:32>>, + CiphertextLen = ChunkLen - TagLen, + <<Ciphertext:CiphertextLen/binary, CipherTag:TagLen/binary>> = Chunk, + block_decrypt( + Params, Seq, crypto_cipher_name(BlockCrypto), + Key, IV, {AAD, Ciphertext, CipherTag}). + +block_decrypt( + #params{rekey_interval = Seq} = Params, Seq, CipherName, Key, IV, Data) -> + %% + KeyLen = byte_size(Key), + IVLen = byte_size(IV), + case crypto:block_decrypt(CipherName, Key, IV, Data) of + <<?REKEY_CHUNK, IV_1:IVLen/binary, KeySalt:KeyLen/binary>> -> + Key_1 = hash_key(Params#params.hash_algorithm, Key, KeySalt), + Params#params{iv = IV_1, key = Key_1}; + _ -> + error(decrypt_error) + end; +block_decrypt(_Params, _Seq, CipherName, Key, IV, Data) -> + crypto:block_decrypt(CipherName, Key, IV, Data). + +%% ------------------------------------------------------------------------- +%% Queue of binaries i.e an iovec queue + +empty_q() -> + {[], 0, []}. + +enq_binary(Bin, {Front, Size, Rear}) -> + {Front, Size + byte_size(Bin), [Bin|Rear]}. + +deq_iovec(GetSize, {Front, Size, Rear}) when GetSize =< Size -> + deq_iovec(GetSize, Front, Size, Rear, []). +%% +deq_iovec(GetSize, Front, Size, Rear) -> + deq_iovec(GetSize, Front, Size, Rear, []). +%% +deq_iovec(GetSize, [], Size, Rear, Acc) -> + deq_iovec(GetSize, lists:reverse(Rear), Size, [], Acc); +deq_iovec(GetSize, [Bin|Front], Size, Rear, Acc) -> + BinSize = byte_size(Bin), + if + BinSize < GetSize -> + deq_iovec( + GetSize - BinSize, Front, Size - BinSize, Rear, [Bin|Acc]); + GetSize < BinSize -> + {Bin1,Bin2} = erlang:split_binary(Bin, GetSize), + {lists:reverse(Acc, [Bin1]), {[Bin2|Front], Size - GetSize, Rear}}; + true -> + {lists:reverse(Acc, [Bin]), {Front, Size - BinSize, Rear}} + end. + +%% ------------------------------------------------------------------------- + +%% Trace point +trace(Term) -> Term. + +%% Keep an eye on this Pid (debug) +monitor_dist_proc(Pid) -> +%%% spawn( +%%% fun () -> +%%% MRef = erlang:monitor(process, Pid), +%%% receive +%%% {'DOWN', MRef, _, _, normal} -> +%%% error_logger:error_report( +%%% [dist_proc_died, +%%% {reason, normal}, +%%% {pid, Pid}]); +%%% {'DOWN', MRef, _, _, Reason} -> +%%% error_logger:info_report( +%%% [dist_proc_died, +%%% {reason, Reason}, +%%% {pid, Pid}]) +%%% end +%%% end), + Pid. + +dbg() -> + dbg:stop(), + dbg:tracer(), + dbg:p(all, c), + dbg:tpl(?MODULE, cx), + dbg:tpl(erlang, dist_ctrl_get_data_notification, cx), + dbg:tpl(erlang, dist_ctrl_get_data, cx), + dbg:tpl(erlang, dist_ctrl_put_data, cx), + ok. diff --git a/lib/ssl/test/make_certs.erl b/lib/ssl/test/make_certs.erl index 7f3371da9a..76bf0fa895 100644 --- a/lib/ssl/test/make_certs.erl +++ b/lib/ssl/test/make_certs.erl @@ -377,7 +377,7 @@ req_cnf(Root, C) -> "default_bits = ", integer_to_list(C#config.default_bits), "\n" "RANDFILE = $ROOTDIR/RAND\n" "encrypt_key = no\n" - "default_md = md5\n" + "default_md = sha1\n" "#string_mask = pkix\n" "x509_extensions = ca_ext\n" "prompt = no\n" @@ -427,7 +427,7 @@ ca_cnf( ["crl_extensions = crl_ext\n" || C#config.v2_crls], "unique_subject = no\n" "default_days = 3600\n" - "default_md = md5\n" + "default_md = sha1\n" "preserve = no\n" "policy = policy_match\n" "\n" @@ -511,7 +511,7 @@ ca_cnf( ["crl_extensions = crl_ext\n" || C#config.v2_crls], "unique_subject = no\n" "default_days = 3600\n" - "default_md = md5\n" + "default_md = sha1\n" "preserve = no\n" "policy = policy_match\n" "\n" diff --git a/lib/ssl/test/openssl_server_cipher_suite_SUITE.erl b/lib/ssl/test/openssl_server_cipher_suite_SUITE.erl new file mode 100644 index 0000000000..907de1abe2 --- /dev/null +++ b/lib/ssl/test/openssl_server_cipher_suite_SUITE.erl @@ -0,0 +1,768 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019-2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(openssl_server_cipher_suite_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- +all() -> + [ + {group, 'tlsv1.2'}, + {group, 'tlsv1.1'}, + {group, 'tlsv1'}, + {group, 'sslv3'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} + ]. + +groups() -> + %% TODO: Enable SRP, PSK suites (needs OpenSSL s_server conf) + %% TODO: Enable all "kex" on DTLS + [ + {'tlsv1.2', [], kex()}, + {'tlsv1.1', [], kex()}, + {'tlsv1', [], kex()}, + {'sslv3', [], kex()}, + {'dtlsv1.2', [], dtls_kex()}, + {'dtlsv1', [], dtls_kex()}, + {dhe_rsa, [],[dhe_rsa_3des_ede_cbc, + dhe_rsa_aes_128_cbc, + dhe_rsa_aes_256_cbc, + dhe_rsa_chacha20_poly1305 + ]}, + {ecdhe_rsa, [], [ecdhe_rsa_3des_ede_cbc, + ecdhe_rsa_aes_128_cbc, + ecdhe_rsa_aes_128_gcm, + ecdhe_rsa_aes_256_cbc, + ecdhe_rsa_aes_256_gcm, + ecdhe_rsa_chacha20_poly1305 + ]}, + {ecdhe_ecdsa, [],[ecdhe_ecdsa_rc4_128, + ecdhe_ecdsa_3des_ede_cbc, + ecdhe_ecdsa_aes_128_cbc, + ecdhe_ecdsa_aes_128_gcm, + ecdhe_ecdsa_aes_256_cbc, + ecdhe_ecdsa_aes_256_gcm, + ecdhe_ecdsa_chacha20_poly1305 + ]}, + {rsa, [], [rsa_3des_ede_cbc, + rsa_aes_128_cbc, + rsa_aes_256_cbc, + rsa_rc4_128 + ]}, + {dhe_dss, [], [dhe_dss_3des_ede_cbc, + dhe_dss_aes_128_cbc, + dhe_dss_aes_256_cbc]}, + %% {srp_rsa, [], [srp_rsa_3des_ede_cbc, + %% srp_rsa_aes_128_cbc, + %% srp_rsa_aes_256_cbc]}, + %% {srp_dss, [], [srp_dss_3des_ede_cbc, + %% srp_dss_aes_128_cbc, + %% srp_dss_aes_256_cbc]}, + %% {rsa_psk, [], [rsa_psk_3des_ede_cbc, + %% rsa_psk_rc4_128, + %% rsa_psk_aes_128_cbc, + %% rsa_psk_aes_256_cbc + %% ]}, + {dh_anon, [], [dh_anon_rc4_128, + dh_anon_3des_ede_cbc, + dh_anon_aes_128_cbc, + dh_anon_aes_128_gcm, + dh_anon_aes_256_cbc, + dh_anon_aes_256_gcm]}, + {ecdh_anon, [], [ecdh_anon_3des_ede_cbc, + ecdh_anon_aes_128_cbc, + ecdh_anon_aes_256_cbc + ]} + %% {srp_anon, [], [srp_anon_3des_ede_cbc, + %% srp_anon_aes_128_cbc, + %% srp_anon_aes_256_cbc]}, + %% {psk, [], [psk_3des_ede_cbc, + %% psk_rc4_128, + %% psk_aes_128_cbc, + %% psk_aes_128_ccm, + %% psk_aes_128_ccm_8, + %% psk_aes_256_cbc, + %% psk_aes_256_ccm, + %% psk_aes_256_ccm_8 + %% ]}, + %% {dhe_psk, [], [dhe_psk_3des_ede_cbc, + %% dhe_psk_rc4_128, + %% dhe_psk_aes_128_cbc, + %% dhe_psk_aes_128_ccm, + %% dhe_psk_aes_128_ccm_8, + %% dhe_psk_aes_256_cbc, + %% dhe_psk_aes_256_ccm, + %% dhe_psk_aes_256_ccm_8 + %% ]}, + %% {ecdhe_psk, [], [ecdhe_psk_3des_ede_cbc, + %% ecdhe_psk_rc4_128, + %% ecdhe_psk_aes_128_cbc, + %% ecdhe_psk_aes_128_ccm, + %% ecdhe_psk_aes_128_ccm_8, + %% ecdhe_psk_aes_256_cbc + %% ]} + ]. + +kex() -> + rsa() ++ ecdsa() ++ dss() ++ anonymous(). + +dtls_kex() -> %% Should be all kex in the future + dtls_rsa() ++ dss() ++ anonymous(). + +rsa() -> + [{group, dhe_rsa}, + {group, ecdhe_rsa}, + {group, rsa} %%, {group, srp_rsa}, + %%{group, rsa_psk} + ]. + +dtls_rsa() -> + [ + {group, rsa} + %%,{group, rsa_psk} + ]. + +ecdsa() -> + [{group, ecdhe_ecdsa}]. + +dss() -> + [{group, dhe_dss} + %%{group, srp_dss} + ]. + +anonymous() -> + [{group, dh_anon}, + {group, ecdh_anon} + %% {group, psk}, + %%{group, dhe_psk}, + %%{group, ecdhe_psk} + %%{group, srp_anon} + ]. + +init_per_suite(Config) -> + catch crypto:stop(), + try crypto:start() of + ok -> + ssl_test_lib:clean_start(), + Config + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(_Config) -> + ssl:stop(), + application:stop(crypto). + +%%-------------------------------------------------------------------- +init_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + case ssl_test_lib:supports_ssl_tls_version(GroupName) of + true -> + do_init_per_group(GroupName, Config); + false -> + {skip, {openssl_does_not_support, GroupName}} + end; + false -> + do_init_per_group(GroupName, Config) + end. + +do_init_per_group(GroupName, Config) when GroupName == ecdh_anon; + GroupName == ecdhe_rsa; + GroupName == ecdhe_psk -> + case proplists:get_bool(ecdh, proplists:get_value(public_keys, crypto:supports())) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing EC crypto support"} + end; +do_init_per_group(ecdhe_ecdsa = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(ecdh, PKAlg) andalso lists:member(ecdsa, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing EC crypto support"} + end; +do_init_per_group(dhe_dss = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dss, PKAlg) andalso lists:member(dh, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing DSS crypto support"} + end; +do_init_per_group(srp_dss = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dss, PKAlg) andalso lists:member(srp, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing DSS_SRP crypto support"} + end; +do_init_per_group(GroupName, Config) when GroupName == srp_anon; + GroupName == srp_rsa -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(srp, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing SRP crypto support"} + end; +do_init_per_group(dhe_psk = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dh, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing SRP crypto support"} + end; +do_init_per_group(GroupName, Config0) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:init_tls_version(GroupName, end_per_group(GroupName, Config0)); + false -> + init_certs(GroupName, Config0) + end. + +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. + +init_per_testcase(TestCase, Config) when TestCase == psk_3des_ede_cbc; + TestCase == srp_anon_3des_ede_cbc; + TestCase == dhe_psk_3des_ede_cbc; + TestCase == ecdhe_psk_3des_ede_cbc; + TestCase == srp_rsa_3des_ede_cbc; + TestCase == srp_dss_3des_ede_cbc; + TestCase == rsa_psk_3des_ede_cbc; + TestCase == rsa_3des_ede_cbc; + TestCase == dhe_rsa_3des_ede_cbc; + TestCase == dhe_dss_3des_ede_cbc; + TestCase == ecdhe_rsa_3des_ede_cbc; + TestCase == srp_anon_dss_3des_ede_cbc; + TestCase == dh_anon_3des_ede_cbc; + TestCase == ecdh_anon_3des_ede_cbc; + TestCase == ecdhe_ecdsa_3des_ede_cbc -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(des_ede3, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing 3DES crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_rc4_128; + TestCase == ecdhe_psk_rc4_128; + TestCase == dhe_psk_rc4_128; + TestCase == rsa_psk_rc4_128; + TestCase == rsa_rc4_128; + TestCase == ecdhe_rsa_rc4_128; + TestCase == ecdhe_ecdsa_rc4_128; + TestCase == dh_anon_rc4_128 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(rc4, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing RC4 crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_aes_128_ccm_8; + TestCase == rsa_psk_aes_128_ccm_8; + TestCase == psk_aes_128_ccm_8; + TestCase == dhe_psk_aes_128_ccm_8; + TestCase == ecdhe_psk_aes_128_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_128_ccm, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing AES_128_CCM crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_aes_256_ccm_8; + TestCase == rsa_psk_aes_256_ccm_8; + TestCase == psk_aes_256_ccm_8; + TestCase == dhe_psk_aes_256_ccm_8; + TestCase == ecdhe_psk_aes_256_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_256_ccm, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing AES_256_CCM crypto support"} + end; +init_per_testcase(TestCase, Config) -> + Cipher = ssl_test_lib:test_cipher(TestCase, Config), + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(Cipher, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, {Cipher, SupCiphers}} + end. + +end_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Initializtion ------------------------------------------ +%%-------------------------------------------------------------------- +init_certs(srp_rsa, Config) -> + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, ssl_test_lib:default_cert_chain_conf()}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, ""), + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}} | ServerOpts], + client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(srp_anon, Config) -> + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}}], + client_config => [{srp_identity, {"Test-User", "secret"}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(rsa_psk, Config) -> + Ext = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, + [[ssl_test_lib:digest()],[ssl_test_lib:digest()], + [ssl_test_lib:digest(), {extensions, Ext}]]}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, "_peer_keyEncipherment"), + PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}} | ServerOpts], + client_config => [{psk_identity, "Test-User"}, + {user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(rsa, Config) -> + Ext = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, + [[ssl_test_lib:digest()],[ssl_test_lib:digest()], + [ssl_test_lib:digest(), {extensions, Ext}]]} + ], + Config, "_peer_keyEncipherment"), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(dhe_dss, Config) -> + {ClientOpts, ServerOpts} = ssl_test_lib:make_dsa_cert_chains([{server_chain, ssl_test_lib:default_cert_chain_conf()}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, ""), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(srp_dss, Config) -> + {ClientOpts, ServerOpts} = ssl_test_lib:make_dsa_cert_chains([{server_chain, ssl_test_lib:default_cert_chain_conf()}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, ""), + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}} | ServerOpts], + client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == dhe_rsa; + GroupName == ecdhe_rsa -> + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, ssl_test_lib:default_cert_chain_conf()}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, ""), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == dhe_ecdsa; + GroupName == ecdhe_ecdsa -> + {ClientOpts, ServerOpts} = ssl_test_lib:make_ecc_cert_chains([{server_chain, ssl_test_lib:default_cert_chain_conf()}, + {client_chain, ssl_test_lib:default_cert_chain_conf()}], + Config, ""), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == psk; + GroupName == dhe_psk; + GroupName == ecdhe_psk -> + PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}}], + client_config => [{psk_identity, "Test-User"}, + {user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(srp, Config) -> + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}}], + client_config => [{srp_identity, {"Test-User", "secret"}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(_GroupName, Config) -> + %% Anonymous does not need certs + [{tls_config, #{server_config => [], + client_config => []}} | + proplists:delete(tls_config, Config)]. +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% SRP -------------------------------------------------------- +%%-------------------------------------------------------------------- +srp_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, '3des_ede_cbc', Config). + +srp_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, 'aes_128_cbc', Config). + +srp_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, 'aes_256_cbc', Config). + +srp_dss_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, '3des_ede_cbc', Config). + +srp_dss_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, 'aes_128_cbc', Config). + +srp_dss_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, 'aes_256_cbc', Config). + +%%-------------------------------------------------------------------- +%% PSK -------------------------------------------------------- +%%-------------------------------------------------------------------- +rsa_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, '3des_ede_cbc', Config). + +rsa_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_cbc', Config). + +rsa_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_ccm', Config). + +rsa_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_ccm_8', Config). + +rsa_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_cbc', Config). + +rsa_psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_ccm', Config). + +rsa_psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_ccm_8', Config). + +rsa_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'rc4_128', Config). + +%%-------------------------------------------------------------------- +%% RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +rsa_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'des_cbc', Config). + +rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, '3des_ede_cbc', Config). + +rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_128_cbc', Config). + +rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_256_cbc', Config). + +rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_128_gcm', Config). + +rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_256_gcm', Config). + +rsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'rc4_128', Config). +%%-------------------------------------------------------------------- +%% DHE_RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +dhe_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, '3des_ede_cbc', Config). + +dhe_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_128_cbc', Config). + +dhe_rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_128_gcm', Config). + +dhe_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_256_cbc', Config). + +dhe_rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_256_gcm', Config). + +dhe_rsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'chacha20_poly1305', Config). +%%-------------------------------------------------------------------- +%% ECDHE_RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +ecdhe_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, '3des_ede_cbc', Config). + +ecdhe_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_128_cbc', Config). + +ecdhe_rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_128_gcm', Config). + +ecdhe_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_256_cbc', Config). + +ecdhe_rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_256_gcm', Config). + +ecdhe_rsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'rc4_128', Config). + +ecdhe_rsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'chacha20_poly1305', Config). + +%%-------------------------------------------------------------------- +%% ECDHE_ECDSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +ecdhe_ecdsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'rc4_128', Config). + +ecdhe_ecdsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, '3des_ede_cbc', Config). + +ecdhe_ecdsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_cbc', Config). + +ecdhe_ecdsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_gcm', Config). + +ecdhe_ecdsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_cbc', Config). + +ecdhe_ecdsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_gcm', Config). + +ecdhe_ecdsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'chacha20_poly1305', Config). +%%-------------------------------------------------------------------- +%% DHE_DSS -------------------------------------------------------- +%%-------------------------------------------------------------------- +dhe_dss_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'des_cbc', Config). + +dhe_dss_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, '3des_ede_cbc', Config). + +dhe_dss_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_128_cbc', Config). + +dhe_dss_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_256_cbc', Config). + +dhe_dss_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_128_gcm', Config). + +dhe_dss_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_256_gcm', Config). + +%%-------------------------------------------------------------------- +%% Anonymous -------------------------------------------------------- +%%-------------------------------------------------------------------- +dh_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, '3des_ede_cbc', Config). + +dh_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_128_cbc', Config). + +dh_anon_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_128_gcm', Config). + +dh_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_256_cbc', Config). + +dh_anon_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_256_gcm', Config). + +dh_anon_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'rc4_128', Config). + +ecdh_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, '3des_ede_cbc', Config). + +ecdh_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, 'aes_128_cbc', Config). + +ecdh_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, 'aes_256_cbc', Config). + +srp_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, '3des_ede_cbc', Config). + +srp_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, 'aes_128_cbc', Config). + +srp_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, 'aes_256_cbc', Config). + +dhe_psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'des_cbc', Config). + +dhe_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'rc4_128', Config). + +dhe_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, '3des_ede_cbc', Config). + +dhe_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_cbc', Config). + +dhe_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_cbc', Config). + +dhe_psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_gcm', Config). + +dhe_psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_gcm', Config). + +dhe_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_ccm', Config). + +dhe_psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_ccm', Config). + +dhe_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_ccm_8', Config). + +dhe_psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_ccm_8', Config). + +ecdhe_psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'des_cbc', Config). + +ecdhe_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'rc4_128', Config). + +ecdhe_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, '3des_ede_cbc', Config). + +ecdhe_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_cbc', Config). + +ecdhe_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_256_cbc', Config). + +ecdhe_psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_gcm', Config). + +ecdhe_psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_256_gcm', Config). + +ecdhe_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_ccm', Config). + +ecdhe_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_ccm_8', Config). + +psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'des_cbc', Config). + +psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(psk, 'rc4_128', Config). + +psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, '3des_ede_cbc', Config). + +psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_cbc', Config). + +psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_cbc', Config). + +psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_gcm', Config). + +psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_gcm', Config). + +psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_ccm', Config). + +psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_ccm', Config). + +psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_ccm_8', Config). + +psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_ccm_8', Config). + +%%-------------------------------------------------------------------- +%% Internal functions ---------------------------------------------- +%%-------------------------------------------------------------------- +run_ciphers_test(Kex, Cipher, Config) -> + Version = ssl_test_lib:protocol_version(Config), + TestCiphers = test_ciphers(Kex, Cipher, Version), + + case TestCiphers of + [_|_] -> + lists:foreach(fun(TestCipher) -> + cipher_suite_test(TestCipher, Version, Config) + end, TestCiphers); + [] -> + {skip, {not_sup, Kex, Cipher, Version}} + end. + +cipher_suite_test(CipherSuite, _Version, Config) -> + #{server_config := SOpts, + client_config := COpts} = proplists:get_value(tls_config, Config), + ServerOpts = ssl_test_lib:ssl_options(SOpts, Config), + ClientOpts = ssl_test_lib:ssl_options(COpts, Config), + ct:log("Testing CipherSuite ~p~n", [CipherSuite]), + ct:log("Server Opts ~p~n", [ServerOpts]), + ct:log("Client Opts ~p~n", [ClientOpts]), + ssl_test_lib:basic_test([{ciphers, [CipherSuite]} | COpts], SOpts, [{client_type, erlang}, + {server_type, openssl} | Config]). + + +test_ciphers(Kex, Cipher, Version) -> + Ciphers = ssl:filter_cipher_suites(ssl:cipher_suites(default, Version) ++ ssl:cipher_suites(anonymous, Version), + [{key_exchange, + fun(Kex0) when Kex0 == Kex -> true; + (_) -> false + end}, + {cipher, + fun(Cipher0) when Cipher0 == Cipher -> true; + (_) -> false + end}]), + ct:log("Version ~p Testing ~p~n", [Version, Ciphers]), + OpenSSLCiphers = openssl_ciphers(), + ct:log("OpenSSLCiphers ~p~n", [OpenSSLCiphers]), + lists:filter(fun(C) -> + ct:log("Cipher ~p~n", [C]), + lists:member(ssl_cipher_format:suite_map_to_openssl_str(C), OpenSSLCiphers) + end, Ciphers). + + +openssl_ciphers() -> + Str = os:cmd("openssl ciphers"), + string:split(string:strip(Str, right, $\n), ":", all). diff --git a/lib/ssl/test/property_test/ssl_eqc_handshake.erl b/lib/ssl/test/property_test/ssl_eqc_handshake.erl new file mode 100644 index 0000000000..38a4b7fb11 --- /dev/null +++ b/lib/ssl/test/property_test/ssl_eqc_handshake.erl @@ -0,0 +1,812 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018-2018. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% + +-module(ssl_eqc_handshake). + +-compile(export_all). + +-proptest(eqc). +-proptest([triq,proper]). + +-ifndef(EQC). +-ifndef(PROPER). +-ifndef(TRIQ). +-define(EQC,true). +-endif. +-endif. +-endif. + +-ifdef(EQC). +-include_lib("eqc/include/eqc.hrl"). +-define(MOD_eqc,eqc). + +-else. +-ifdef(PROPER). +-include_lib("proper/include/proper.hrl"). +-define(MOD_eqc,proper). + +-else. +-ifdef(TRIQ). +-define(MOD_eqc,triq). +-include_lib("triq/include/triq.hrl"). + +-endif. +-endif. +-endif. + +-include_lib("kernel/include/inet.hrl"). +-include_lib("ssl/src/tls_handshake_1_3.hrl"). +-include_lib("ssl/src/tls_handshake.hrl"). +-include_lib("ssl/src/ssl_handshake.hrl"). +-include_lib("ssl/src/ssl_alert.hrl"). +-include_lib("ssl/src/ssl_internal.hrl"). + +-define('TLS_v1.3', {3,4}). +-define('TLS_v1.2', {3,3}). +-define('TLS_v1.1', {3,2}). +-define('TLS_v1', {3,1}). +-define('SSL_v3', {3,0}). + +%%-------------------------------------------------------------------- +%% Properties -------------------------------------------------------- +%%-------------------------------------------------------------------- + +prop_tls_hs_encode_decode() -> + ?FORALL({Handshake, TLSVersion}, ?LET(Version, tls_version(), {tls_msg(Version), Version}), + try + [Type, _Length, Data] = tls_handshake:encode_handshake(Handshake, TLSVersion), + case tls_handshake:decode_handshake(TLSVersion, Type, Data) of + Handshake -> + true; + _ -> + false + end + catch + throw:#alert{} -> + true + end + ). + +%%-------------------------------------------------------------------- +%% Message Generators ----------------------------------------------- +%%-------------------------------------------------------------------- + +tls_msg(?'TLS_v1.3'= Version) -> + oneof([client_hello(Version), + server_hello(Version), + %%new_session_ticket() + #end_of_early_data{}, + encrypted_extensions(), + certificate_1_3(), + %%certificate_request_1_3, + certificate_verify_1_3(), + finished(), + key_update() + ]); +tls_msg(Version) -> + oneof([ + #hello_request{}, + client_hello(Version), + server_hello(Version), + certificate(), + %%server_key_exchange() + certificate_request(Version), + #server_hello_done{}, + %%certificate_verify() + %%client_key_exchange() + finished() + ]). + +%% +%% Shared messages +%% +client_hello(?'TLS_v1.3' = Version) -> + #client_hello{session_id = session_id(), + client_version = ?'TLS_v1.2', + cipher_suites = cipher_suites(Version), + compression_methods = compressions(Version), + random = client_random(Version), + extensions = client_hello_extensions(Version) + }; +client_hello(Version) -> + #client_hello{session_id = session_id(), + client_version = Version, + cipher_suites = cipher_suites(Version), + compression_methods = compressions(Version), + random = client_random(Version), + extensions = client_hello_extensions(Version) + }. + +server_hello(?'TLS_v1.3' = Version) -> + #server_hello{server_version = ?'TLS_v1.2', + session_id = session_id(), + random = server_random(Version), + cipher_suite = cipher_suite(Version), + compression_method = compression(Version), + extensions = server_hello_extensions(Version) + }; +server_hello(Version) -> + #server_hello{server_version = Version, + session_id = session_id(), + random = server_random(Version), + cipher_suite = cipher_suite(Version), + compression_method = compression(Version), + extensions = server_hello_extensions(Version) + }. + +certificate() -> + #certificate{ + asn1_certificates = certificate_chain() + }. + +certificate_1_3() -> + ?LET(Certs, certificate_chain(), + #certificate_1_3{ + certificate_request_context = certificate_request_context(), + certificate_list = certificate_entries(Certs, []) + }). + +certificate_verify_1_3() -> + ?LET(Certs, certificate_chain(), + #certificate_verify_1_3{ + algorithm = sig_scheme(), + signature = signature() + }). + +finished() -> + ?LET(Size, digest_size(), + #finished{verify_data = crypto:strong_rand_bytes(Size)}). + +%% +%% TLS 1.0-1.2 messages +%% + + + +%% +%% TLS 1.3 messages +%% + +encrypted_extensions() -> + ?LET(Exts, extensions(?'TLS_v1.3', encrypted_extensions), + #encrypted_extensions{extensions = Exts}). + + +key_update() -> + #key_update{request_update = request_update()}. + + +%%-------------------------------------------------------------------- +%% Messge Data Generators ------------------------------------------- +%%-------------------------------------------------------------------- + +tls_version() -> + oneof([?'TLS_v1.3', ?'TLS_v1.2', ?'TLS_v1.1', ?'TLS_v1', ?'SSL_v3']). + +cipher_suite(Version) -> + oneof(cipher_suites(Version)). + +cipher_suites(Version) -> + ssl_cipher:suites(Version). + +session_id() -> + crypto:strong_rand_bytes(?NUM_OF_SESSION_ID_BYTES). + +compression(Version) -> + oneof(compressions(Version)). + +compressions(_) -> + ssl_record:compressions(). + +client_random(_) -> + crypto:strong_rand_bytes(32). + +server_random(_) -> + crypto:strong_rand_bytes(32). + + +client_hello_extensions(Version) -> + ?LET(Exts, extensions(Version, client_hello), + maps:merge(ssl_handshake:empty_extensions(Version, client_hello), + Exts)). + +server_hello_extensions(Version) -> + ?LET(Exts, extensions(Version, server_hello), + maps:merge(ssl_handshake:empty_extensions(Version, server_hello), + Exts)). + +key_share_client_hello() -> + oneof([undefined]). + %%oneof([#key_share_client_hello{}, undefined]). + +key_share_server_hello() -> + oneof([undefined]). + %%oneof([#key_share_server_hello{}, undefined]). + +pre_shared_keyextension() -> + oneof([undefined]). + %%oneof([#pre_shared_keyextension{},undefined]). + +%% +--------------------------------------------------+-------------+ +%% | Extension | TLS 1.3 | +%% +--------------------------------------------------+-------------+ +%% | server_name [RFC6066] | CH, EE | +%% | | | +%% | max_fragment_length [RFC6066] | CH, EE | +%% | | | +%% | status_request [RFC6066] | CH, CR, CT | +%% | | | +%% | supported_groups [RFC7919] | CH, EE | +%% | | | +%% | signature_algorithms (RFC 8446) | CH, CR | +%% | | | +%% | use_srtp [RFC5764] | CH, EE | +%% | | | +%% | heartbeat [RFC6520] | CH, EE | +%% | | | +%% | application_layer_protocol_negotiation [RFC7301] | CH, EE | +%% | | | +%% | signed_certificate_timestamp [RFC6962] | CH, CR, CT | +%% | | | +%% | client_certificate_type [RFC7250] | CH, EE | +%% | | | +%% | server_certificate_type [RFC7250] | CH, EE | +%% | | | +%% | padding [RFC7685] | CH | +%% | | | +%% | key_share (RFC 8446) | CH, SH, HRR | +%% | | | +%% | pre_shared_key (RFC 8446) | CH, SH | +%% | | | +%% | psk_key_exchange_modes (RFC 8446) | CH | +%% | | | +%% | early_data (RFC 8446) | CH, EE, NST | +%% | | | +%% | cookie (RFC 8446) | CH, HRR | +%% | | | +%% | supported_versions (RFC 8446) | CH, SH, HRR | +%% | | | +%% | certificate_authorities (RFC 8446) | CH, CR | +%% | | | +%% | oid_filters (RFC 8446) | CR | +%% | | | +%% | post_handshake_auth (RFC 8446) | CH | +%% | | | +%% | signature_algorithms_cert (RFC 8446) | CH, CR | +%% +--------------------------------------------------+-------------+ +extensions(?'TLS_v1.3' = Version, client_hello) -> + ?LET({ + ServerName, + %% MaxFragmentLength, + %% StatusRequest, + SupportedGroups, + SignatureAlgorithms, + %% UseSrtp, + %% Heartbeat, + ALPN, + %% SignedCertTimestamp, + %% ClientCertiticateType, + %% ServerCertificateType, + %% Padding, + KeyShare, + %% PreSharedKey, + %% PSKKeyExchangeModes, + %% EarlyData, + %% Cookie, + SupportedVersions, + %% CertAuthorities, + %% PostHandshakeAuth, + SignatureAlgorithmsCert + }, + { + oneof([server_name(), undefined]), + %% oneof([max_fragment_length(), undefined]), + %% oneof([status_request(), undefined]), + oneof([supported_groups(Version), undefined]), + oneof([signature_algs(Version), undefined]), + %% oneof([use_srtp(), undefined]), + %% oneof([heartbeat(), undefined]), + oneof([alpn(), undefined]), + %% oneof([signed_cert_timestamp(), undefined]), + %% oneof([client_cert_type(), undefined]), + %% oneof([server_cert_type(), undefined]), + %% oneof([padding(), undefined]), + oneof([key_share(client_hello), undefined]), + %% oneof([pre_shared_key(), undefined]), + %% oneof([psk_key_exchange_modes(), undefined]), + %% oneof([early_data(), undefined]), + %% oneof([cookie(), undefined]), + oneof([client_hello_versions(Version)]), + %% oneof([cert_authorities(), undefined]), + %% oneof([post_handshake_auth(), undefined]), + oneof([signature_algs_cert(), undefined]) + }, + maps:filter(fun(_, undefined) -> + false; + (_,_) -> + true + end, + #{ + sni => ServerName, + %% max_fragment_length => MaxFragmentLength, + %% status_request => StatusRequest, + elliptic_curves => SupportedGroups, + signature_algs => SignatureAlgorithms, + %% use_srtp => UseSrtp, + %% heartbeat => Heartbeat, + alpn => ALPN, + %% signed_cert_timestamp => SignedCertTimestamp, + %% client_cert_type => ClientCertificateType, + %% server_cert_type => ServerCertificateType, + %% padding => Padding, + key_share => KeyShare, + %% pre_shared_key => PreSharedKey, + %% psk_key_exhange_modes => PSKKeyExchangeModes, + %% early_data => EarlyData, + %% cookie => Cookie, + client_hello_versions => SupportedVersions, + %% cert_authorities => CertAuthorities, + %% post_handshake_auth => PostHandshakeAuth, + signature_algs_cert => SignatureAlgorithmsCert + })); +extensions(?'SSL_v3', client_hello) -> + #{}; +extensions(Version, client_hello) -> + ?LET({ + SNI, + ECPoitF, + ECCurves, + ALPN, + NextP, + SRP + %% RenegotiationInfo + }, + { + oneof([sni(), undefined]), + oneof([ec_point_formats(), undefined]), + oneof([elliptic_curves(Version), undefined]), + oneof([alpn(), undefined]), + oneof([next_protocol_negotiation(), undefined]), + oneof([srp(), undefined]) + %% oneof([renegotiation_info(), undefined]) + }, + maps:filter(fun(_, undefined) -> + false; + (_,_) -> + true + end, + #{ + sni => SNI, + ec_point_formats => ECPoitF, + elliptic_curves => ECCurves, + alpn => ALPN, + next_protocol_negotiation => NextP, + srp => SRP + %% renegotiation_info => RenegotiationInfo + })); +extensions(?'TLS_v1.3' = Version, server_hello) -> + ?LET({ + KeyShare, + %% PreSharedKeys, + SupportedVersions + }, + { + oneof([key_share(server_hello), undefined]), + %% oneof([pre_shared_keys(), undefined]), + oneof([server_hello_selected_version()]) + }, + maps:filter(fun(_, undefined) -> + false; + (_,_) -> + true + end, + #{ + key_share => KeyShare, + %% pre_shared_keys => PreSharedKeys, + server_hello_selected_version => SupportedVersions + })); +extensions(Version, server_hello) -> + ?LET({ + ECPoitF, + ALPN, + NextP + %% RenegotiationInfo, + }, + { + oneof([ec_point_formats(), undefined]), + oneof([alpn(), undefined]), + oneof([next_protocol_negotiation(), undefined]) + %% oneof([renegotiation_info(), undefined]), + }, + maps:filter(fun(_, undefined) -> + false; + (_,_) -> + true + end, + #{ + ec_point_formats => ECPoitF, + alpn => ALPN, + next_protocol_negotiation => NextP + %% renegotiation_info => RenegotiationInfo + })); +extensions(?'TLS_v1.3' = Version, encrypted_extensions) -> + ?LET({ + ServerName, + %% MaxFragmentLength, + SupportedGroups, + %% UseSrtp, + %% Heartbeat, + ALPN + %% ClientCertiticateType, + %% ServerCertificateType, + %% EarlyData + }, + { + oneof([server_name(), undefined]), + %% oneof([max_fragment_length(), undefined]), + oneof([supported_groups(Version), undefined]), + %% oneof([use_srtp(), undefined]), + %% oneof([heartbeat(), undefined]), + oneof([alpn(), undefined]) + %% oneof([client_cert_type(), undefined]), + %% oneof([server_cert_type(), undefined]), + %% oneof([early_data(), undefined]) + }, + maps:filter(fun(_, undefined) -> + false; + (_,_) -> + true + end, + #{ + sni => ServerName, + %% max_fragment_length => MaxFragmentLength, + elliptic_curves => SupportedGroups, + %% use_srtp => UseSrtp, + %% heartbeat => Heartbeat, + alpn => ALPN + %% client_cert_type => ClientCertificateType, + %% server_cert_type => ServerCertificateType, + %% early_data => EarlyData + })). + +server_name() -> + ?LET(ServerName, sni(), + ServerName). + %% sni(). + +signature_algs_cert() -> + ?LET(List, sig_scheme_list(), + #signature_algorithms_cert{signature_scheme_list = List}). + +signature_algorithms() -> + ?LET(List, sig_scheme_list(), + #signature_algorithms{signature_scheme_list = List}). + +sig_scheme_list() -> + oneof([[rsa_pkcs1_sha256], + [rsa_pkcs1_sha256, ecdsa_sha1], + [rsa_pkcs1_sha256, + rsa_pkcs1_sha384, + rsa_pkcs1_sha512, + ecdsa_secp256r1_sha256, + ecdsa_secp384r1_sha384, + ecdsa_secp521r1_sha512, + rsa_pss_rsae_sha256, + rsa_pss_rsae_sha384, + rsa_pss_rsae_sha512, + rsa_pss_pss_sha256, + rsa_pss_pss_sha384, + rsa_pss_pss_sha512, + rsa_pkcs1_sha1, + ecdsa_sha1] + ]). + +sig_scheme() -> + oneof([rsa_pkcs1_sha256, + rsa_pkcs1_sha384, + rsa_pkcs1_sha512, + ecdsa_secp256r1_sha256, + ecdsa_secp384r1_sha384, + ecdsa_secp521r1_sha512, + rsa_pss_rsae_sha256, + rsa_pss_rsae_sha384, + rsa_pss_rsae_sha512, + rsa_pss_pss_sha256, + rsa_pss_pss_sha384, + rsa_pss_pss_sha512, + rsa_pkcs1_sha1, + ecdsa_sha1]). + +signature() -> + <<44,119,215,137,54,84,156,26,121,212,64,173,189,226, + 191,46,76,89,204,2,78,79,163,228,90,21,89,179,4,198, + 109,14,52,26,230,22,56,8,170,129,86,0,7,132,245,81, + 181,131,62,70,79,167,112,85,14,171,175,162,110,29, + 212,198,45,188,83,176,251,197,224,104,95,74,89,59, + 26,60,63,79,238,196,137,65,23,199,127,145,176,184, + 216,3,48,116,172,106,97,83,227,172,246,137,91,79, + 173,119,169,60,67,1,177,117,9,93,38,86,232,253,73, + 140,17,147,130,110,136,245,73,10,91,70,105,53,225, + 158,107,60,190,30,14,26,92,147,221,60,117,104,53,70, + 142,204,7,131,11,183,192,120,246,243,68,99,147,183, + 49,149,48,188,8,218,17,150,220,121,2,99,194,140,35, + 13,249,201,37,216,68,45,87,58,18,10,106,11,132,241, + 71,170,225,216,197,212,29,107,36,80,189,184,202,56, + 86,213,45,70,34,74,71,48,137,79,212,194,172,151,57, + 57,30,126,24,157,198,101,220,84,162,89,105,185,245, + 76,105,212,176,25,6,148,49,194,106,253,241,212,200, + 37,154,227,53,49,216,72,82,163>>. + +client_hello_versions(?'TLS_v1.3') -> + ?LET(SupportedVersions, + oneof([[{3,4}], + %% This list breaks the property but can be used for negative tests + %% [{3,3},{3,4}], + [{3,4},{3,3}], + [{3,4},{3,3},{3,2},{3,1},{3,0}] + ]), + #client_hello_versions{versions = SupportedVersions}); +client_hello_versions(_) -> + ?LET(SupportedVersions, + oneof([[{3,3}], + [{3,3},{3,2}], + [{3,3},{3,2},{3,1},{3,0}] + ]), + #client_hello_versions{versions = SupportedVersions}). + +server_hello_selected_version() -> + #server_hello_selected_version{selected_version = {3,4}}. + +request_update() -> + oneof([?UPDATE_NOT_REQUESTED, ?UPDATE_REQUESTED]). + +certificate_chain()-> + Conf = cert_conf(), + ?LET(Chain, + choose_certificate_chain(Conf), + Chain). + +choose_certificate_chain(#{server_config := ServerConf, + client_config := ClientConf}) -> + oneof([certificate_chain(ServerConf), certificate_chain(ClientConf)]). + +certificate_request_context() -> + oneof([<<>>, + <<1>>, + <<"foobar">> + ]). +certificate_entries([], Acc) -> + lists:reverse(Acc); +certificate_entries([Cert | Rest], Acc) -> + certificate_entries(Rest, [certificate_entry(Cert) | Acc]). + +certificate_entry(Cert) -> + #certificate_entry{data = Cert, + extensions = certificate_entry_extensions() + }. +certificate_entry_extensions() -> + #{}. + +certificate_chain(Conf) -> + CAs = proplists:get_value(cacerts, Conf), + Cert = proplists:get_value(cert, Conf), + %% Middle argument are of correct type but will not be used + {ok, _, Chain} = ssl_certificate:certificate_chain(Cert, ets:new(foo, []), make_ref(), CAs), + Chain. + +cert_conf()-> + Hostname = net_adm:localhost(), + {ok, #hostent{h_addr_list = [_IP |_]}} = inet:gethostbyname(net_adm:localhost()), + public_key:pkix_test_data(#{server_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(1)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(2)}]], + peer => [{extensions, [#'Extension'{extnID = + ?'id-ce-subjectAltName', + extnValue = [{dNSName, Hostname}], + critical = false}]}, + {key, ssl_test_lib:hardcode_rsa_key(3)} + ]}, + client_chain => + #{root => [{key, ssl_test_lib:hardcode_rsa_key(4)}], + intermediates => [[{key, ssl_test_lib:hardcode_rsa_key(5)}]], + peer => [{key, ssl_test_lib:hardcode_rsa_key(6)}]}}). + +certificate_request(Version) -> + #certificate_request{certificate_types = certificate_types(Version), + hashsign_algorithms = hashsign_algorithms(Version), + certificate_authorities = certificate_authorities()}. + +certificate_types(?'TLS_v1.3') -> + iolist_to_binary([<<?BYTE(?ECDSA_SIGN)>>, <<?BYTE(?RSA_SIGN)>>]); +certificate_types(?'TLS_v1.2') -> + iolist_to_binary([<<?BYTE(?ECDSA_SIGN)>>, <<?BYTE(?RSA_SIGN)>>, <<?BYTE(?DSS_SIGN)>>]); +certificate_types(_) -> + iolist_to_binary([<<?BYTE(?ECDSA_SIGN)>>, <<?BYTE(?RSA_SIGN)>>, <<?BYTE(?DSS_SIGN)>>]). + + + +signature_algs({3,4}) -> + ?LET(Algs, signature_algorithms(), + Algs); +signature_algs({3,3} = Version) -> + #hash_sign_algos{hash_sign_algos = hash_alg_list(Version)}; +signature_algs(Version) when Version < {3,3} -> + undefined. + + + +hashsign_algorithms({_, N} = Version) when N >= 3 -> + #hash_sign_algos{hash_sign_algos = hash_alg_list(Version)}; +hashsign_algorithms(_) -> + undefined. + +hash_alg_list(Version) -> + ?LET(NumOf, choose(1,15), + ?LET(List, [hash_alg(Version) || _ <- lists:seq(1,NumOf)], + lists:usort(List) + )). + +hash_alg(Version) -> + ?LET(Alg, sign_algorithm(Version), + {hash_algorithm(Version, Alg), Alg} + ). + +hash_algorithm(?'TLS_v1.3', _) -> + oneof([sha, sha224, sha256, sha384, sha512]); +hash_algorithm(?'TLS_v1.2', rsa) -> + oneof([sha, sha224, sha256, sha384, sha512]); +hash_algorithm(_, rsa) -> + oneof([md5, sha, sha224, sha256, sha384, sha512]); +hash_algorithm(_, ecdsa) -> + oneof([sha, sha224, sha256, sha384, sha512]); +hash_algorithm(_, dsa) -> + sha. + +sign_algorithm(?'TLS_v1.3') -> + oneof([rsa, ecdsa]); +sign_algorithm(_) -> + oneof([rsa, dsa, ecdsa]). + +certificate_authorities() -> + #{server_config := ServerConf} = cert_conf(), + Authorities = proplists:get_value(cacerts, ServerConf), + Enc = fun(#'OTPCertificate'{tbsCertificate=TBSCert}) -> + OTPSubj = TBSCert#'OTPTBSCertificate'.subject, + DNEncodedBin = public_key:pkix_encode('Name', OTPSubj, otp), + DNEncodedLen = byte_size(DNEncodedBin), + <<?UINT16(DNEncodedLen), DNEncodedBin/binary>> + end, + list_to_binary([Enc(public_key:pkix_decode_cert(DERCert, otp)) || DERCert <- Authorities]). + +digest_size()-> + oneof([160,224,256,384,512]). + +key_share_entry() -> + undefined. + %%#key_share_entry{}. + +server_hello_selected_version(Version) -> + #server_hello_selected_version{selected_version = Version}. + +sni() -> + #sni{hostname = net_adm:localhost()}. + +ec_point_formats() -> + #ec_point_formats{ec_point_format_list = ec_point_format_list()}. + +ec_point_format_list() -> + [?ECPOINT_UNCOMPRESSED]. + +elliptic_curves({_, Minor}) when Minor < 4 -> + Curves = tls_v1:ecc_curves(Minor), + #elliptic_curves{elliptic_curve_list = Curves}. + +%% RFC 8446 (TLS 1.3) renamed the "elliptic_curve" extension. +supported_groups({_, Minor}) when Minor >= 4 -> + SupportedGroups = tls_v1:groups(Minor), + #supported_groups{supported_groups = SupportedGroups}. + + +alpn() -> + ?LET(ExtD, alpn_protocols(), #alpn{extension_data = ExtD}). + +alpn_protocols() -> + oneof([<<"spdy/2">>, <<"spdy/3">>, <<"http/2">>, <<"http/1.0">>, <<"http/1.1">>]). + +next_protocol_negotiation() -> + %% Predecessor to APLN + ?LET(ExtD, alpn_protocols(), #next_protocol_negotiation{extension_data = ExtD}). + +srp() -> + ?LET(Name, gen_name(), #srp{username = list_to_binary(Name)}). + +renegotiation_info() -> + #renegotiation_info{renegotiated_connection = 0}. + +gen_name() -> + ?LET(Size, choose(1,10), gen_string(Size)). + +gen_char() -> + choose($a,$z). + +gen_string(N) -> + gen_string(N, []). + +gen_string(0, Acc) -> + Acc; +gen_string(N, Acc) -> + ?LET(Char, gen_char(), gen_string(N-1, [Char | Acc])). + +key_share(client_hello) -> + ?LET(ClientShares, key_share_entry_list(), + #key_share_client_hello{ + client_shares = ClientShares}); +key_share(server_hello) -> + ?LET([ServerShare], key_share_entry_list(1), + #key_share_server_hello{ + server_share = ServerShare}). + +key_share_entry_list() -> + Max = length(ssl:groups()), + ?LET(Size, choose(1,Max), key_share_entry_list(Size)). +%% +key_share_entry_list(N) -> + key_share_entry_list(N, ssl:groups(), []). +%% +key_share_entry_list(0, _Pool, Acc) -> + Acc; +key_share_entry_list(N, Pool, Acc) -> + R = rand:uniform(length(Pool)), + G = lists:nth(R, Pool), + P = generate_public_key(G), + KeyShareEntry = + #key_share_entry{ + group = G, + key_exchange = P}, + key_share_entry_list(N - 1, Pool -- [G], [KeyShareEntry|Acc]). + +%% TODO: fix curve generation +generate_public_key(Group) + when Group =:= secp256r1 orelse + Group =:= secp384r1 orelse + Group =:= secp521r1 orelse + Group =:= x448 orelse + Group =:= x25519 -> + #'ECPrivateKey'{publicKey = PublicKey} = + public_key:generate_key({namedCurve, secp256r1}), + PublicKey; +generate_public_key(Group) -> + {PublicKey, _} = + public_key:generate_key(ssl_dh_groups:dh_params(Group)), + PublicKey. + +groups() -> + Max = length(ssl:groups()), + ?LET(Size, choose(1,Max), group_list(Size)). + +group_list(N) -> + group_list(N, ssl:groups(), []). +%% +group_list(0, _Pool, Acc) -> + Acc; +group_list(N, Pool, Acc) -> + R = rand:uniform(length(Pool)), + G = lists:nth(R, Pool), + group_list(N - 1, Pool -- [G], [G|Acc]). diff --git a/lib/ssl/test/ssl.spec b/lib/ssl/test/ssl.spec index cb54168d36..15587abecd 100644 --- a/lib/ssl/test/ssl.spec +++ b/lib/ssl/test/ssl.spec @@ -1,4 +1,12 @@ -{suites,"../ssl_test",all}. -{skip_suites, "../ssl_test", - [ssl_bench_SUITE, ssl_dist_bench_SUITE], - "Benchmarks run separately"}. +% {merge_tests,false}. +{alias,dir,"../ssl_test"}. + +{suites,dir,all}. +{skip_groups,dir,ssl_bench_SUITE,setup,"Benchmarks run separately"}. +{skip_groups,dir,ssl_bench_SUITE,payload,"Benchmarks run separately"}. +{skip_groups,dir,ssl_bench_SUITE,pem_cache,"Benchmarks run separately"}. +{skip_groups,dir,ssl_dist_bench_SUITE,setup,"Benchmarks run separately"}. +{skip_groups,dir,ssl_dist_bench_SUITE,roundtrip,"Benchmarks run separately"}. +{skip_groups,dir,ssl_dist_bench_SUITE,throughput,"Benchmarks run separately"}. +{skip_groups,dir,ssl_dist_bench_SUITE,sched_utilization,"Benchmarks run separately"}. + diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index c64358960c..d02888793c 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -51,35 +51,7 @@ groups() -> ]. test_cases()-> - key_cert_combinations() - ++ misc() - ++ ecc_negotiation(). - -key_cert_combinations() -> - server_ecdh_rsa() ++ - server_ecdhe_rsa() ++ - server_ecdh_ecdsa() ++ - server_ecdhe_ecdsa(). - -server_ecdh_rsa() -> - [client_ecdh_rsa_server_ecdh_rsa, - client_ecdhe_rsa_server_ecdh_rsa, - client_ecdhe_ecdsa_server_ecdh_rsa]. - -server_ecdhe_rsa() -> - [client_ecdh_rsa_server_ecdhe_rsa, - client_ecdhe_rsa_server_ecdhe_rsa, - client_ecdhe_ecdsa_server_ecdhe_rsa]. - -server_ecdh_ecdsa() -> - [client_ecdh_ecdsa_server_ecdh_ecdsa, - client_ecdhe_rsa_server_ecdh_ecdsa, - client_ecdhe_ecdsa_server_ecdh_ecdsa]. - -server_ecdhe_ecdsa() -> - [client_ecdh_rsa_server_ecdhe_ecdsa, - client_ecdh_ecdsa_server_ecdhe_ecdsa, - client_ecdhe_ecdsa_server_ecdhe_ecdsa]. + misc() ++ ecc_negotiation(). misc()-> [client_ecdsa_server_ecdsa_with_raw_key]. @@ -160,35 +132,6 @@ end_per_testcase(_TestCase, Config) -> %% Test diffrent certificate chain types, note that it is the servers %% chain that affect what cipher suit that will be choosen -%% ECDH_RSA -client_ecdh_rsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdh_rsa(Config). -client_ecdhe_rsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdh_rsa(Config). -client_ecdhe_ecdsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdh_rsa(Config). -%% ECDHE_RSA -client_ecdh_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdhe_rsa(Config). -client_ecdhe_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdhe_rsa(Config). -client_ecdhe_ecdsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_rsa(Config). -%% ECDH_ECDSA -client_ecdh_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_ecdsa_server_ecdh_ecdsa(Config). -client_ecdhe_rsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdh_ecdsa(Config). -client_ecdhe_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdh_ecdsa(Config). -%% ECDHE_ECDSA -client_ecdh_rsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdhe_ecdsa(Config). -client_ecdh_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_ecdsa_server_ecdhe_ecdsa(Config). -client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config). - client_ecdsa_server_ecdsa_with_raw_key(Config) when is_list(Config) -> Default = ssl_test_lib:default_cert_chain_conf(), {COpts0, SOpts0} = ssl_test_lib:make_ec_cert_chains([{server_chain, Default}, diff --git a/lib/ssl/test/ssl_ECC_openssl_SUITE.erl b/lib/ssl/test/ssl_ECC_openssl_SUITE.erl index 81a7dfd2da..68d4e910fd 100644 --- a/lib/ssl/test/ssl_ECC_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_openssl_SUITE.erl @@ -33,77 +33,23 @@ %%-------------------------------------------------------------------- all() -> - case test_cases() of - [_|_] -> - all_groups(); - [] -> - [skip] - end. - -all_groups() -> case ssl_test_lib:openssl_sane_dtls() of true -> [{group, 'tlsv1.2'}, - {group, 'tlsv1.1'}, - {group, 'tlsv1'}, - {group, 'dtlsv1.2'}, - {group, 'dtlsv1'}]; + {group, 'dtlsv1.2'}]; false -> - [{group, 'tlsv1.2'}, - {group, 'tlsv1.1'}, - {group, 'tlsv1'}] + [{group, 'tlsv1.2'}] end. groups() -> case ssl_test_lib:openssl_sane_dtls() of true -> - [{'tlsv1.2', [], [mix_sign | test_cases()]}, - {'tlsv1.1', [], test_cases()}, - {'tlsv1', [], test_cases()}, - {'dtlsv1.2', [], [mix_sign | test_cases()]}, - {'dtlsv1', [], test_cases()}]; + [{'tlsv1.2', [], [mix_sign]}, + {'dtlsv1.2', [], [mix_sign]}]; false -> - [{'tlsv1.2', [], [mix_sign | test_cases()]}, - {'tlsv1.1', [], test_cases()}, - {'tlsv1', [], test_cases()}] + [{'tlsv1.2', [], [mix_sign]}] end. - -test_cases()-> - cert_combinations(). -cert_combinations() -> - lists:append(lists:map(fun({Name, Suites}) -> - case ssl_test_lib:openssl_filter(Name) of - [] -> - []; - [_|_] -> - Suites - end - end, [{"ECDH-ECDSA", server_ecdh_ecdsa()}, - {"ECDH-RSA", server_ecdh_rsa()}, - {"ECDHE-RSA", server_ecdhe_rsa()}, - {"ECDHE-ECDSA", server_ecdhe_ecdsa()} - ])). -server_ecdh_rsa() -> - [client_ecdh_rsa_server_ecdh_rsa, - client_ecdhe_rsa_server_ecdh_rsa, - client_ecdhe_ecdsa_server_ecdh_rsa]. - -server_ecdhe_rsa() -> - [client_ecdh_rsa_server_ecdhe_rsa, - client_ecdhe_rsa_server_ecdhe_rsa, - client_ecdhe_ecdsa_server_ecdhe_rsa]. - -server_ecdh_ecdsa() -> - [client_ecdh_ecdsa_server_ecdh_ecdsa, - client_ecdhe_rsa_server_ecdh_ecdsa, - client_ecdhe_ecdsa_server_ecdh_ecdsa]. - -server_ecdhe_ecdsa() -> - [client_ecdh_rsa_server_ecdhe_ecdsa, - client_ecdh_ecdsa_server_ecdhe_ecdsa, - client_ecdhe_ecdsa_server_ecdhe_ecdsa]. - %%-------------------------------------------------------------------- init_per_suite(Config0) -> end_per_suite(Config0), @@ -171,38 +117,6 @@ end_per_testcase(_TestCase, Config) -> skip(Config) when is_list(Config) -> {skip, openssl_does_not_support_ECC}. -%% Test diffrent certificate chain types, note that it is the servers -%% chain that affect what cipher suit that will be choosen - -%% ECDH_RSA -client_ecdh_rsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdh_rsa(Config). -client_ecdhe_rsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdh_rsa(Config). -client_ecdhe_ecdsa_server_ecdh_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdh_rsa(Config). -%% ECDHE_RSA -client_ecdh_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdhe_rsa(Config). -client_ecdhe_rsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdhe_rsa(Config). -client_ecdhe_ecdsa_server_ecdhe_rsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_rsa(Config). -%% ECDH_ECDSA -client_ecdh_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_ecdsa_server_ecdh_ecdsa(Config). -client_ecdhe_rsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_rsa_server_ecdh_ecdsa(Config). -client_ecdhe_ecdsa_server_ecdh_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdh_ecdsa(Config). -%% ECDHE_ECDSA -client_ecdh_rsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_rsa_server_ecdhe_ecdsa(Config). -client_ecdh_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdh_ecdsa_server_ecdhe_ecdsa(Config). -client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config) when is_list(Config) -> - ssl_ECC:client_ecdhe_ecdsa_server_ecdhe_ecdsa(Config). - mix_sign(Config) -> {COpts0, SOpts0} = ssl_test_lib:make_mix_cert(Config), COpts = ssl_test_lib:ssl_options(COpts0, Config), diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 4452adaea7..20d9f28512 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -29,6 +29,7 @@ -include_lib("public_key/include/public_key.hrl"). -include("ssl_api.hrl"). +-include("ssl_cipher.hrl"). -include("ssl_internal.hrl"). -include("ssl_alert.hrl"). -include("ssl_internal.hrl"). @@ -53,7 +54,8 @@ all() -> {group, options_tls}, {group, session}, {group, 'dtlsv1.2'}, - {group, 'dtlsv1'}, + {group, 'dtlsv1'}, + {group, 'tlsv1.3'}, {group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, @@ -67,17 +69,16 @@ groups() -> {options_tls, [], options_tests_tls()}, {'dtlsv1.2', [], all_versions_groups()}, {'dtlsv1', [], all_versions_groups()}, + {'tlsv1.3', [], tls13_test_group()}, {'tlsv1.2', [], all_versions_groups() ++ tls_versions_groups() ++ [conf_signature_algs, no_common_signature_algs]}, {'tlsv1.1', [], all_versions_groups() ++ tls_versions_groups()}, {'tlsv1', [], all_versions_groups() ++ tls_versions_groups() ++ rizzo_tests()}, {'sslv3', [], all_versions_groups() ++ tls_versions_groups() ++ rizzo_tests() ++ [tls_ciphersuite_vs_version]}, {api,[], api_tests()}, {api_tls,[], api_tests_tls()}, - {tls_ciphers,[], tls_cipher_tests()}, {session, [], session_tests()}, {renegotiate, [], renegotiate_tests()}, {ciphers, [], cipher_tests()}, - {ciphers_ec, [], cipher_tests_ec()}, {error_handling_tests, [], error_handling_tests()}, {error_handling_tests_tls, [], error_handling_tests_tls()} ]. @@ -85,14 +86,12 @@ groups() -> tls_versions_groups ()-> [ {group, api_tls}, - {group, tls_ciphers}, {group, error_handling_tests_tls}]. all_versions_groups ()-> [{group, api}, {group, renegotiate}, {group, ciphers}, - {group, ciphers_ec}, {group, error_handling_tests}]. @@ -208,38 +207,11 @@ renegotiate_tests() -> renegotiate_dos_mitigate_passive, renegotiate_dos_mitigate_absolute]. -tls_cipher_tests() -> - [rc4_rsa_cipher_suites, - rc4_ecdh_rsa_cipher_suites, - rc4_ecdsa_cipher_suites]. - cipher_tests() -> [old_cipher_suites, - cipher_suites_mix, - ciphers_rsa_signed_certs, - ciphers_rsa_signed_certs_openssl_names, - ciphers_dsa_signed_certs, - ciphers_dsa_signed_certs_openssl_names, - chacha_rsa_cipher_suites, - chacha_ecdsa_cipher_suites, - anonymous_cipher_suites, - psk_cipher_suites, - psk_with_hint_cipher_suites, - psk_anon_cipher_suites, - psk_anon_with_hint_cipher_suites, - srp_cipher_suites, - srp_anon_cipher_suites, - srp_dsa_cipher_suites, - des_rsa_cipher_suites, - des_ecdh_rsa_cipher_suites, + cipher_suites_mix, default_reject_anonymous]. -cipher_tests_ec() -> - [ciphers_ecdsa_signed_certs, - ciphers_ecdsa_signed_certs_openssl_names, - ciphers_ecdh_rsa_signed_certs, - ciphers_ecdh_rsa_signed_certs_openssl_names]. - error_handling_tests()-> [close_transport_accept, recv_active, @@ -269,6 +241,26 @@ rizzo_tests() -> rizzo_zero_n, rizzo_disabled]. +%% For testing TLS 1.3 features and possible regressions +tls13_test_group() -> + [tls13_enable_client_side, + tls13_enable_server_side, + tls_record_1_3_encode_decode, + tls13_finished_verify_data, + tls13_1_RTT_handshake, + tls13_basic_ssl_server_openssl_client, + tls13_custom_groups_ssl_server_openssl_client, + tls13_hello_retry_request_ssl_server_openssl_client, + tls13_client_auth_empty_cert_alert_ssl_server_openssl_client, + tls13_client_auth_empty_cert_ssl_server_openssl_client, + tls13_client_auth_ssl_server_openssl_client, + tls13_hrr_client_auth_empty_cert_alert_ssl_server_openssl_client, + tls13_hrr_client_auth_empty_cert_ssl_server_openssl_client, + tls13_hrr_client_auth_ssl_server_openssl_client, + tls13_unsupported_sign_algo_client_auth_ssl_server_openssl_client, + tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client, + tls13_connection_information]. + %%-------------------------------------------------------------------- init_per_suite(Config0) -> catch crypto:stop(), @@ -298,7 +290,8 @@ init_per_group(GroupName, Config) when GroupName == basic_tls; GroupName == options; GroupName == basic; GroupName == session; - GroupName == error_handling_tests_tls + GroupName == error_handling_tests_tls; + GroupName == tls13_test_group -> ssl_test_lib:clean_tls_version(Config); init_per_group(GroupName, Config) -> @@ -386,26 +379,7 @@ init_per_testcase(TestCase, Config) when TestCase == client_renegotiate; ct:timetrap({seconds, ?SEC_RENEGOTIATION_TIMEOUT + 5}), Config; -init_per_testcase(TestCase, Config) when TestCase == psk_cipher_suites; - TestCase == psk_with_hint_cipher_suites; - TestCase == ciphers_rsa_signed_certs; - TestCase == ciphers_rsa_signed_certs_openssl_names; - TestCase == ciphers_ecdh_rsa_signed_certs_openssl_names; - TestCase == ciphers_ecdh_rsa_signed_certs; - TestCase == ciphers_dsa_signed_certs; - TestCase == ciphers_dsa_signed_certs_openssl_names; - TestCase == anonymous_cipher_suites; - TestCase == ciphers_ecdsa_signed_certs; - TestCase == ciphers_ecdsa_signed_certs_openssl_names; - TestCase == anonymous_cipher_suites; - TestCase == psk_anon_cipher_suites; - TestCase == psk_anon_with_hint_cipher_suites; - TestCase == srp_cipher_suites; - TestCase == srp_anon_cipher_suites; - TestCase == srp_dsa_cipher_suites; - TestCase == des_rsa_cipher_suites; - TestCase == des_ecdh_rsa_cipher_suites; - TestCase == versions_option; +init_per_testcase(TestCase, Config) when TestCase == versions_option; TestCase == tls_tcp_connect_big -> ssl_test_lib:ct_log_supported_protocol_versions(Config), ct:timetrap({seconds, 60}), @@ -2682,144 +2656,6 @@ tls_shutdown_error(Config) when is_list(Config) -> ok = ssl:close(Listen), {error, closed} = ssl:shutdown(Listen, read_write). -%%------------------------------------------------------------------- -ciphers_rsa_signed_certs() -> - [{doc,"Test all rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_rsa_signed_certs(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:rsa_suites(crypto), - run_suites(Ciphers, Config, rsa). -%%------------------------------------------------------------------- -ciphers_rsa_signed_certs_openssl_names() -> - [{doc,"Test all rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_rsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_rsa_suites(), - run_suites(Ciphers, Config, rsa). - -%%------------------------------------------------------------------- -ciphers_dsa_signed_certs() -> - [{doc,"Test all dsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_dsa_signed_certs(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:dsa_suites(NVersion), - run_suites(Ciphers, Config, dsa). -%%------------------------------------------------------------------- -ciphers_dsa_signed_certs_openssl_names() -> - [{doc,"Test all dsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_dsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_dsa_suites(), - run_suites(Ciphers, Config, dsa). - -%%------------------------------------------------------------------- -chacha_rsa_cipher_suites()-> - [{doc,"Test the cacha with ECDSA signed certs ciphersuites"}]. -chacha_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {KeyEx,_,_} = S <- ssl_test_lib:chacha_suites(NVersion), - KeyEx == ecdhe_rsa, KeyEx == dhe_rsa], - run_suites(Ciphers, Config, chacha_ecdsa). - -%%------------------------------------------------------------------- -chacha_ecdsa_cipher_suites()-> - [{doc,"Test the cacha with ECDSA signed certs ciphersuites"}]. -chacha_ecdsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {ecdhe_ecdsa,_,_} = S <- ssl_test_lib:chacha_suites(NVersion)], - run_suites(Ciphers, Config, chacha_rsa). -%%----------------------------------------------------------------- -anonymous_cipher_suites()-> - [{doc,"Test the anonymous ciphersuites"}]. -anonymous_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:ecdh_dh_anonymous_suites(NVersion), - run_suites(Ciphers, Config, anonymous). -%%------------------------------------------------------------------- -psk_cipher_suites() -> - [{doc, "Test the PSK ciphersuites WITHOUT server supplied identity hint"}]. -psk_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_suites(NVersion), - run_suites(Ciphers, Config, psk). -%%------------------------------------------------------------------- -psk_with_hint_cipher_suites()-> - [{doc, "Test the PSK ciphersuites WITH server supplied identity hint"}]. -psk_with_hint_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_suites(NVersion), - run_suites(Ciphers, Config, psk_with_hint). -%%------------------------------------------------------------------- -psk_anon_cipher_suites() -> - [{doc, "Test the anonymous PSK ciphersuites WITHOUT server supplied identity hint"}]. -psk_anon_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_anon_suites(NVersion), - run_suites(Ciphers, Config, psk_anon). -%%------------------------------------------------------------------- -psk_anon_with_hint_cipher_suites()-> - [{doc, "Test the anonymous PSK ciphersuites WITH server supplied identity hint"}]. -psk_anon_with_hint_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_anon_suites(NVersion), - run_suites(Ciphers, Config, psk_anon_with_hint). -%%------------------------------------------------------------------- -srp_cipher_suites()-> - [{doc, "Test the SRP ciphersuites"}]. -srp_cipher_suites(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:srp_suites(), - run_suites(Ciphers, Config, srp). -%%------------------------------------------------------------------- -srp_anon_cipher_suites()-> - [{doc, "Test the anonymous SRP ciphersuites"}]. -srp_anon_cipher_suites(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:srp_anon_suites(), - run_suites(Ciphers, Config, srp_anon). -%%------------------------------------------------------------------- -srp_dsa_cipher_suites()-> - [{doc, "Test the SRP DSA ciphersuites"}]. -srp_dsa_cipher_suites(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:srp_dss_suites(), - run_suites(Ciphers, Config, srp_dsa). -%%------------------------------------------------------------------- -rc4_rsa_cipher_suites()-> - [{doc, "Test the RC4 ciphersuites"}]. -rc4_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {rsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], - run_suites(Ciphers, Config, rc4_rsa). -%------------------------------------------------------------------- -rc4_ecdh_rsa_cipher_suites()-> - [{doc, "Test the RC4 ciphersuites"}]. -rc4_ecdh_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {ecdh_rsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], - run_suites(Ciphers, Config, rc4_ecdh_rsa). - -%%------------------------------------------------------------------- -rc4_ecdsa_cipher_suites()-> - [{doc, "Test the RC4 ciphersuites"}]. -rc4_ecdsa_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Ciphers = [S || {ecdhe_ecdsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], - run_suites(Ciphers, Config, rc4_ecdsa). - -%%------------------------------------------------------------------- -des_rsa_cipher_suites()-> - [{doc, "Test the des_rsa ciphersuites"}]. -des_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Ciphers = [S || {rsa,_,_} = S <- ssl_test_lib:des_suites(NVersion)], - run_suites(Ciphers, Config, des_rsa). -%------------------------------------------------------------------- -des_ecdh_rsa_cipher_suites()-> - [{doc, "Test ECDH rsa signed ciphersuites"}]. -des_ecdh_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {dhe_rsa,_,_} = S <- ssl_test_lib:des_suites(NVersion)], - run_suites(Ciphers, Config, des_dhe_rsa). - %%-------------------------------------------------------------------- default_reject_anonymous()-> [{doc,"Test that by default anonymous cipher suites are rejected "}]. @@ -2846,36 +2682,6 @@ default_reject_anonymous(Config) when is_list(Config) -> ssl_test_lib:check_server_alert(Server, Client, insufficient_security). %%-------------------------------------------------------------------- -ciphers_ecdsa_signed_certs() -> - [{doc, "Test all ecdsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdsa_signed_certs(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:ecdsa_suites(NVersion), - run_suites(Ciphers, Config, ecdsa). -%%-------------------------------------------------------------------- -ciphers_ecdsa_signed_certs_openssl_names() -> - [{doc, "Test all ecdsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_ecdsa_suites(), - run_suites(Ciphers, Config, ecdsa). -%%-------------------------------------------------------------------- -ciphers_ecdh_rsa_signed_certs() -> - [{doc, "Test all ecdh_rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdh_rsa_signed_certs(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:ecdh_rsa_suites(NVersion), - run_suites(Ciphers, Config, ecdh_rsa). -%%-------------------------------------------------------------------- -ciphers_ecdh_rsa_signed_certs_openssl_names() -> - [{doc, "Test all ecdh_rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdh_rsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_ecdh_rsa_suites(), - run_suites(Ciphers, Config, ecdh_rsa). -%%-------------------------------------------------------------------- reuse_session() -> [{doc,"Test reuse of sessions (short handshake)"}]. reuse_session(Config) when is_list(Config) -> @@ -2966,8 +2772,8 @@ make_sure_expired(Host, Port, Id) -> server_does_not_want_to_reuse_session() -> [{doc,"Test reuse of sessions (short handshake)"}]. server_does_not_want_to_reuse_session(Config) when is_list(Config) -> - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = @@ -3437,9 +3243,9 @@ defaults(Config) when is_list(Config)-> true = lists:member(sslv3, proplists:get_value(available, Versions)), false = lists:member(sslv3, proplists:get_value(supported, Versions)), true = lists:member('tlsv1', proplists:get_value(available, Versions)), - true = lists:member('tlsv1', proplists:get_value(supported, Versions)), + false = lists:member('tlsv1', proplists:get_value(supported, Versions)), true = lists:member('tlsv1.1', proplists:get_value(available, Versions)), - true = lists:member('tlsv1.1', proplists:get_value(supported, Versions)), + false = lists:member('tlsv1.1', proplists:get_value(supported, Versions)), true = lists:member('tlsv1.2', proplists:get_value(available, Versions)), true = lists:member('tlsv1.2', proplists:get_value(supported, Versions)), false = lists:member({rsa,rc4_128,sha}, ssl:cipher_suites()), @@ -3451,7 +3257,7 @@ defaults(Config) when is_list(Config)-> true = lists:member('dtlsv1.2', proplists:get_value(available_dtls, Versions)), true = lists:member('dtlsv1', proplists:get_value(available_dtls, Versions)), true = lists:member('dtlsv1.2', proplists:get_value(supported_dtls, Versions)), - true = lists:member('dtlsv1', proplists:get_value(supported_dtls, Versions)). + false = lists:member('dtlsv1', proplists:get_value(supported_dtls, Versions)). %%-------------------------------------------------------------------- reuseaddr() -> @@ -3612,7 +3418,7 @@ honor_cipher_order(Config, Honor, ServerCiphers, ClientCiphers, Expected) -> %%-------------------------------------------------------------------- tls_ciphersuite_vs_version() -> - [{doc,"Test a SSLv3 client can not negotiate a TLSv* cipher suite."}]. + [{doc,"Test a SSLv3 client cannot negotiate a TLSv* cipher suite."}]. tls_ciphersuite_vs_version(Config) when is_list(Config) -> {_ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), @@ -3655,14 +3461,14 @@ conf_signature_algs(Config) when is_list(Config) -> ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, send_recv_result, []}}, - {options, [{active, false}, {signature_algs, [{sha256, rsa}]} | ServerOpts]}]), + {options, [{active, false}, {signature_algs, [{sha, rsa}]} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, send_recv_result, []}}, - {options, [{active, false}, {signature_algs, [{sha256, rsa}]} | ClientOpts]}]), + {options, [{active, false}, {signature_algs, [{sha, rsa}]} | ClientOpts]}]), ct:log("Testcase ~p, Client ~p Server ~p ~n", [self(), Client, Server]), @@ -4559,6 +4365,1292 @@ accept_pool(Config) when is_list(Config) -> ssl_test_lib:close(Client1), ssl_test_lib:close(Client2). +%%-------------------------------------------------------------------- +%% TLS 1.3 +%%-------------------------------------------------------------------- + +tls13_enable_client_side() -> + [{doc,"Test that a TLS 1.3 client can connect to a TLS 1.2 server."}]. + +tls13_enable_client_side(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, protocol_info_result, []}}, + {options, [{versions, + ['tlsv1.1', 'tlsv1.2']} | ServerOpts] }]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, protocol_info_result, []}}, + {options, [{versions, + ['tlsv1.2', 'tlsv1.3']} | ClientOpts]}]), + + ServerMsg = ClientMsg = {ok, 'tlsv1.2'}, + ssl_test_lib:check_result(Server, ServerMsg, Client, ClientMsg). + +tls13_enable_server_side() -> + [{doc,"Test that a TLS 1.2 client can connect to a TLS 1.3 server."}]. + +tls13_enable_server_side(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, protocol_info_result, []}}, + {options, [{versions, + ['tlsv1.2', 'tlsv1.3']} | ServerOpts] }]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, protocol_info_result, []}}, + {options, [{versions, + ['tlsv1.2', 'tlsv1.1']} | ClientOpts]}]), + + ServerMsg = ClientMsg = {ok, 'tlsv1.2'}, + ssl_test_lib:check_result(Server, ServerMsg, Client, ClientMsg). + +tls_record_1_3_encode_decode() -> + [{doc,"Test TLS 1.3 record encode/decode functions"}]. + +tls_record_1_3_encode_decode(_Config) -> + ConnectionStates = + #{current_read => + #{beast_mitigation => one_n_minus_one, + cipher_state => + {cipher_state, + <<14,172,111,243,199,170,242,203,126,205,34,93,122,115,226,14, + 15,117,155,48,24,112,61,15,113,208,127,51,179,227,194,232>>, + <<197,54,168,218,54,91,157,58,30,201,197,142,51,58,53,231,228, + 131,57,122,170,78,82,196,30,48,23,16,95,255,185,236>>, + undefined,undefined,undefined,16}, + client_verify_data => undefined,compression_state => undefined, + mac_secret => undefined,secure_renegotiation => undefined, + security_parameters => + {security_parameters, + <<19,2>>, + 0,8,2,undefined,undefined,undefined,undefined,undefined, + sha384,undefined,undefined, + {handshake_secret, + <<128,229,186,211,62,127,182,20,62,166,233,23,135,64,121, + 3,104,251,214,161,253,31,3,2,232,37,8,221,189,72,64,218, + 121,41,112,148,254,34,68,164,228,60,161,201,132,55,56, + 157>>}, + undefined, + <<92,24,205,75,244,60,136,212,250,32,214,20,37,3,213,87,61,207, + 147,61,168,145,177,118,160,153,33,53,48,108,191,174>>, + undefined}, + sequence_number => 0,server_verify_data => undefined}, + current_write => + #{beast_mitigation => one_n_minus_one, + cipher_state => + {cipher_state, + <<14,172,111,243,199,170,242,203,126,205,34,93,122,115,226,14, + 15,117,155,48,24,112,61,15,113,208,127,51,179,227,194,232>>, + <<197,54,168,218,54,91,157,58,30,201,197,142,51,58,53,231,228, + 131,57,122,170,78,82,196,30,48,23,16,95,255,185,236>>, + undefined,undefined,undefined,16}, + client_verify_data => undefined,compression_state => undefined, + mac_secret => undefined,secure_renegotiation => undefined, + security_parameters => + {security_parameters, + <<19,2>>, + 0,8,2,undefined,undefined,undefined,undefined,undefined, + sha384,undefined,undefined, + {handshake_secret, + <<128,229,186,211,62,127,182,20,62,166,233,23,135,64,121, + 3,104,251,214,161,253,31,3,2,232,37,8,221,189,72,64,218, + 121,41,112,148,254,34,68,164,228,60,161,201,132,55,56, + 157>>}, + undefined, + <<92,24,205,75,244,60,136,212,250,32,214,20,37,3,213,87,61,207, + 147,61,168,145,177,118,160,153,33,53,48,108,191,174>>, + undefined}, + sequence_number => 0,server_verify_data => undefined}}, + + PlainText = [11, + <<0,2,175>>, + <<0,0,2,171,0,2,166,48,130,2,162,48,130,1,138,2,9,0,186,57,220,137,88,255, + 191,235,48,13,6,9,42,134,72,134,247,13,1,1,11,5,0,48,18,49,16,48,14,6,3,85, + 4,3,12,7,84,101,115,116,32,67,65,48,30,23,13,49,56,48,53,48,52,49,52,49,50, + 51,56,90,23,13,50,56,48,50,48,52,49,52,49,50,51,56,90,48,20,49,18,48,16,6, + 3,85,4,3,12,9,108,111,99,97,108,104,111,115,116,48,130,1,34,48,13,6,9,42, + 134,72,134,247,13,1,1,1,5,0,3,130,1,15,0,48,130,1,10,2,130,1,1,0,169,40, + 144,176,121,63,134,97,144,126,243,183,225,157,37,131,183,225,87,243,23,88, + 230,70,9,134,32,147,7,27,167,98,51,81,224,75,199,12,229,251,195,207,75,179, + 181,78,128,3,255,44,58,39,43,172,142,45,186,58,51,65,187,199,154,153,245, + 70,133,137,1,27,87,42,116,65,251,129,109,145,233,97,171,71,54,213,185,74, + 209,166,11,218,189,119,206,86,170,60,212,213,85,189,30,50,215,23,185,53, + 132,238,132,176,198,250,139,251,198,221,225,128,109,113,23,220,39,143,71, + 30,59,189,51,244,61,158,214,146,180,196,103,169,189,221,136,78,129,216,148, + 2,9,8,65,37,224,215,233,13,209,21,235,20,143,33,74,59,53,208,90,152,94,251, + 54,114,171,39,88,230,227,158,211,135,37,182,67,205,161,59,20,138,58,253,15, + 53,48,8,157,9,95,197,9,177,116,21,54,9,125,78,109,182,83,20,16,234,223,116, + 41,155,123,87,77,17,120,153,246,239,124,130,105,219,166,146,242,151,66,198, + 75,72,63,28,246,86,16,244,223,22,36,50,15,247,222,98,6,152,136,154,72,150, + 73,127,2,3,1,0,1,48,13,6,9,42,134,72,134,247,13,1,1,11,5,0,3,130,1,1,0,76, + 33,54,160,229,219,219,193,150,116,245,252,18,39,235,145,86,12,167,171,52, + 117,166,30,83,5,216,245,177,217,247,95,1,136,94,246,212,108,248,230,111, + 225,202,189,6,129,8,70,128,245,18,204,215,87,82,129,253,227,122,66,182,184, + 189,30,193,169,144,218,216,109,105,110,215,144,60,104,162,178,101,164,218, + 122,60,37,41,143,57,150,52,59,51,112,238,113,239,168,114,69,183,143,154,73, + 61,58,80,247,172,95,251,55,28,186,28,200,206,230,118,243,92,202,189,49,76, + 124,252,76,0,247,112,85,194,69,59,222,163,228,103,49,110,104,109,251,155, + 138,9,37,167,49,189,48,134,52,158,185,129,24,96,153,196,251,90,206,76,239, + 175,119,174,165,133,108,222,125,237,125,187,149,152,83,190,16,202,94,202, + 201,40,218,22,254,63,189,41,174,97,140,203,70,18,196,118,237,175,134,79,78, + 246,2,61,54,77,186,112,32,17,193,192,188,217,252,215,200,7,245,180,179,132, + 183,212,229,155,15,152,206,135,56,81,88,3,123,244,149,110,182,72,109,70,62, + 146,152,146,151,107,126,216,210,9,93,0,0>>], + + {[_Header|Encoded], _} = tls_record_1_3:encode_plain_text(22, PlainText, ConnectionStates), + CipherText = #ssl_tls{type = 23, version = {3,3}, fragment = Encoded}, + + {#ssl_tls{type = 22, version = {3,4}, fragment = DecodedText}, _} = + tls_record_1_3:decode_cipher_text(CipherText, ConnectionStates), + + DecodedText = iolist_to_binary(PlainText), + ct:log("Decoded: ~p ~n", [DecodedText]), + ok. + +tls13_1_RTT_handshake() -> + [{doc,"Test TLS 1.3 1-RTT Handshake"}]. + +tls13_1_RTT_handshake(_Config) -> + %% ConnectionStates with NULL cipher + ConnStatesNull = + #{current_write => + #{security_parameters => + #security_parameters{cipher_suite = ?TLS_NULL_WITH_NULL_NULL}, + sequence_number => 0 + } + }, + + %% {client} construct a ClientHello handshake message: + %% + %% ClientHello (196 octets): 01 00 00 c0 03 03 cb 34 ec b1 e7 81 63 + %% ba 1c 38 c6 da cb 19 6a 6d ff a2 1a 8d 99 12 ec 18 a2 ef 62 83 + %% 02 4d ec e7 00 00 06 13 01 13 03 13 02 01 00 00 91 00 00 00 0b + %% 00 09 00 00 06 73 65 72 76 65 72 ff 01 00 01 00 00 0a 00 14 00 + %% 12 00 1d 00 17 00 18 00 19 01 00 01 01 01 02 01 03 01 04 00 23 + %% 00 00 00 33 00 26 00 24 00 1d 00 20 99 38 1d e5 60 e4 bd 43 d2 + %% 3d 8e 43 5a 7d ba fe b3 c0 6e 51 c1 3c ae 4d 54 13 69 1e 52 9a + %% af 2c 00 2b 00 03 02 03 04 00 0d 00 20 00 1e 04 03 05 03 06 03 + %% 02 03 08 04 08 05 08 06 04 01 05 01 06 01 02 01 04 02 05 02 06 + %% 02 02 02 00 2d 00 02 01 01 00 1c 00 02 40 01 + %% + %% {client} send handshake record: + %% + %% payload (196 octets): 01 00 00 c0 03 03 cb 34 ec b1 e7 81 63 ba + %% 1c 38 c6 da cb 19 6a 6d ff a2 1a 8d 99 12 ec 18 a2 ef 62 83 02 + %% 4d ec e7 00 00 06 13 01 13 03 13 02 01 00 00 91 00 00 00 0b 00 + %% 09 00 00 06 73 65 72 76 65 72 ff 01 00 01 00 00 0a 00 14 00 12 + %% 00 1d 00 17 00 18 00 19 01 00 01 01 01 02 01 03 01 04 00 23 00 + %% 00 00 33 00 26 00 24 00 1d 00 20 99 38 1d e5 60 e4 bd 43 d2 3d + %% 8e 43 5a 7d ba fe b3 c0 6e 51 c1 3c ae 4d 54 13 69 1e 52 9a af + %% 2c 00 2b 00 03 02 03 04 00 0d 00 20 00 1e 04 03 05 03 06 03 02 + %% 03 08 04 08 05 08 06 04 01 05 01 06 01 02 01 04 02 05 02 06 02 + %% 02 02 00 2d 00 02 01 01 00 1c 00 02 40 01 + %% + %% complete record (201 octets): 16 03 01 00 c4 01 00 00 c0 03 03 cb + %% 34 ec b1 e7 81 63 ba 1c 38 c6 da cb 19 6a 6d ff a2 1a 8d 99 12 + %% ec 18 a2 ef 62 83 02 4d ec e7 00 00 06 13 01 13 03 13 02 01 00 + %% 00 91 00 00 00 0b 00 09 00 00 06 73 65 72 76 65 72 ff 01 00 01 + %% 00 00 0a 00 14 00 12 00 1d 00 17 00 18 00 19 01 00 01 01 01 02 + %% 01 03 01 04 00 23 00 00 00 33 00 26 00 24 00 1d 00 20 99 38 1d + %% e5 60 e4 bd 43 d2 3d 8e 43 5a 7d ba fe b3 c0 6e 51 c1 3c ae 4d + %% 54 13 69 1e 52 9a af 2c 00 2b 00 03 02 03 04 00 0d 00 20 00 1e + %% 04 03 05 03 06 03 02 03 08 04 08 05 08 06 04 01 05 01 06 01 02 + %% 01 04 02 05 02 06 02 02 02 00 2d 00 02 01 01 00 1c 00 02 40 01 + ClientHello = + hexstr2bin("01 00 00 c0 03 03 cb 34 ec b1 e7 81 63 + ba 1c 38 c6 da cb 19 6a 6d ff a2 1a 8d 99 12 ec 18 a2 ef 62 83 + 02 4d ec e7 00 00 06 13 01 13 03 13 02 01 00 00 91 00 00 00 0b + 00 09 00 00 06 73 65 72 76 65 72 ff 01 00 01 00 00 0a 00 14 00 + 12 00 1d 00 17 00 18 00 19 01 00 01 01 01 02 01 03 01 04 00 23 + 00 00 00 33 00 26 00 24 00 1d 00 20 99 38 1d e5 60 e4 bd 43 d2 + 3d 8e 43 5a 7d ba fe b3 c0 6e 51 c1 3c ae 4d 54 13 69 1e 52 9a + af 2c 00 2b 00 03 02 03 04 00 0d 00 20 00 1e 04 03 05 03 06 03 + 02 03 08 04 08 05 08 06 04 01 05 01 06 01 02 01 04 02 05 02 06 + 02 02 02 00 2d 00 02 01 01 00 1c 00 02 40 01"), + + ClientHelloRecord = + %% Current implementation always sets + %% legacy_record_version to Ox0303 + hexstr2bin("16 03 03 00 c4 01 00 00 c0 03 03 cb + 34 ec b1 e7 81 63 ba 1c 38 c6 da cb 19 6a 6d ff a2 1a 8d 99 12 + ec 18 a2 ef 62 83 02 4d ec e7 00 00 06 13 01 13 03 13 02 01 00 + 00 91 00 00 00 0b 00 09 00 00 06 73 65 72 76 65 72 ff 01 00 01 + 00 00 0a 00 14 00 12 00 1d 00 17 00 18 00 19 01 00 01 01 01 02 + 01 03 01 04 00 23 00 00 00 33 00 26 00 24 00 1d 00 20 99 38 1d + e5 60 e4 bd 43 d2 3d 8e 43 5a 7d ba fe b3 c0 6e 51 c1 3c ae 4d + 54 13 69 1e 52 9a af 2c 00 2b 00 03 02 03 04 00 0d 00 20 00 1e + 04 03 05 03 06 03 02 03 08 04 08 05 08 06 04 01 05 01 06 01 02 + 01 04 02 05 02 06 02 02 02 00 2d 00 02 01 01 00 1c 00 02 40 01"), + + {CHEncrypted, _} = + tls_record:encode_handshake(ClientHello, {3,4}, ConnStatesNull), + ClientHelloRecord = iolist_to_binary(CHEncrypted), + + %% {server} extract secret "early": + %% + %% salt: 0 (all zero octets) + %% + %% IKM (32 octets): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + %% 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + %% + %% secret (32 octets): 33 ad 0a 1c 60 7e c0 3b 09 e6 cd 98 93 68 0c + %% e2 10 ad f3 00 aa 1f 26 60 e1 b2 2e 10 f1 70 f9 2a + HKDFAlgo = sha256, + Salt = binary:copy(<<?BYTE(0)>>, 32), + IKM = binary:copy(<<?BYTE(0)>>, 32), + EarlySecret = + hexstr2bin("33 ad 0a 1c 60 7e c0 3b 09 e6 cd 98 93 68 0c + e2 10 ad f3 00 aa 1f 26 60 e1 b2 2e 10 f1 70 f9 2a"), + + {early_secret, EarlySecret} = tls_v1:key_schedule(early_secret, HKDFAlgo, {psk, Salt}), + + %% {client} create an ephemeral x25519 key pair: + %% + %% private key (32 octets): 49 af 42 ba 7f 79 94 85 2d 71 3e f2 78 + %% 4b cb ca a7 91 1d e2 6a dc 56 42 cb 63 45 40 e7 ea 50 05 + %% + %% public key (32 octets): 99 38 1d e5 60 e4 bd 43 d2 3d 8e 43 5a 7d + %% ba fe b3 c0 6e 51 c1 3c ae 4d 54 13 69 1e 52 9a af 2c + CPublicKey = + hexstr2bin("99 38 1d e5 60 e4 bd 43 d2 3d 8e 43 5a 7d + ba fe b3 c0 6e 51 c1 3c ae 4d 54 13 69 1e 52 9a af 2c"), + + %% {server} create an ephemeral x25519 key pair: + %% + %% private key (32 octets): b1 58 0e ea df 6d d5 89 b8 ef 4f 2d 56 + %% 52 57 8c c8 10 e9 98 01 91 ec 8d 05 83 08 ce a2 16 a2 1e + %% + %% public key (32 octets): c9 82 88 76 11 20 95 fe 66 76 2b db f7 c6 + %% 72 e1 56 d6 cc 25 3b 83 3d f1 dd 69 b1 b0 4e 75 1f 0f + SPrivateKey = + hexstr2bin("b1 58 0e ea df 6d d5 89 b8 ef 4f 2d 56 + 52 57 8c c8 10 e9 98 01 91 ec 8d 05 83 08 ce a2 16 a2 1e"), + + SPublicKey = + hexstr2bin("c9 82 88 76 11 20 95 fe 66 76 2b db f7 c6 + 72 e1 56 d6 cc 25 3b 83 3d f1 dd 69 b1 b0 4e 75 1f 0f"), + + %% {server} construct a ServerHello handshake message: + %% + %% ServerHello (90 octets): 02 00 00 56 03 03 a6 af 06 a4 12 18 60 + %% dc 5e 6e 60 24 9c d3 4c 95 93 0c 8a c5 cb 14 34 da c1 55 77 2e + %% d3 e2 69 28 00 13 01 00 00 2e 00 33 00 24 00 1d 00 20 c9 82 88 + %% 76 11 20 95 fe 66 76 2b db f7 c6 72 e1 56 d6 cc 25 3b 83 3d f1 + %% dd 69 b1 b0 4e 75 1f 0f 00 2b 00 02 03 04 + ServerHello = + hexstr2bin("02 00 00 56 03 03 a6 af 06 a4 12 18 60 + dc 5e 6e 60 24 9c d3 4c 95 93 0c 8a c5 cb 14 34 da c1 55 77 2e + d3 e2 69 28 00 13 01 00 00 2e 00 33 00 24 00 1d 00 20 c9 82 88 + 76 11 20 95 fe 66 76 2b db f7 c6 72 e1 56 d6 cc 25 3b 83 3d f1 + dd 69 b1 b0 4e 75 1f 0f 00 2b 00 02 03 04"), + + %% {server} derive secret for handshake "tls13 derived": + %% + %% PRK (32 octets): 33 ad 0a 1c 60 7e c0 3b 09 e6 cd 98 93 68 0c e2 + %% 10 ad f3 00 aa 1f 26 60 e1 b2 2e 10 f1 70 f9 2a + %% + %% hash (32 octets): e3 b0 c4 42 98 fc 1c 14 9a fb f4 c8 99 6f b9 24 + %% 27 ae 41 e4 64 9b 93 4c a4 95 99 1b 78 52 b8 55 + %% + %% info (49 octets): 00 20 0d 74 6c 73 31 33 20 64 65 72 69 76 65 64 + %% 20 e3 b0 c4 42 98 fc 1c 14 9a fb f4 c8 99 6f b9 24 27 ae 41 e4 + %% 64 9b 93 4c a4 95 99 1b 78 52 b8 55 + %% + %% expanded (32 octets): 6f 26 15 a1 08 c7 02 c5 67 8f 54 fc 9d ba + %% b6 97 16 c0 76 18 9c 48 25 0c eb ea c3 57 6c 36 11 ba + Hash = + hexstr2bin("e3 b0 c4 42 98 fc 1c 14 9a fb f4 c8 99 6f b9 24 + 27 ae 41 e4 64 9b 93 4c a4 95 99 1b 78 52 b8 55"), + + Hash = crypto:hash(HKDFAlgo, <<>>), + + Info = + hexstr2bin("00 20 0d 74 6c 73 31 33 20 64 65 72 69 76 65 64 + 20 e3 b0 c4 42 98 fc 1c 14 9a fb f4 c8 99 6f b9 24 27 ae 41 e4 + 64 9b 93 4c a4 95 99 1b 78 52 b8 55"), + + Info = tls_v1:create_info(<<"derived">>, Hash, ssl_cipher:hash_size(HKDFAlgo)), + + Expanded = + hexstr2bin("6f 26 15 a1 08 c7 02 c5 67 8f 54 fc 9d ba + b6 97 16 c0 76 18 9c 48 25 0c eb ea c3 57 6c 36 11 ba"), + + Expanded = tls_v1:derive_secret(EarlySecret, <<"derived">>, <<>>, HKDFAlgo), + + %% {server} extract secret "handshake": + %% + %% salt (32 octets): 6f 26 15 a1 08 c7 02 c5 67 8f 54 fc 9d ba b6 97 + %% 16 c0 76 18 9c 48 25 0c eb ea c3 57 6c 36 11 ba + %% + %% IKM (32 octets): 8b d4 05 4f b5 5b 9d 63 fd fb ac f9 f0 4b 9f 0d + %% 35 e6 d6 3f 53 75 63 ef d4 62 72 90 0f 89 49 2d + %% + %% secret (32 octets): 1d c8 26 e9 36 06 aa 6f dc 0a ad c1 2f 74 1b + %% 01 04 6a a6 b9 9f 69 1e d2 21 a9 f0 ca 04 3f be ac + + %% salt = Expanded + HandshakeIKM = + hexstr2bin("8b d4 05 4f b5 5b 9d 63 fd fb ac f9 f0 4b 9f 0d + 35 e6 d6 3f 53 75 63 ef d4 62 72 90 0f 89 49 2d"), + + HandshakeSecret = + hexstr2bin("1d c8 26 e9 36 06 aa 6f dc 0a ad c1 2f 74 1b + 01 04 6a a6 b9 9f 69 1e d2 21 a9 f0 ca 04 3f be ac"), + + HandshakeIKM = crypto:compute_key(ecdh, CPublicKey, SPrivateKey, x25519), + + {handshake_secret, HandshakeSecret} = + tls_v1:key_schedule(handshake_secret, HKDFAlgo, HandshakeIKM, + {early_secret, EarlySecret}), + + %% {server} derive secret "tls13 c hs traffic": + %% + %% PRK (32 octets): 1d c8 26 e9 36 06 aa 6f dc 0a ad c1 2f 74 1b 01 + %% 04 6a a6 b9 9f 69 1e d2 21 a9 f0 ca 04 3f be ac + %% + %% hash (32 octets): 86 0c 06 ed c0 78 58 ee 8e 78 f0 e7 42 8c 58 ed + %% d6 b4 3f 2c a3 e6 e9 5f 02 ed 06 3c f0 e1 ca d8 + %% + %% info (54 octets): 00 20 12 74 6c 73 31 33 20 63 20 68 73 20 74 72 + %% 61 66 66 69 63 20 86 0c 06 ed c0 78 58 ee 8e 78 f0 e7 42 8c 58 + %% ed d6 b4 3f 2c a3 e6 e9 5f 02 ed 06 3c f0 e1 ca d8 + %% + %% expanded (32 octets): b3 ed db 12 6e 06 7f 35 a7 80 b3 ab f4 5e + %% 2d 8f 3b 1a 95 07 38 f5 2e 96 00 74 6a 0e 27 a5 5a 21 + + %% PRK = HandshakeSecret + CHSTHash = + hexstr2bin("86 0c 06 ed c0 78 58 ee 8e 78 f0 e7 42 8c 58 ed + d6 b4 3f 2c a3 e6 e9 5f 02 ed 06 3c f0 e1 ca d8"), + + CHSTInfo = + hexstr2bin("00 20 12 74 6c 73 31 33 20 63 20 68 73 20 74 72 + 61 66 66 69 63 20 86 0c 06 ed c0 78 58 ee 8e 78 f0 e7 42 8c 58 + ed d6 b4 3f 2c a3 e6 e9 5f 02 ed 06 3c f0 e1 ca d8"), + + CHSTrafficSecret = + hexstr2bin(" b3 ed db 12 6e 06 7f 35 a7 80 b3 ab f4 5e + 2d 8f 3b 1a 95 07 38 f5 2e 96 00 74 6a 0e 27 a5 5a 21"), + + CHSH = <<ClientHello/binary,ServerHello/binary>>, + CHSTHash = crypto:hash(HKDFAlgo, CHSH), + CHSTInfo = tls_v1:create_info(<<"c hs traffic">>, CHSTHash, ssl_cipher:hash_size(HKDFAlgo)), + + CHSTrafficSecret = + tls_v1:client_handshake_traffic_secret(HKDFAlgo, {handshake_secret, HandshakeSecret}, CHSH), + + %% {server} derive secret "tls13 s hs traffic": + %% + %% PRK (32 octets): 1d c8 26 e9 36 06 aa 6f dc 0a ad c1 2f 74 1b 01 + %% 04 6a a6 b9 9f 69 1e d2 21 a9 f0 ca 04 3f be ac + %% + %% hash (32 octets): 86 0c 06 ed c0 78 58 ee 8e 78 f0 e7 42 8c 58 ed + %% d6 b4 3f 2c a3 e6 e9 5f 02 ed 06 3c f0 e1 ca d8 + %% + %% info (54 octets): 00 20 12 74 6c 73 31 33 20 73 20 68 73 20 74 72 + %% 61 66 66 69 63 20 86 0c 06 ed c0 78 58 ee 8e 78 f0 e7 42 8c 58 + %% ed d6 b4 3f 2c a3 e6 e9 5f 02 ed 06 3c f0 e1 ca d8 + %% + %% expanded (32 octets): b6 7b 7d 69 0c c1 6c 4e 75 e5 42 13 cb 2d + %% 37 b4 e9 c9 12 bc de d9 10 5d 42 be fd 59 d3 91 ad 38 + + %% PRK = HandshakeSecret + %% hash = CHSTHash + SHSTInfo = + hexstr2bin("00 20 12 74 6c 73 31 33 20 73 20 68 73 20 74 72 + 61 66 66 69 63 20 86 0c 06 ed c0 78 58 ee 8e 78 f0 e7 42 8c 58 + ed d6 b4 3f 2c a3 e6 e9 5f 02 ed 06 3c f0 e1 ca d8"), + + SHSTrafficSecret = + hexstr2bin("b6 7b 7d 69 0c c1 6c 4e 75 e5 42 13 cb 2d + 37 b4 e9 c9 12 bc de d9 10 5d 42 be fd 59 d3 91 ad 38"), + + SHSTInfo = tls_v1:create_info(<<"s hs traffic">>, CHSTHash, ssl_cipher:hash_size(HKDFAlgo)), + + SHSTrafficSecret = + tls_v1:server_handshake_traffic_secret(HKDFAlgo, {handshake_secret, HandshakeSecret}, CHSH), + + + %% {server} derive secret for master "tls13 derived": + %% + %% PRK (32 octets): 1d c8 26 e9 36 06 aa 6f dc 0a ad c1 2f 74 1b 01 + %% 04 6a a6 b9 9f 69 1e d2 21 a9 f0 ca 04 3f be ac + %% + %% hash (32 octets): e3 b0 c4 42 98 fc 1c 14 9a fb f4 c8 99 6f b9 24 + %% 27 ae 41 e4 64 9b 93 4c a4 95 99 1b 78 52 b8 55 + %% + %% info (49 octets): 00 20 0d 74 6c 73 31 33 20 64 65 72 69 76 65 64 + %% 20 e3 b0 c4 42 98 fc 1c 14 9a fb f4 c8 99 6f b9 24 27 ae 41 e4 + %% 64 9b 93 4c a4 95 99 1b 78 52 b8 55 + %% + %% expanded (32 octets): 43 de 77 e0 c7 77 13 85 9a 94 4d b9 db 25 + %% 90 b5 31 90 a6 5b 3e e2 e4 f1 2d d7 a0 bb 7c e2 54 b4 + + %% PRK = HandshakeSecret + %% hash = Hash + %% info = Info + MasterDeriveSecret = + hexstr2bin("43 de 77 e0 c7 77 13 85 9a 94 4d b9 db 25 + 90 b5 31 90 a6 5b 3e e2 e4 f1 2d d7 a0 bb 7c e2 54 b4"), + + MasterDeriveSecret = tls_v1:derive_secret(HandshakeSecret, <<"derived">>, <<>>, HKDFAlgo), + + %% {server} extract secret "master": + %% + %% salt (32 octets): 43 de 77 e0 c7 77 13 85 9a 94 4d b9 db 25 90 b5 + %% 31 90 a6 5b 3e e2 e4 f1 2d d7 a0 bb 7c e2 54 b4 + %% + %% IKM (32 octets): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + %% 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + %% + %% secret (32 octets): 18 df 06 84 3d 13 a0 8b f2 a4 49 84 4c 5f 8a + %% 47 80 01 bc 4d 4c 62 79 84 d5 a4 1d a8 d0 40 29 19 + + %% salt = MasterDeriveSecret + %% IKM = IKM + MasterSecret = + hexstr2bin("18 df 06 84 3d 13 a0 8b f2 a4 49 84 4c 5f 8a + 47 80 01 bc 4d 4c 62 79 84 d5 a4 1d a8 d0 40 29 19"), + + {master_secret, MasterSecret} = + tls_v1:key_schedule(master_secret, HKDFAlgo, {handshake_secret, HandshakeSecret}), + + %% {server} send handshake record: + %% + %% payload (90 octets): 02 00 00 56 03 03 a6 af 06 a4 12 18 60 dc 5e + %% 6e 60 24 9c d3 4c 95 93 0c 8a c5 cb 14 34 da c1 55 77 2e d3 e2 + %% 69 28 00 13 01 00 00 2e 00 33 00 24 00 1d 00 20 c9 82 88 76 11 + %% 20 95 fe 66 76 2b db f7 c6 72 e1 56 d6 cc 25 3b 83 3d f1 dd 69 + %% b1 b0 4e 75 1f 0f 00 2b 00 02 03 04 + %% + %% complete record (95 octets): 16 03 03 00 5a 02 00 00 56 03 03 a6 + %% af 06 a4 12 18 60 dc 5e 6e 60 24 9c d3 4c 95 93 0c 8a c5 cb 14 + %% 34 da c1 55 77 2e d3 e2 69 28 00 13 01 00 00 2e 00 33 00 24 00 + %% 1d 00 20 c9 82 88 76 11 20 95 fe 66 76 2b db f7 c6 72 e1 56 d6 + %% cc 25 3b 83 3d f1 dd 69 b1 b0 4e 75 1f 0f 00 2b 00 02 03 04 + + %% payload = ServerHello + ServerHelloRecord = + hexstr2bin("16 03 03 00 5a 02 00 00 56 03 03 a6 + af 06 a4 12 18 60 dc 5e 6e 60 24 9c d3 4c 95 93 0c 8a c5 cb 14 + 34 da c1 55 77 2e d3 e2 69 28 00 13 01 00 00 2e 00 33 00 24 00 + 1d 00 20 c9 82 88 76 11 20 95 fe 66 76 2b db f7 c6 72 e1 56 d6 + cc 25 3b 83 3d f1 dd 69 b1 b0 4e 75 1f 0f 00 2b 00 02 03 04"), + + {SHEncrypted, _} = + tls_record:encode_handshake(ServerHello, {3,4}, ConnStatesNull), + ServerHelloRecord = iolist_to_binary(SHEncrypted), + + %% {server} derive write traffic keys for handshake data: + %% + %% PRK (32 octets): b6 7b 7d 69 0c c1 6c 4e 75 e5 42 13 cb 2d 37 b4 + %% e9 c9 12 bc de d9 10 5d 42 be fd 59 d3 91 ad 38 + %% + %% key info (13 octets): 00 10 09 74 6c 73 31 33 20 6b 65 79 00 + %% + %% key expanded (16 octets): 3f ce 51 60 09 c2 17 27 d0 f2 e4 e8 6e + %% e4 03 bc + %% + %% iv info (12 octets): 00 0c 08 74 6c 73 31 33 20 69 76 00 + %% + %% iv expanded (12 octets): 5d 31 3e b2 67 12 76 ee 13 00 0b 30 + + %% PRK = SHSTrafficSecret + WriteKeyInfo = + hexstr2bin("00 10 09 74 6c 73 31 33 20 6b 65 79 00"), + + WriteKey = + hexstr2bin("3f ce 51 60 09 c2 17 27 d0 f2 e4 e8 6e e4 03 bc"), + + WriteIVInfo = + hexstr2bin("00 0c 08 74 6c 73 31 33 20 69 76 00"), + + WriteIV = + hexstr2bin(" 5d 31 3e b2 67 12 76 ee 13 00 0b 30"), + + Cipher = aes_128_gcm, %% TODO: get from ServerHello + + WriteKeyInfo = tls_v1:create_info(<<"key">>, <<>>, ssl_cipher:key_material(Cipher)), + %% TODO: remove hardcoded IV size + WriteIVInfo = tls_v1:create_info(<<"iv">>, <<>>, 12), + + {WriteKey, WriteIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, SHSTrafficSecret), + + %% {server} construct an EncryptedExtensions handshake message: + %% + %% EncryptedExtensions (40 octets): 08 00 00 24 00 22 00 0a 00 14 00 + %% 12 00 1d 00 17 00 18 00 19 01 00 01 01 01 02 01 03 01 04 00 1c + %% 00 02 40 01 00 00 00 00 + %% + %% {server} construct a Certificate handshake message: + %% + %% Certificate (445 octets): 0b 00 01 b9 00 00 01 b5 00 01 b0 30 82 + %% 01 ac 30 82 01 15 a0 03 02 01 02 02 01 02 30 0d 06 09 2a 86 48 + %% 86 f7 0d 01 01 0b 05 00 30 0e 31 0c 30 0a 06 03 55 04 03 13 03 + %% 72 73 61 30 1e 17 0d 31 36 30 37 33 30 30 31 32 33 35 39 5a 17 + %% 0d 32 36 30 37 33 30 30 31 32 33 35 39 5a 30 0e 31 0c 30 0a 06 + %% 03 55 04 03 13 03 72 73 61 30 81 9f 30 0d 06 09 2a 86 48 86 f7 + %% 0d 01 01 01 05 00 03 81 8d 00 30 81 89 02 81 81 00 b4 bb 49 8f + %% 82 79 30 3d 98 08 36 39 9b 36 c6 98 8c 0c 68 de 55 e1 bd b8 26 + %% d3 90 1a 24 61 ea fd 2d e4 9a 91 d0 15 ab bc 9a 95 13 7a ce 6c + %% 1a f1 9e aa 6a f9 8c 7c ed 43 12 09 98 e1 87 a8 0e e0 cc b0 52 + %% 4b 1b 01 8c 3e 0b 63 26 4d 44 9a 6d 38 e2 2a 5f da 43 08 46 74 + %% 80 30 53 0e f0 46 1c 8c a9 d9 ef bf ae 8e a6 d1 d0 3e 2b d1 93 + %% ef f0 ab 9a 80 02 c4 74 28 a6 d3 5a 8d 88 d7 9f 7f 1e 3f 02 03 + %% 01 00 01 a3 1a 30 18 30 09 06 03 55 1d 13 04 02 30 00 30 0b 06 + %% 03 55 1d 0f 04 04 03 02 05 a0 30 0d 06 09 2a 86 48 86 f7 0d 01 + %% 01 0b 05 00 03 81 81 00 85 aa d2 a0 e5 b9 27 6b 90 8c 65 f7 3a + %% 72 67 17 06 18 a5 4c 5f 8a 7b 33 7d 2d f7 a5 94 36 54 17 f2 ea + %% e8 f8 a5 8c 8f 81 72 f9 31 9c f3 6b 7f d6 c5 5b 80 f2 1a 03 01 + %% 51 56 72 60 96 fd 33 5e 5e 67 f2 db f1 02 70 2e 60 8c ca e6 be + %% c1 fc 63 a4 2a 99 be 5c 3e b7 10 7c 3c 54 e9 b9 eb 2b d5 20 3b + %% 1c 3b 84 e0 a8 b2 f7 59 40 9b a3 ea c9 d9 1d 40 2d cc 0c c8 f8 + %% 96 12 29 ac 91 87 b4 2b 4d e1 00 00 + %% + %% {server} construct a CertificateVerify handshake message: + %% + %% CertificateVerify (136 octets): 0f 00 00 84 08 04 00 80 5a 74 7c + %% 5d 88 fa 9b d2 e5 5a b0 85 a6 10 15 b7 21 1f 82 4c d4 84 14 5a + %% b3 ff 52 f1 fd a8 47 7b 0b 7a bc 90 db 78 e2 d3 3a 5c 14 1a 07 + %% 86 53 fa 6b ef 78 0c 5e a2 48 ee aa a7 85 c4 f3 94 ca b6 d3 0b + %% be 8d 48 59 ee 51 1f 60 29 57 b1 54 11 ac 02 76 71 45 9e 46 44 + %% 5c 9e a5 8c 18 1e 81 8e 95 b8 c3 fb 0b f3 27 84 09 d3 be 15 2a + %% 3d a5 04 3e 06 3d da 65 cd f5 ae a2 0d 53 df ac d4 2f 74 f3 + EncryptedExtensions = + hexstr2bin("08 00 00 24 00 22 00 0a 00 14 00 + 12 00 1d 00 17 00 18 00 19 01 00 01 01 01 02 01 03 01 04 00 1c + 00 02 40 01 00 00 00 00"), + + Certificate = + hexstr2bin("0b 00 01 b9 00 00 01 b5 00 01 b0 30 82 + 01 ac 30 82 01 15 a0 03 02 01 02 02 01 02 30 0d 06 09 2a 86 48 + 86 f7 0d 01 01 0b 05 00 30 0e 31 0c 30 0a 06 03 55 04 03 13 03 + 72 73 61 30 1e 17 0d 31 36 30 37 33 30 30 31 32 33 35 39 5a 17 + 0d 32 36 30 37 33 30 30 31 32 33 35 39 5a 30 0e 31 0c 30 0a 06 + 03 55 04 03 13 03 72 73 61 30 81 9f 30 0d 06 09 2a 86 48 86 f7 + 0d 01 01 01 05 00 03 81 8d 00 30 81 89 02 81 81 00 b4 bb 49 8f + 82 79 30 3d 98 08 36 39 9b 36 c6 98 8c 0c 68 de 55 e1 bd b8 26 + d3 90 1a 24 61 ea fd 2d e4 9a 91 d0 15 ab bc 9a 95 13 7a ce 6c + 1a f1 9e aa 6a f9 8c 7c ed 43 12 09 98 e1 87 a8 0e e0 cc b0 52 + 4b 1b 01 8c 3e 0b 63 26 4d 44 9a 6d 38 e2 2a 5f da 43 08 46 74 + 80 30 53 0e f0 46 1c 8c a9 d9 ef bf ae 8e a6 d1 d0 3e 2b d1 93 + ef f0 ab 9a 80 02 c4 74 28 a6 d3 5a 8d 88 d7 9f 7f 1e 3f 02 03 + 01 00 01 a3 1a 30 18 30 09 06 03 55 1d 13 04 02 30 00 30 0b 06 + 03 55 1d 0f 04 04 03 02 05 a0 30 0d 06 09 2a 86 48 86 f7 0d 01 + 01 0b 05 00 03 81 81 00 85 aa d2 a0 e5 b9 27 6b 90 8c 65 f7 3a + 72 67 17 06 18 a5 4c 5f 8a 7b 33 7d 2d f7 a5 94 36 54 17 f2 ea + e8 f8 a5 8c 8f 81 72 f9 31 9c f3 6b 7f d6 c5 5b 80 f2 1a 03 01 + 51 56 72 60 96 fd 33 5e 5e 67 f2 db f1 02 70 2e 60 8c ca e6 be + c1 fc 63 a4 2a 99 be 5c 3e b7 10 7c 3c 54 e9 b9 eb 2b d5 20 3b + 1c 3b 84 e0 a8 b2 f7 59 40 9b a3 ea c9 d9 1d 40 2d cc 0c c8 f8 + 96 12 29 ac 91 87 b4 2b 4d e1 00 00"), + + CertificateVerify = + hexstr2bin("0f 00 00 84 08 04 00 80 5a 74 7c + 5d 88 fa 9b d2 e5 5a b0 85 a6 10 15 b7 21 1f 82 4c d4 84 14 5a + b3 ff 52 f1 fd a8 47 7b 0b 7a bc 90 db 78 e2 d3 3a 5c 14 1a 07 + 86 53 fa 6b ef 78 0c 5e a2 48 ee aa a7 85 c4 f3 94 ca b6 d3 0b + be 8d 48 59 ee 51 1f 60 29 57 b1 54 11 ac 02 76 71 45 9e 46 44 + 5c 9e a5 8c 18 1e 81 8e 95 b8 c3 fb 0b f3 27 84 09 d3 be 15 2a + 3d a5 04 3e 06 3d da 65 cd f5 ae a2 0d 53 df ac d4 2f 74 f3"), + + %% {server} calculate finished "tls13 finished": + %% + %% PRK (32 octets): b6 7b 7d 69 0c c1 6c 4e 75 e5 42 13 cb 2d 37 b4 + %% e9 c9 12 bc de d9 10 5d 42 be fd 59 d3 91 ad 38 + %% + %% hash (0 octets): (empty) + %% + %% info (18 octets): 00 20 0e 74 6c 73 31 33 20 66 69 6e 69 73 68 65 + %% 64 00 + %% + %% expanded (32 octets): 00 8d 3b 66 f8 16 ea 55 9f 96 b5 37 e8 85 + %% c3 1f c0 68 bf 49 2c 65 2f 01 f2 88 a1 d8 cd c1 9f c8 + %% + %% finished (32 octets): 9b 9b 14 1d 90 63 37 fb d2 cb dc e7 1d f4 + %% de da 4a b4 2c 30 95 72 cb 7f ff ee 54 54 b7 8f 07 18 + + %% PRK = SHSTrafficSecret + FInfo = + hexstr2bin("00 20 0e 74 6c 73 31 33 20 66 69 6e 69 73 68 65 + 64 00"), + + FExpanded = + hexstr2bin("00 8d 3b 66 f8 16 ea 55 9f 96 b5 37 e8 85 + c3 1f c0 68 bf 49 2c 65 2f 01 f2 88 a1 d8 cd c1 9f c8"), + + FinishedVerifyData = + hexstr2bin("9b 9b 14 1d 90 63 37 fb d2 cb dc e7 1d f4 + de da 4a b4 2c 30 95 72 cb 7f ff ee 54 54 b7 8f 07 18"), + + FInfo = tls_v1:create_info(<<"finished">>, <<>>, ssl_cipher:hash_size(HKDFAlgo)), + + FExpanded = tls_v1:finished_key(SHSTrafficSecret, HKDFAlgo), + + MessageHistory0 = [CertificateVerify, + Certificate, + EncryptedExtensions, + ServerHello, + ClientHello], + + FinishedVerifyData = tls_v1:finished_verify_data(FExpanded, HKDFAlgo, MessageHistory0), + + %% {server} construct a Finished handshake message: + %% + %% Finished (36 octets): 14 00 00 20 9b 9b 14 1d 90 63 37 fb d2 cb + %% dc e7 1d f4 de da 4a b4 2c 30 95 72 cb 7f ff ee 54 54 b7 8f 07 + %% 18 + FinishedHSBin = + hexstr2bin("14 00 00 20 9b 9b 14 1d 90 63 37 fb d2 cb + dc e7 1d f4 de da 4a b4 2c 30 95 72 cb 7f ff ee 54 54 b7 8f 07 + 18"), + + FinishedHS = #finished{verify_data = FinishedVerifyData}, + + FinishedIOList = tls_handshake:encode_handshake(FinishedHS, {3,4}), + FinishedHSBin = iolist_to_binary(FinishedIOList), + + %% {server} derive secret "tls13 c ap traffic": + %% + %% PRK (32 octets): 18 df 06 84 3d 13 a0 8b f2 a4 49 84 4c 5f 8a 47 + %% 80 01 bc 4d 4c 62 79 84 d5 a4 1d a8 d0 40 29 19 + %% + %% hash (32 octets): 96 08 10 2a 0f 1c cc 6d b6 25 0b 7b 7e 41 7b 1a + %% 00 0e aa da 3d aa e4 77 7a 76 86 c9 ff 83 df 13 + %% + %% info (54 octets): 00 20 12 74 6c 73 31 33 20 63 20 61 70 20 74 72 + %% 61 66 66 69 63 20 96 08 10 2a 0f 1c cc 6d b6 25 0b 7b 7e 41 7b + %% 1a 00 0e aa da 3d aa e4 77 7a 76 86 c9 ff 83 df 13 + %% + %% expanded (32 octets): 9e 40 64 6c e7 9a 7f 9d c0 5a f8 88 9b ce + %% 65 52 87 5a fa 0b 06 df 00 87 f7 92 eb b7 c1 75 04 a5 + + %% PRK = MasterSecret + CAPTHash = + hexstr2bin("96 08 10 2a 0f 1c cc 6d b6 25 0b 7b 7e 41 7b 1a + 00 0e aa da 3d aa e4 77 7a 76 86 c9 ff 83 df 13"), + CAPTInfo = + hexstr2bin("00 20 12 74 6c 73 31 33 20 63 20 61 70 20 74 72 + 61 66 66 69 63 20 96 08 10 2a 0f 1c cc 6d b6 25 0b 7b 7e 41 7b + 1a 00 0e aa da 3d aa e4 77 7a 76 86 c9 ff 83 df 13"), + + CAPTrafficSecret = + hexstr2bin("9e 40 64 6c e7 9a 7f 9d c0 5a f8 88 9b ce + 65 52 87 5a fa 0b 06 df 00 87 f7 92 eb b7 c1 75 04 a5"), + + CHSF = <<ClientHello/binary, + ServerHello/binary, + EncryptedExtensions/binary, + Certificate/binary, + CertificateVerify/binary, + FinishedHSBin/binary>>, + + CAPTHash = crypto:hash(HKDFAlgo, CHSF), + + CAPTInfo = + tls_v1:create_info(<<"c ap traffic">>, CAPTHash, ssl_cipher:hash_size(HKDFAlgo)), + + CAPTrafficSecret = + tls_v1:client_application_traffic_secret_0(HKDFAlgo, {master_secret, MasterSecret}, CHSF), + + %% {server} derive secret "tls13 s ap traffic": + %% + %% PRK (32 octets): 18 df 06 84 3d 13 a0 8b f2 a4 49 84 4c 5f 8a 47 + %% 80 01 bc 4d 4c 62 79 84 d5 a4 1d a8 d0 40 29 19 + %% + %% hash (32 octets): 96 08 10 2a 0f 1c cc 6d b6 25 0b 7b 7e 41 7b 1a + %% 00 0e aa da 3d aa e4 77 7a 76 86 c9 ff 83 df 13 + %% + %% info (54 octets): 00 20 12 74 6c 73 31 33 20 73 20 61 70 20 74 72 + %% 61 66 66 69 63 20 96 08 10 2a 0f 1c cc 6d b6 25 0b 7b 7e 41 7b + %% 1a 00 0e aa da 3d aa e4 77 7a 76 86 c9 ff 83 df 13 + %% + %% expanded (32 octets): a1 1a f9 f0 55 31 f8 56 ad 47 11 6b 45 a9 + %% 50 32 82 04 b4 f4 4b fb 6b 3a 4b 4f 1f 3f cb 63 16 43 + + %% PRK = MasterSecret + %% hash = CAPTHash + SAPTInfo = + hexstr2bin(" 00 20 12 74 6c 73 31 33 20 73 20 61 70 20 74 72 + 61 66 66 69 63 20 96 08 10 2a 0f 1c cc 6d b6 25 0b 7b 7e 41 7b + 1a 00 0e aa da 3d aa e4 77 7a 76 86 c9 ff 83 df 13"), + + SAPTrafficSecret = + hexstr2bin("a1 1a f9 f0 55 31 f8 56 ad 47 11 6b 45 a9 + 50 32 82 04 b4 f4 4b fb 6b 3a 4b 4f 1f 3f cb 63 16 43"), + + SAPTInfo = + tls_v1:create_info(<<"s ap traffic">>, CAPTHash, ssl_cipher:hash_size(HKDFAlgo)), + + SAPTrafficSecret = + tls_v1:server_application_traffic_secret_0(HKDFAlgo, {master_secret, MasterSecret}, CHSF), + + %% {server} derive secret "tls13 exp master": + %% + %% PRK (32 octets): 18 df 06 84 3d 13 a0 8b f2 a4 49 84 4c 5f 8a 47 + %% 80 01 bc 4d 4c 62 79 84 d5 a4 1d a8 d0 40 29 19 + %% + %% hash (32 octets): 96 08 10 2a 0f 1c cc 6d b6 25 0b 7b 7e 41 7b 1a + %% 00 0e aa da 3d aa e4 77 7a 76 86 c9 ff 83 df 13 + %% + %% info (52 octets): 00 20 10 74 6c 73 31 33 20 65 78 70 20 6d 61 73 + %% 74 65 72 20 96 08 10 2a 0f 1c cc 6d b6 25 0b 7b 7e 41 7b 1a 00 + %% 0e aa da 3d aa e4 77 7a 76 86 c9 ff 83 df 13 + %% + %% expanded (32 octets): fe 22 f8 81 17 6e da 18 eb 8f 44 52 9e 67 + %% 92 c5 0c 9a 3f 89 45 2f 68 d8 ae 31 1b 43 09 d3 cf 50 + + %% PRK = MasterSecret + %% hash = CAPTHash + ExporterInfo = + hexstr2bin("00 20 10 74 6c 73 31 33 20 65 78 70 20 6d 61 73 + 74 65 72 20 96 08 10 2a 0f 1c cc 6d b6 25 0b 7b 7e 41 7b 1a 00 + 0e aa da 3d aa e4 77 7a 76 86 c9 ff 83 df 13"), + + ExporterMasterSecret = + hexstr2bin("fe 22 f8 81 17 6e da 18 eb 8f 44 52 9e 67 + 92 c5 0c 9a 3f 89 45 2f 68 d8 ae 31 1b 43 09 d3 cf 50"), + + ExporterInfo = + tls_v1:create_info(<<"exp master">>, CAPTHash, ssl_cipher:hash_size(HKDFAlgo)), + + ExporterMasterSecret = + tls_v1:exporter_master_secret(HKDFAlgo, {master_secret, MasterSecret}, CHSF), + + %% {server} derive write traffic keys for application data: + %% + %% PRK (32 octets): a1 1a f9 f0 55 31 f8 56 ad 47 11 6b 45 a9 50 32 + %% 82 04 b4 f4 4b fb 6b 3a 4b 4f 1f 3f cb 63 16 43 + %% + %% key info (13 octets): 00 10 09 74 6c 73 31 33 20 6b 65 79 00 + %% + %% key expanded (16 octets): 9f 02 28 3b 6c 9c 07 ef c2 6b b9 f2 ac + %% 92 e3 56 + %% + %% iv info (12 octets): 00 0c 08 74 6c 73 31 33 20 69 76 00 + %% + %% iv expanded (12 octets): cf 78 2b 88 dd 83 54 9a ad f1 e9 84 + + %% PRK = SAPTrafficsecret + %% key info = WriteKeyInfo + %% iv info = WrtieIVInfo + SWKey = + hexstr2bin("9f 02 28 3b 6c 9c 07 ef c2 6b b9 f2 ac 92 e3 56"), + + SWIV = + hexstr2bin("cf 78 2b 88 dd 83 54 9a ad f1 e9 84"), + + {SWKey, SWIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, SAPTrafficSecret), + + %% {server} derive read traffic keys for handshake data: + %% + %% PRK (32 octets): b3 ed db 12 6e 06 7f 35 a7 80 b3 ab f4 5e 2d 8f + %% 3b 1a 95 07 38 f5 2e 96 00 74 6a 0e 27 a5 5a 21 + %% + %% key info (13 octets): 00 10 09 74 6c 73 31 33 20 6b 65 79 00 + %% + %% key expanded (16 octets): db fa a6 93 d1 76 2c 5b 66 6a f5 d9 50 + %% 25 8d 01 + %% + %% iv info (12 octets): 00 0c 08 74 6c 73 31 33 20 69 76 00 + %% + %% iv expanded (12 octets): 5b d3 c7 1b 83 6e 0b 76 bb 73 26 5f + + %% PRK = CHSTrafficsecret + %% key info = WriteKeyInfo + %% iv info = WrtieIVInfo + SRKey = + hexstr2bin("db fa a6 93 d1 76 2c 5b 66 6a f5 d9 50 25 8d 01"), + + SRIV = + hexstr2bin("5b d3 c7 1b 83 6e 0b 76 bb 73 26 5f"), + + {SRKey, SRIV} = tls_v1:calculate_traffic_keys(HKDFAlgo, Cipher, CHSTrafficSecret). + + +tls13_finished_verify_data() -> + [{doc,"Test TLS 1.3 Finished message handling"}]. + +tls13_finished_verify_data(_Config) -> + ClientHello = + hexstr2bin("01 00 00 c6 03 03 00 01 02 03 04 05 06 07 08 09 + 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 + 1a 1b 1c 1d 1e 1f 20 e0 e1 e2 e3 e4 e5 e6 e7 e8 + e9 ea eb ec ed ee ef f0 f1 f2 f3 f4 f5 f6 f7 f8 + f9 fa fb fc fd fe ff 00 06 13 01 13 02 13 03 01 + 00 00 77 00 00 00 18 00 16 00 00 13 65 78 61 6d + 70 6c 65 2e 75 6c 66 68 65 69 6d 2e 6e 65 74 00 + 0a 00 08 00 06 00 1d 00 17 00 18 00 0d 00 14 00 + 12 04 03 08 04 04 01 05 03 08 05 05 01 08 06 06 + 01 02 01 00 33 00 26 00 24 00 1d 00 20 35 80 72 + d6 36 58 80 d1 ae ea 32 9a df 91 21 38 38 51 ed + 21 a2 8e 3b 75 e9 65 d0 d2 cd 16 62 54 00 2d 00 + 02 01 01 00 2b 00 03 02 03 04"), + + ServerHello = + hexstr2bin("02 00 00 76 03 03 70 71 72 73 74 75 76 77 78 79 + 7a 7b 7c 7d 7e 7f 80 81 82 83 84 85 86 87 88 89 + 8a 8b 8c 8d 8e 8f 20 e0 e1 e2 e3 e4 e5 e6 e7 e8 + e9 ea eb ec ed ee ef f0 f1 f2 f3 f4 f5 f6 f7 f8 + f9 fa fb fc fd fe ff 13 01 00 00 2e 00 33 00 24 + 00 1d 00 20 9f d7 ad 6d cf f4 29 8d d3 f9 6d 5b + 1b 2a f9 10 a0 53 5b 14 88 d7 f8 fa bb 34 9a 98 + 28 80 b6 15 00 2b 00 02 03 04"), + + EncryptedExtensions = + hexstr2bin("08 00 00 02 00 00"), + + Certificate = + hexstr2bin("0b 00 03 2e 00 00 03 2a 00 03 25 30 82 03 21 30 + 82 02 09 a0 03 02 01 02 02 08 15 5a 92 ad c2 04 + 8f 90 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 + 00 30 22 31 0b 30 09 06 03 55 04 06 13 02 55 53 + 31 13 30 11 06 03 55 04 0a 13 0a 45 78 61 6d 70 + 6c 65 20 43 41 30 1e 17 0d 31 38 31 30 30 35 30 + 31 33 38 31 37 5a 17 0d 31 39 31 30 30 35 30 31 + 33 38 31 37 5a 30 2b 31 0b 30 09 06 03 55 04 06 + 13 02 55 53 31 1c 30 1a 06 03 55 04 03 13 13 65 + 78 61 6d 70 6c 65 2e 75 6c 66 68 65 69 6d 2e 6e + 65 74 30 82 01 22 30 0d 06 09 2a 86 48 86 f7 0d + 01 01 01 05 00 03 82 01 0f 00 30 82 01 0a 02 82 + 01 01 00 c4 80 36 06 ba e7 47 6b 08 94 04 ec a7 + b6 91 04 3f f7 92 bc 19 ee fb 7d 74 d7 a8 0d 00 + 1e 7b 4b 3a 4a e6 0f e8 c0 71 fc 73 e7 02 4c 0d + bc f4 bd d1 1d 39 6b ba 70 46 4a 13 e9 4a f8 3d + f3 e1 09 59 54 7b c9 55 fb 41 2d a3 76 52 11 e1 + f3 dc 77 6c aa 53 37 6e ca 3a ec be c3 aa b7 3b + 31 d5 6c b6 52 9c 80 98 bc c9 e0 28 18 e2 0b f7 + f8 a0 3a fd 17 04 50 9e ce 79 bd 9f 39 f1 ea 69 + ec 47 97 2e 83 0f b5 ca 95 de 95 a1 e6 04 22 d5 + ee be 52 79 54 a1 e7 bf 8a 86 f6 46 6d 0d 9f 16 + 95 1a 4c f7 a0 46 92 59 5c 13 52 f2 54 9e 5a fb + 4e bf d7 7a 37 95 01 44 e4 c0 26 87 4c 65 3e 40 + 7d 7d 23 07 44 01 f4 84 ff d0 8f 7a 1f a0 52 10 + d1 f4 f0 d5 ce 79 70 29 32 e2 ca be 70 1f df ad + 6b 4b b7 11 01 f4 4b ad 66 6a 11 13 0f e2 ee 82 + 9e 4d 02 9d c9 1c dd 67 16 db b9 06 18 86 ed c1 + ba 94 21 02 03 01 00 01 a3 52 30 50 30 0e 06 03 + 55 1d 0f 01 01 ff 04 04 03 02 05 a0 30 1d 06 03 + 55 1d 25 04 16 30 14 06 08 2b 06 01 05 05 07 03 + 02 06 08 2b 06 01 05 05 07 03 01 30 1f 06 03 55 + 1d 23 04 18 30 16 80 14 89 4f de 5b cc 69 e2 52 + cf 3e a3 00 df b1 97 b8 1d e1 c1 46 30 0d 06 09 + 2a 86 48 86 f7 0d 01 01 0b 05 00 03 82 01 01 00 + 59 16 45 a6 9a 2e 37 79 e4 f6 dd 27 1a ba 1c 0b + fd 6c d7 55 99 b5 e7 c3 6e 53 3e ff 36 59 08 43 + 24 c9 e7 a5 04 07 9d 39 e0 d4 29 87 ff e3 eb dd + 09 c1 cf 1d 91 44 55 87 0b 57 1d d1 9b df 1d 24 + f8 bb 9a 11 fe 80 fd 59 2b a0 39 8c de 11 e2 65 + 1e 61 8c e5 98 fa 96 e5 37 2e ef 3d 24 8a fd e1 + 74 63 eb bf ab b8 e4 d1 ab 50 2a 54 ec 00 64 e9 + 2f 78 19 66 0d 3f 27 cf 20 9e 66 7f ce 5a e2 e4 + ac 99 c7 c9 38 18 f8 b2 51 07 22 df ed 97 f3 2e + 3e 93 49 d4 c6 6c 9e a6 39 6d 74 44 62 a0 6b 42 + c6 d5 ba 68 8e ac 3a 01 7b dd fc 8e 2c fc ad 27 + cb 69 d3 cc dc a2 80 41 44 65 d3 ae 34 8c e0 f3 + 4a b2 fb 9c 61 83 71 31 2b 19 10 41 64 1c 23 7f + 11 a5 d6 5c 84 4f 04 04 84 99 38 71 2b 95 9e d6 + 85 bc 5c 5d d6 45 ed 19 90 94 73 40 29 26 dc b4 + 0e 34 69 a1 59 41 e8 e2 cc a8 4b b6 08 46 36 a0 + 00 00"), + + CertificateVerify = + hexstr2bin("0f 00 01 04 08 04 01 00 17 fe b5 33 ca 6d 00 7d + 00 58 25 79 68 42 4b bc 3a a6 90 9e 9d 49 55 75 + 76 a5 20 e0 4a 5e f0 5f 0e 86 d2 4f f4 3f 8e b8 + 61 ee f5 95 22 8d 70 32 aa 36 0f 71 4e 66 74 13 + 92 6e f4 f8 b5 80 3b 69 e3 55 19 e3 b2 3f 43 73 + df ac 67 87 06 6d cb 47 56 b5 45 60 e0 88 6e 9b + 96 2c 4a d2 8d ab 26 ba d1 ab c2 59 16 b0 9a f2 + 86 53 7f 68 4f 80 8a ef ee 73 04 6c b7 df 0a 84 + fb b5 96 7a ca 13 1f 4b 1c f3 89 79 94 03 a3 0c + 02 d2 9c bd ad b7 25 12 db 9c ec 2e 5e 1d 00 e5 + 0c af cf 6f 21 09 1e bc 4f 25 3c 5e ab 01 a6 79 + ba ea be ed b9 c9 61 8f 66 00 6b 82 44 d6 62 2a + aa 56 88 7c cf c6 6a 0f 38 51 df a1 3a 78 cf f7 + 99 1e 03 cb 2c 3a 0e d8 7d 73 67 36 2e b7 80 5b + 00 b2 52 4f f2 98 a4 da 48 7c ac de af 8a 23 36 + c5 63 1b 3e fa 93 5b b4 11 e7 53 ca 13 b0 15 fe + c7 e4 a7 30 f1 36 9f 9e"), + + BaseKey = + hexstr2bin("a2 06 72 65 e7 f0 65 2a 92 3d 5d 72 ab 04 67 c4 + 61 32 ee b9 68 b6 a3 2d 31 1c 80 58 68 54 88 14"), + + VerifyData = + hexstr2bin("ea 6e e1 76 dc cc 4a f1 85 9e 9e 4e 93 f7 97 ea + c9 a7 8c e4 39 30 1e 35 27 5a d4 3f 3c dd bd e3"), + + Messages = [CertificateVerify, + Certificate, + EncryptedExtensions, + ServerHello, + ClientHello], + + FinishedKey = tls_v1:finished_key(BaseKey, sha256), + VerifyData = tls_v1:finished_verify_data(FinishedKey, sha256, Messages). + +tls13_basic_ssl_server_openssl_client() -> + [{doc,"Test TLS 1.3 basic connection between ssl server and openssl s_client"}]. + +tls13_basic_ssl_server_openssl_client(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + +tls13_custom_groups_ssl_server_openssl_client() -> + [{doc,"Test that ssl server can select a common group for key-exchange"}]. + +tls13_custom_groups_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {supported_groups, [x448, secp256r1, secp384r1]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + ClientOpts = [{groups,"P-384:P-256:X25519"}|ClientOpts0], + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + +tls13_hello_retry_request_ssl_server_openssl_client() -> + [{doc,"Test that ssl server can request a new group when the client's first key share" + "is not supported"}]. + +tls13_hello_retry_request_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts0], + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + +tls13_client_auth_empty_cert_alert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to true."}]. + +tls13_client_auth_empty_cert_alert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts = proplists:delete(keyfile, ClientOpts1), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, + {error, + {tls_alert, + {certificate_required, + "received SERVER ALERT: Fatal - Certificate required - certificate_required"}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + +tls13_client_auth_empty_cert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to false."}]. + +tls13_client_auth_empty_cert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts = proplists:delete(keyfile, ClientOpts1), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, false}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication."}]. + +tls13_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_hrr_client_auth_empty_cert_alert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3 (HelloRetryRequest): Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to true."}]. + +tls13_hrr_client_auth_empty_cert_alert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts2 = proplists:delete(keyfile, ClientOpts1), + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts2], + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, + {error, + {tls_alert, + {certificate_required, + "received SERVER ALERT: Fatal - Certificate required - certificate_required"}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_hrr_client_auth_empty_cert_ssl_server_openssl_client() -> + [{doc,"TLS 1.3 (HelloRetryRequest): Test client authentication when client sends an empty certificate and fail_if_no_peer_cert is set to false."}]. + +tls13_hrr_client_auth_empty_cert_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + %% Delete Client Cert and Key + ClientOpts1 = proplists:delete(certfile, ClientOpts0), + ClientOpts2 = proplists:delete(keyfile, ClientOpts1), + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts2], + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, false}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_hrr_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3 (HelloRetryRequest): Test client authentication."}]. + +tls13_hrr_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ClientOpts = [{groups,"P-256:X25519"}|ClientOpts0], + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}, + {supported_groups, [x448, x25519]}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_unsupported_sign_algo_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication with unsupported signature_algorithm"}]. + +tls13_unsupported_sign_algo_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + %% Skip rsa_pkcs1_sha256! + {signature_algs, [rsa_pkcs1_sha384, rsa_pkcs1_sha512]}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result( + Server, + {error, + {tls_alert, + {insufficient_security, + "received SERVER ALERT: Fatal - Insufficient Security - " + "\"No suitable signature algorithm\""}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +%% Triggers Client Alert as openssl s_client does not have a certificate with a +%% signature algorithm supported by the server (signature_algorithms_cert extension +%% of CertificateRequest does not contain the algorithm of the client certificate). +tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client() -> + [{doc,"TLS 1.3: Test client authentication with unsupported signature_algorithm_cert"}]. + +tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {log_level, debug}, + {verify, verify_peer}, + {signature_algs, [rsa_pkcs1_sha256, rsa_pkcs1_sha384, rsa_pss_rsae_sha256]}, + %% Skip rsa_pkcs1_sha256! + {signature_algs_cert, [rsa_pkcs1_sha384, rsa_pkcs1_sha512]}, + {fail_if_no_peer_cert, true}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result( + Server, + {error, + {tls_alert, + {illegal_parameter, + "received CLIENT ALERT: Fatal - Illegal Parameter"}}}), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_connection_information() -> + [{doc,"Test the API function ssl:connection_information/1 in a TLS 1.3 connection"}]. + +tls13_connection_information(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, connection_information_result, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ @@ -5043,147 +6135,6 @@ client_server_opts(#{key_exchange := KeyAlgo}, Config) when KeyAlgo == ecdh_rsa {ssl_test_lib:ssl_options(client_opts, Config), ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)}. -run_suites(Ciphers, Config, Type) -> - Version = ssl_test_lib:protocol_version(Config), - ct:log("Running cipher suites ~p~n", [Ciphers]), - {ClientOpts, ServerOpts} = - case Type of - rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_opts, Config)]}; - dsa -> - {ssl_test_lib:ssl_options(client_dsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_dsa_opts, Config)]}; - anonymous -> - %% No certs in opts! - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options([], Config)]}; - psk -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk, Config)]}; - psk_with_hint -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk_hint, Config) - ]}; - psk_anon -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk_anon, Config)]}; - psk_anon_with_hint -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk_anon_hint, Config)]}; - srp -> - {ssl_test_lib:ssl_options(client_srp, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_srp, Config)]}; - srp_anon -> - {ssl_test_lib:ssl_options(client_srp, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_srp_anon, Config)]}; - srp_dsa -> - {ssl_test_lib:ssl_options(client_srp_dsa, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_srp_dsa, Config)]}; - ecdsa -> - {ssl_test_lib:ssl_options(client_ecdsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdsa_opts, Config)]}; - ecdh_rsa -> - {ssl_test_lib:ssl_options(client_ecdh_rsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)]}; - rc4_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_verify_opts, Config)]}; - rc4_ecdh_rsa -> - {ssl_test_lib:ssl_options(client_ecdh_rsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)]}; - rc4_ecdsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdsa_opts, Config)]}; - des_dhe_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_verification_opts, Config)]}; - des_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_verify_opts, Config)]}; - chacha_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_verify_opts, Config)]}; - chacha_ecdsa -> - {ssl_test_lib:ssl_options(client_ecdsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdsa_opts, Config)]} - end, - Suites = ssl_test_lib:filter_suites(Ciphers, Version), - ct:pal("ssl_test_lib:filter_suites(~p ~p) -> ~p ", [Ciphers, Version, Suites]), - Results0 = lists:map(fun(Cipher) -> - cipher(Cipher, Version, Config, ClientOpts, ServerOpts) end, - ssl_test_lib:filter_suites(Ciphers, Version)), - Results = lists:flatten(Results0), - true = length(Results) == length(Suites), - check_cipher_result(Results). - -check_cipher_result([]) -> - ok; -check_cipher_result([ok | Rest]) -> - check_cipher_result(Rest); -check_cipher_result([_ |_] = Error) -> - ct:fail(Error). - -erlang_cipher_suite(Suite) when is_list(Suite)-> - ssl_cipher_format:suite_definition(ssl_cipher_format:openssl_suite(Suite)); -erlang_cipher_suite(Suite) -> - Suite. - -cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> - %% process_flag(trap_exit, true), - ct:log("Testing CipherSuite ~p~n", [CipherSuite]), - ct:log("Server Opts ~p~n", [ServerOpts]), - ct:log("Client Opts ~p~n", [ClientOpts]), - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - - ErlangCipherSuite = erlang_cipher_suite(CipherSuite), - - ConnectionInfo = {ok, {Version, ErlangCipherSuite}}, - - Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, - {from, self()}, - {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, - {options, ServerOpts}]), - Port = ssl_test_lib:inet_port(Server), - Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, - {options, - [{ciphers,[CipherSuite]} | - ClientOpts]}]), - - Result = ssl_test_lib:wait_for_result(Server, ok, Client, ok), - - ssl_test_lib:close(Server), - ssl_test_lib:close(Client), - - case Result of - ok -> - [ok]; - Error -> - [{ErlangCipherSuite, Error}] - end. - connection_information_result(Socket) -> {ok, Info = [_ | _]} = ssl:connection_information(Socket), case length(Info) > 3 of @@ -5347,3 +6298,31 @@ tls_or_dtls('dtlsv1.2') -> dtls; tls_or_dtls(_) -> tls. + +hexstr2int(S) -> + B = hexstr2bin(S), + Bits = size(B) * 8, + <<Integer:Bits/integer>> = B, + Integer. + +hexstr2bin(S) when is_binary(S) -> + hexstr2bin(S, <<>>); +hexstr2bin(S) -> + hexstr2bin(list_to_binary(S), <<>>). +%% +hexstr2bin(<<>>, Acc) -> + Acc; +hexstr2bin(<<C,T/binary>>, Acc) when C =:= 32; %% SPACE + C =:= 10; %% LF + C =:= 13 -> %% CR + hexstr2bin(T, Acc); +hexstr2bin(<<X,Y,T/binary>>, Acc) -> + I = hex2int(X) * 16 + hex2int(Y), + hexstr2bin(T, <<Acc/binary,I>>). + +hex2int(C) when $0 =< C, C =< $9 -> + C - $0; +hex2int(C) when $A =< C, C =< $F -> + C - $A + 10; +hex2int(C) when $a =< C, C =< $f -> + C - $a + 10. diff --git a/lib/ssl/test/ssl_bench.spec b/lib/ssl/test/ssl_bench.spec index 8b746c5ca9..217cc6fc83 100644 --- a/lib/ssl/test/ssl_bench.spec +++ b/lib/ssl/test/ssl_bench.spec @@ -1 +1,2 @@ {suites,"../ssl_test",[ssl_bench_SUITE, ssl_dist_bench_SUITE]}. +{skip_groups,"../ssl_test",ssl_bench_SUITE,basic,"Benchmarks run separately"}. diff --git a/lib/ssl/test/ssl_bench_SUITE.erl b/lib/ssl/test/ssl_bench_SUITE.erl index 13097b08b6..35efa2b8a3 100644 --- a/lib/ssl/test/ssl_bench_SUITE.erl +++ b/lib/ssl/test/ssl_bench_SUITE.erl @@ -25,10 +25,11 @@ suite() -> [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}]. -all() -> [{group, setup}, {group, payload}, {group, pem_cache}]. +all() -> [{group, basic}, {group, setup}, {group, payload}, {group, pem_cache}]. groups() -> - [{setup, [{repeat, 3}], [setup_sequential, setup_concurrent]}, + [{basic, [], [basic_pem_cache]}, + {setup, [{repeat, 3}], [setup_sequential, setup_concurrent]}, {payload, [{repeat, 3}], [payload_simple]}, {pem_cache, [{repeat, 3}], [use_pem_cache, bypass_pem_cache]} ]. @@ -40,6 +41,7 @@ end_per_group(_GroupName, _Config) -> ok. init_per_suite(Config) -> + ct:timetrap({minutes, 1}), case node() of nonode@nohost -> {skipped, "Node not distributed"}; @@ -51,29 +53,21 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok. -init_per_testcase(use_pem_cache, Conf) -> +init_per_testcase(TC, Conf) when TC =:= use_pem_cache; + TC =:= bypass_pem_cache; + TC =:= basic_pem_cache -> case bypass_pem_cache_supported() of false -> {skipped, "PEM cache bypass support required"}; true -> application:set_env(ssl, bypass_pem_cache, false), Conf end; -init_per_testcase(bypass_pem_cache, Conf) -> - case bypass_pem_cache_supported() of - false -> {skipped, "PEM cache bypass support required"}; - true -> - application:set_env(ssl, bypass_pem_cache, true), - Conf - end; init_per_testcase(_Func, Conf) -> Conf. -end_per_testcase(use_pem_cache, _Config) -> - case bypass_pem_cache_supported() of - false -> ok; - true -> application:set_env(ssl, bypass_pem_cache, false) - end; -end_per_testcase(bypass_pem_cache, _Config) -> +end_per_testcase(TC, _Config) when TC =:= use_pem_cache; + TC =:= bypass_pem_cache; + TC =:= basic_pem_cache -> case bypass_pem_cache_supported() of false -> ok; true -> application:set_env(ssl, bypass_pem_cache, false) @@ -119,6 +113,9 @@ payload_simple(Config) -> {suite, "ssl"}, {name, "Payload simple"}]}), ok. +basic_pem_cache(_Config) -> + do_test(ssl, pem_cache, 10, 5, node()). + use_pem_cache(_Config) -> {ok, Result} = do_test(ssl, pem_cache, 100, 500, node()), ct_event:notify(#event{name = benchmark_data, @@ -167,7 +164,7 @@ do_test(Type, TC, Loop, ParallellConnections, Server) -> end end, Spawn = fun(Id) -> - Pid = spawn(fun() -> Test(Id) end), + Pid = spawn_link(fun() -> Test(Id) end), receive {Pid, init} -> Pid end end, Pids = [Spawn(Id) || Id <- lists:seq(ParallellConnections, 1, -1)], @@ -184,42 +181,42 @@ do_test(Type, TC, Loop, ParallellConnections, Server) -> {ok, TestPerSecond}. server_init(ssl, setup_connection, _, _, Server) -> - {ok, Socket} = ssl:listen(0, ssl_opts(listen)), - {ok, {_Host, Port}} = ssl:sockname(Socket), + {ok, LSocket} = ssl:listen(0, ssl_opts(listen)), + {ok, {_Host, Port}} = ssl:sockname(LSocket), {ok, Host} = inet:gethostname(), ?FPROF_SERVER andalso start_profile(fprof, [whereis(ssl_manager), new]), %%?EPROF_SERVER andalso start_profile(eprof, [ssl_connection_sup, ssl_manager]), ?EPROF_SERVER andalso start_profile(eprof, [ssl_manager]), Server ! {self(), {init, Host, Port}}, Test = fun(TSocket) -> - ok = ssl:ssl_accept(TSocket), - ssl:close(TSocket) + {ok, Socket} = ssl:handshake(TSocket), + ssl:close(Socket) end, - setup_server_connection(Socket, Test); + setup_server_connection(LSocket, Test); server_init(ssl, payload, Loop, _, Server) -> - {ok, Socket} = ssl:listen(0, ssl_opts(listen)), - {ok, {_Host, Port}} = ssl:sockname(Socket), + {ok, LSocket} = ssl:listen(0, ssl_opts(listen)), + {ok, {_Host, Port}} = ssl:sockname(LSocket), {ok, Host} = inet:gethostname(), Server ! {self(), {init, Host, Port}}, Test = fun(TSocket) -> - ok = ssl:ssl_accept(TSocket), + {ok, Socket} = ssl:handshake(TSocket), Size = byte_size(msg()), - server_echo(TSocket, Size, Loop), - ssl:close(TSocket) + server_echo(Socket, Size, Loop), + ssl:close(Socket) end, - setup_server_connection(Socket, Test); + setup_server_connection(LSocket, Test); server_init(ssl, pem_cache, Loop, _, Server) -> - {ok, Socket} = ssl:listen(0, ssl_opts(listen_der)), - {ok, {_Host, Port}} = ssl:sockname(Socket), + {ok, LSocket} = ssl:listen(0, ssl_opts(listen_der)), + {ok, {_Host, Port}} = ssl:sockname(LSocket), {ok, Host} = inet:gethostname(), Server ! {self(), {init, Host, Port}}, Test = fun(TSocket) -> - ok = ssl:ssl_accept(TSocket), + {ok, Socket} = ssl:handshake(TSocket), Size = byte_size(msg()), - server_echo(TSocket, Size, Loop), - ssl:close(TSocket) + server_echo(Socket, Size, Loop), + ssl:close(Socket) end, - setup_server_connection(Socket, Test); + setup_server_connection(LSocket, Test); server_init(Type, Tc, _, _, Server) -> io:format("No server init code for ~p ~p~n",[Type, Tc]), diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index e89104a999..55dee9a48f 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -147,6 +147,7 @@ init_per_testcase(_TestCase, Config) -> ssl:stop(), ssl:start(), ssl_test_lib:ct_log_supported_protocol_versions(Config), + ct:pal(" ~p", [ dtls_record:supported_protocol_versions()]), ct:timetrap({seconds, 10}), Config. @@ -481,7 +482,7 @@ verify_fun_always_run_client(Config) when is_list(Config) -> Port = ssl_test_lib:inet_port(Server), %% If user verify fun is called correctly we fail the connection. - %% otherwise we can not tell this case apart form where we miss + %% otherwise we cannot tell this case apart form where we miss %% to call users verify fun FunAndState = {fun(_,{extension, _}, UserState) -> {unknown, UserState}; @@ -514,7 +515,7 @@ verify_fun_always_run_server(Config) when is_list(Config) -> {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), %% If user verify fun is called correctly we fail the connection. - %% otherwise we can not tell this case apart form where we miss + %% otherwise we cannot tell this case apart form where we miss %% to call users verify fun FunAndState = {fun(_,{extension, _}, UserState) -> {unknown, UserState}; diff --git a/lib/ssl/test/ssl_cipher_suite_SUITE.erl b/lib/ssl/test/ssl_cipher_suite_SUITE.erl new file mode 100644 index 0000000000..51788c29e7 --- /dev/null +++ b/lib/ssl/test/ssl_cipher_suite_SUITE.erl @@ -0,0 +1,759 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019-2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssl_cipher_suite_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- +all() -> + [ + {group, 'tlsv1.2'}, + {group, 'tlsv1.1'}, + {group, 'tlsv1'}, + {group, 'sslv3'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} + ]. + +groups() -> + [ + {'tlsv1.2', [], kex()}, + {'tlsv1.1', [], kex()}, + {'tlsv1', [], kex()}, + {'sslv3', [], kex()}, + {'dtlsv1.2', [], kex()}, + {'dtlsv1', [], kex()}, + {dhe_rsa, [],[dhe_rsa_3des_ede_cbc, + dhe_rsa_aes_128_cbc, + dhe_rsa_aes_256_cbc, + dhe_rsa_chacha20_poly1305 + ]}, + {ecdhe_rsa, [], [ecdhe_rsa_3des_ede_cbc, + ecdhe_rsa_aes_128_cbc, + ecdhe_rsa_aes_128_gcm, + ecdhe_rsa_aes_256_cbc, + ecdhe_rsa_aes_256_gcm, + ecdhe_rsa_chacha20_poly1305 + ]}, + {ecdhe_ecdsa, [],[ecdhe_ecdsa_rc4_128, + ecdhe_ecdsa_3des_ede_cbc, + ecdhe_ecdsa_aes_128_cbc, + ecdhe_ecdsa_aes_128_gcm, + ecdhe_ecdsa_aes_256_cbc, + ecdhe_ecdsa_aes_256_gcm, + ecdhe_ecdsa_chacha20_poly1305 + ]}, + {rsa, [], [rsa_3des_ede_cbc, + rsa_aes_128_cbc, + rsa_aes_256_cbc, + rsa_rc4_128 + ]}, + {dhe_dss, [], [dhe_dss_3des_ede_cbc, + dhe_dss_aes_128_cbc, + dhe_dss_aes_256_cbc]}, + {srp_rsa, [], [srp_rsa_3des_ede_cbc, + srp_rsa_aes_128_cbc, + srp_rsa_aes_256_cbc]}, + {srp_dss, [], [srp_dss_3des_ede_cbc, + srp_dss_aes_128_cbc, + srp_dss_aes_256_cbc]}, + {rsa_psk, [], [rsa_psk_3des_ede_cbc, + rsa_psk_rc4_128, + rsa_psk_aes_128_cbc, + rsa_psk_aes_256_cbc + ]}, + {dh_anon, [], [dh_anon_rc4_128, + dh_anon_3des_ede_cbc, + dh_anon_aes_128_cbc, + dh_anon_aes_128_gcm, + dh_anon_aes_256_cbc, + dh_anon_aes_256_gcm]}, + {ecdh_anon, [], [ecdh_anon_3des_ede_cbc, + ecdh_anon_aes_128_cbc, + ecdh_anon_aes_256_cbc + ]}, + {srp_anon, [], [srp_anon_3des_ede_cbc, + srp_anon_aes_128_cbc, + srp_anon_aes_256_cbc]}, + {psk, [], [psk_3des_ede_cbc, + psk_rc4_128, + psk_aes_128_cbc, + psk_aes_128_ccm, + psk_aes_128_ccm_8, + psk_aes_256_cbc, + psk_aes_256_ccm, + psk_aes_256_ccm_8 + ]}, + {dhe_psk, [], [dhe_psk_3des_ede_cbc, + dhe_psk_rc4_128, + dhe_psk_aes_128_cbc, + dhe_psk_aes_128_ccm, + dhe_psk_aes_128_ccm_8, + dhe_psk_aes_256_cbc, + dhe_psk_aes_256_ccm, + dhe_psk_aes_256_ccm_8 + ]}, + {ecdhe_psk, [], [ecdhe_psk_3des_ede_cbc, + ecdhe_psk_rc4_128, + ecdhe_psk_aes_128_cbc, + ecdhe_psk_aes_128_ccm, + ecdhe_psk_aes_128_ccm_8, + ecdhe_psk_aes_256_cbc + ]} + ]. + +kex() -> + rsa() ++ ecdsa() ++ dss() ++ anonymous(). + +rsa() -> + [{group, dhe_rsa}, + {group, ecdhe_rsa}, + {group, rsa}, + {group, srp_rsa}, + {group, rsa_psk} + ]. + +ecdsa() -> + [{group, ecdhe_ecdsa}]. + +dss() -> + [{group, dhe_dss}, + {group, srp_dss}]. + +anonymous() -> + [{group, dh_anon}, + {group, ecdh_anon}, + {group, psk}, + {group, dhe_psk}, + {group, ecdhe_psk}, + {group, srp_anon} + ]. + +init_per_suite(Config) -> + catch crypto:stop(), + try crypto:start() of + ok -> + ssl_test_lib:clean_start(), + Config + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(_Config) -> + ssl:stop(), + application:stop(crypto). + + +init_per_group(GroupName, Config) when GroupName == ecdh_anon; + GroupName == ecdhe_rsa; + GroupName == ecdhe_psk -> + case proplists:get_bool(ecdh, proplists:get_value(public_keys, crypto:supports())) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing EC crypto support"} + end; +init_per_group(ecdhe_ecdsa = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(ecdh, PKAlg) andalso lists:member(ecdsa, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing EC crypto support"} + end; +init_per_group(dhe_dss = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dss, PKAlg) andalso lists:member(dh, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing DSS crypto support"} + end; +init_per_group(srp_dss = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dss, PKAlg) andalso lists:member(srp, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing DSS_SRP crypto support"} + end; +init_per_group(GroupName, Config) when GroupName == srp_anon; + GroupName == srp_rsa -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(srp, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing SRP crypto support"} + end; +init_per_group(dhe_psk = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dh, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing SRP crypto support"} + end; +init_per_group(GroupName, Config0) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:init_tls_version(GroupName, end_per_group(GroupName, Config0)); + false -> + init_certs(GroupName, Config0) + end. + +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. + +init_per_testcase(TestCase, Config) when TestCase == psk_3des_ede_cbc; + TestCase == srp_anon_3des_ede_cbc; + TestCase == dhe_psk_3des_ede_cbc; + TestCase == ecdhe_psk_3des_ede_cbc; + TestCase == srp_rsa_3des_ede_cbc; + TestCase == srp_dss_3des_ede_cbc; + TestCase == rsa_psk_3des_ede_cbc; + TestCase == rsa_3des_ede_cbc; + TestCase == dhe_rsa_3des_ede_cbc; + TestCase == dhe_dss_3des_ede_cbc; + TestCase == ecdhe_rsa_3des_ede_cbc; + TestCase == srp_anon_dss_3des_ede_cbc; + TestCase == dh_anon_3des_ede_cbc; + TestCase == ecdh_anon_3des_ede_cbc; + TestCase == ecdhe_ecdsa_3des_ede_cbc -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(des_ede3, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing 3DES crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_rc4_128; + TestCase == ecdhe_psk_rc4_128; + TestCase == dhe_psk_rc4_128; + TestCase == rsa_psk_rc4_128; + TestCase == rsa_rc4_128; + TestCase == ecdhe_rsa_rc4_128; + TestCase == ecdhe_ecdsa_rc4_128; + TestCase == dh_anon_rc4_128 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(rc4, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing RC4 crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_aes_128_ccm_8; + TestCase == rsa_psk_aes_128_ccm_8; + TestCase == psk_aes_128_ccm_8; + TestCase == dhe_psk_aes_128_ccm_8; + TestCase == ecdhe_psk_aes_128_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_128_ccm, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing AES_128_CCM crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_aes_256_ccm_8; + TestCase == rsa_psk_aes_256_ccm_8; + TestCase == psk_aes_256_ccm_8; + TestCase == dhe_psk_aes_256_ccm_8; + TestCase == ecdhe_psk_aes_256_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_256_ccm, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing AES_256_CCM crypto support"} + end; +init_per_testcase(TestCase, Config) -> + Cipher = ssl_test_lib:test_cipher(TestCase, Config), + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(Cipher, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, {Cipher, SupCiphers}} + end. + +end_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Initializtion ------------------------------------------ +%%-------------------------------------------------------------------- + +init_certs(srp_rsa, Config) -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(rsa, rsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}} | ServerOpts], + client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(srp_anon, Config) -> + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}}], + client_config => [{srp_identity, {"Test-User", "secret"}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(rsa_psk, Config) -> + ClientExt = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, + [[],[],[{extensions, ClientExt}]]}], + Config, "_peer_keyEncipherment"), + PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}} | ServerOpts], + client_config => [{psk_identity, "Test-User"}, + {user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(rsa, Config) -> + ClientExt = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, + [[],[],[{extensions, ClientExt}]]}], + Config, "_peer_keyEncipherment"), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(dhe_dss, Config) -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(dsa, dsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(srp_dss, Config) -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(dsa, dsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}} | ServerOpts], + client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == dhe_rsa; + GroupName == ecdhe_rsa -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(rsa, rsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == dhe_ecdsa; + GroupName == ecdhe_ecdsa -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(ecdsa, ecdsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == psk; + GroupName == dhe_psk; + GroupName == ecdhe_psk -> + PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}}], + client_config => [{psk_identity, "Test-User"}, + {user_lookup_fun, {fun ssl_test_lib:user_lookup/3, PskSharedSecret}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(srp, Config) -> + [{tls_config, #{server_config => [{user_lookup_fun, {fun ssl_test_lib:user_lookup/3, undefined}}], + client_config => [{srp_identity, {"Test-User", "secret"}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(_GroupName, Config) -> + %% Anonymous does not need certs + [{tls_config, #{server_config => [], + client_config => []}} | + proplists:delete(tls_config, Config)]. + +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% SRP -------------------------------------------------------- +%%-------------------------------------------------------------------- +srp_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, '3des_ede_cbc', Config). + +srp_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, 'aes_128_cbc', Config). + +srp_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, 'aes_256_cbc', Config). + +srp_dss_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, '3des_ede_cbc', Config). + +srp_dss_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, 'aes_128_cbc', Config). + +srp_dss_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, 'aes_256_cbc', Config). + +%%-------------------------------------------------------------------- +%% PSK -------------------------------------------------------- +%%-------------------------------------------------------------------- +rsa_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, '3des_ede_cbc', Config). + +rsa_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_cbc', Config). + +rsa_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_ccm', Config). + +rsa_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_ccm_8', Config). + +rsa_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_cbc', Config). + +rsa_psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_ccm', Config). + +rsa_psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_ccm_8', Config). + +rsa_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'rc4_128', Config). + +%%-------------------------------------------------------------------- +%% RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +rsa_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'des_cbc', Config). + +rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, '3des_ede_cbc', Config). + +rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_128_cbc', Config). + +rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_256_cbc', Config). + +rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_128_gcm', Config). + +rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_256_gcm', Config). + +rsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'rc4_128', Config). +%%-------------------------------------------------------------------- +%% DHE_RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +dhe_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, '3des_ede_cbc', Config). + +dhe_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_128_cbc', Config). + +dhe_rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_128_gcm', Config). + +dhe_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_256_cbc', Config). + +dhe_rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_256_gcm', Config). + +dhe_rsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'chacha20_poly1305', Config). +%%-------------------------------------------------------------------- +%% ECDHE_RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +ecdhe_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, '3des_ede_cbc', Config). + +ecdhe_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_128_cbc', Config). + +ecdhe_rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_128_gcm', Config). + +ecdhe_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_256_cbc', Config). + +ecdhe_rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_256_gcm', Config). + +ecdhe_rsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'rc4_128', Config). + +ecdhe_rsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'chacha20_poly1305', Config). + +%%-------------------------------------------------------------------- +%% ECDHE_ECDSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +ecdhe_ecdsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'rc4_128', Config). + +ecdhe_ecdsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, '3des_ede_cbc', Config). + +ecdhe_ecdsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_cbc', Config). + +ecdhe_ecdsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_gcm', Config). + +ecdhe_ecdsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_cbc', Config). + +ecdhe_ecdsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_gcm', Config). + +ecdhe_ecdsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'chacha20_poly1305', Config). +%%-------------------------------------------------------------------- +%% DHE_DSS -------------------------------------------------------- +%%-------------------------------------------------------------------- +dhe_dss_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'des_cbc', Config). + +dhe_dss_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, '3des_ede_cbc', Config). + +dhe_dss_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_128_cbc', Config). + +dhe_dss_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_256_cbc', Config). + +dhe_dss_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_128_gcm', Config). + +dhe_dss_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_256_gcm', Config). + +%%-------------------------------------------------------------------- +%% Anonymous -------------------------------------------------------- +%%-------------------------------------------------------------------- +dh_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, '3des_ede_cbc', Config). + +dh_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_128_cbc', Config). + +dh_anon_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_128_gcm', Config). + +dh_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_256_cbc', Config). + +dh_anon_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_256_gcm', Config). + +dh_anon_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'rc4_128', Config). + +ecdh_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, '3des_ede_cbc', Config). + +ecdh_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, 'aes_128_cbc', Config). + +ecdh_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, 'aes_256_cbc', Config). + +srp_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, '3des_ede_cbc', Config). + +srp_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, 'aes_128_cbc', Config). + +srp_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, 'aes_256_cbc', Config). + +dhe_psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'des_cbc', Config). + +dhe_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'rc4_128', Config). + +dhe_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, '3des_ede_cbc', Config). + +dhe_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_cbc', Config). + +dhe_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_cbc', Config). + +dhe_psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_gcm', Config). + +dhe_psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_gcm', Config). + +dhe_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_ccm', Config). + +dhe_psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_ccm', Config). + +dhe_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_ccm_8', Config). + +dhe_psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_ccm_8', Config). + +ecdhe_psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'des_cbc', Config). + +ecdhe_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'rc4_128', Config). + +ecdhe_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, '3des_ede_cbc', Config). + +ecdhe_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_cbc', Config). + +ecdhe_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_256_cbc', Config). + +ecdhe_psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_gcm', Config). + +ecdhe_psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_256_gcm', Config). + +ecdhe_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_ccm', Config). + +ecdhe_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_ccm_8', Config). + +psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'des_cbc', Config). + +psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(psk, 'rc4_128', Config). + +psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, '3des_ede_cbc', Config). + +psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_cbc', Config). + +psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_cbc', Config). + +psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_gcm', Config). + +psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_gcm', Config). + +psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_ccm', Config). + +psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_ccm', Config). + +psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_ccm_8', Config). + +psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_ccm_8', Config). + +%%-------------------------------------------------------------------- +%% Internal functions ---------------------------------------------- +%%-------------------------------------------------------------------- +run_ciphers_test(Kex, Cipher, Config) -> + Version = ssl_test_lib:protocol_version(Config), + TestCiphers = test_ciphers(Kex, Cipher, Version), + + case TestCiphers of + [_|_] -> + lists:foreach(fun(TestCipher) -> + cipher_suite_test(TestCipher, Version, Config) + end, TestCiphers); + [] -> + {skip, {not_sup, Kex, Cipher, Version}} + end. + +cipher_suite_test(ErlangCipherSuite, Version, Config) -> + #{server_config := SOpts, + client_config := COpts} = proplists:get_value(tls_config, Config), + ServerOpts = ssl_test_lib:ssl_options(SOpts, Config), + ClientOpts = ssl_test_lib:ssl_options(COpts, Config), + ct:log("Testing CipherSuite ~p~n", [ErlangCipherSuite]), + ct:log("Server Opts ~p~n", [ServerOpts]), + ct:log("Client Opts ~p~n", [ClientOpts]), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + ConnectionInfo = {ok, {Version, ErlangCipherSuite}}, + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, + {options, [{versions, [Version]}, {ciphers, [ErlangCipherSuite]} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, + {options, [{versions, [Version]}, {ciphers, [ErlangCipherSuite]} | + ClientOpts]}]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + + +test_ciphers(Kex, Cipher, Version) -> + ssl:filter_cipher_suites(ssl:cipher_suites(all, Version) ++ ssl:cipher_suites(anonymous, Version), + [{key_exchange, + fun(Kex0) when Kex0 == Kex -> true; + (_) -> false + end}, + {cipher, + fun(Cipher0) when Cipher0 == Cipher -> true; + (_) -> false + end}]). + diff --git a/lib/ssl/test/ssl_dist_bench_SUITE.erl b/lib/ssl/test/ssl_dist_bench_SUITE.erl index 618ad0789e..1fea6f6f72 100644 --- a/lib/ssl/test/ssl_dist_bench_SUITE.erl +++ b/lib/ssl/test/ssl_dist_bench_SUITE.erl @@ -32,6 +32,7 @@ -export( [setup/1, roundtrip/1, + sched_utilization/1, throughput_0/1, throughput_64/1, throughput_1024/1, @@ -48,14 +49,19 @@ suite() -> [{ct_hooks, [{ts_install_cth, [{nodenames, 2}]}]}]. -all() -> [{group, ssl}, {group, plain}]. +all() -> + [{group, ssl}, + {group, crypto}, + {group, plain}]. groups() -> [{ssl, all_groups()}, + {crypto, all_groups()}, {plain, all_groups()}, %% {setup, [{repeat, 1}], [setup]}, {roundtrip, [{repeat, 1}], [roundtrip]}, + {sched_utilization,[{repeat, 1}], [sched_utilization]}, {throughput, [{repeat, 1}], [throughput_0, throughput_64, @@ -69,7 +75,9 @@ groups() -> all_groups() -> [{group, setup}, {group, roundtrip}, - {group, throughput}]. + {group, throughput}, + {group, sched_utilization} + ]. init_per_suite(Config) -> Digest = sha1, @@ -160,6 +168,17 @@ end_per_suite(Config) -> init_per_group(ssl, Config) -> [{ssl_dist, true}, {ssl_dist_prefix, "SSL"}|Config]; +init_per_group(crypto, Config) -> + case inet_crypto_dist:is_supported() of + true -> + [{ssl_dist, false}, {ssl_dist_prefix, "Crypto"}, + {ssl_dist_args, + "-proto_dist inet_crypto " + "-inet_crypto '#{secret => \"123456\"}'"} + |Config]; + false -> + {skip, "Not supported on this OTP version"} + end; init_per_group(plain, Config) -> [{ssl_dist, false}, {ssl_dist_prefix, "Plain"}|Config]; init_per_group(_GroupName, Config) -> @@ -351,6 +370,120 @@ roundtrip_client(Pid, Mon, StartTime, N) -> roundtrip_client(Pid, Mon, StartTime, N - 1) end. +%%--------------------------------------- +%% Scheduler utilization at constant load + + +sched_utilization(Config) -> + run_nodepair_test( + fun(A, B, Prefix, HA, HB) -> + sched_utilization(A, B, Prefix, HA, HB, proplists:get_value(ssl_dist, Config)) + end, Config). + +sched_utilization(A, B, Prefix, HA, HB, SSL) -> + [] = ssl_apply(HA, erlang, nodes, []), + [] = ssl_apply(HB, erlang, nodes, []), + {ClientMsacc, ServerMsacc, Msgs} = + ssl_apply(HA, fun () -> sched_util_runner(A, B, SSL) end), + [B] = ssl_apply(HA, erlang, nodes, []), + [A] = ssl_apply(HB, erlang, nodes, []), + msacc:print(ClientMsacc), + msacc:print(ServerMsacc), + ct:pal("Got ~p busy_dist_port msgs",[length(Msgs)]), + ct:log("Stats of B from A: ~p", + [ssl_apply(HA, net_kernel, node_info, [B])]), + ct:log("Stats of A from B: ~p", + [ssl_apply(HB, net_kernel, node_info, [A])]), + SchedUtilClient = + round(10000 * msacc:stats(system_runtime,ClientMsacc) / + msacc:stats(system_realtime,ClientMsacc)), + SchedUtilServer = + round(10000 * msacc:stats(system_runtime,ServerMsacc) / + msacc:stats(system_realtime,ServerMsacc)), + Verdict = + case Msgs of + [] -> + ""; + _ -> + " ???" + end, + {comment, ClientComment} = + report(Prefix ++ " Sched Utilization Client" ++ Verdict, + SchedUtilClient, "/100 %" ++ Verdict), + {comment, ServerComment} = + report(Prefix++" Sched Utilization Server" ++ Verdict, + SchedUtilServer, "/100 %" ++ Verdict), + {comment, "Client " ++ ClientComment ++ ", Server " ++ ServerComment}. + +%% Runs on node A and spawns a server on node B +%% We want to avoid getting busy_dist_port as it hides the true SU usage +%% of the receiver and sender. +sched_util_runner(A, B, true) -> + sched_util_runner(A, B, 250); +sched_util_runner(A, B, false) -> + sched_util_runner(A, B, 250); +sched_util_runner(A, B, Senders) -> + Payload = payload(5), + [A] = rpc:call(B, erlang, nodes, []), + ServerPids = + [erlang:spawn_link( + B, fun () -> throughput_server() end) + || _ <- lists:seq(1, Senders)], + ServerMsacc = + erlang:spawn( + B, + fun() -> + receive + {start,Pid} -> + msacc:start(10000), + receive + {done,Pid} -> + Pid ! {self(),msacc:stats()} + end + end + end), + erlang:system_monitor(self(),[busy_dist_port]), + %% We spawn 250 senders which should mean that we + %% have a load of 250 msgs/msec + [spawn_link( + fun() -> + throughput_client(Pid, Payload) + end) || Pid <- ServerPids], + %% + receive after 1000 -> ok end, + ServerMsacc ! {start,self()}, + msacc:start(10000), + ClientMsaccStats = msacc:stats(), + receive after 1000 -> ok end, + ServerMsacc ! {done,self()}, + ServerMsaccStats = receive {ServerMsacc,Stats} -> Stats end, + %% + {ClientMsaccStats,ServerMsaccStats, flush()}. + +flush() -> + receive + M -> + [M | flush()] + after 0 -> + [] + end. + +throughput_server() -> + receive _ -> ok end, + receive _ -> ok end, + receive _ -> ok end, + receive _ -> ok end, + receive _ -> ok end, + receive _ -> ok end, + receive _ -> ok end, + receive _ -> ok end, + receive _ -> ok end, + receive _ -> ok end, + throughput_server(). + +throughput_client(Pid, Payload) -> + Pid ! Payload, + receive after 1 -> throughput_client(Pid, Payload) end. %%----------------- %% Throughput speed @@ -425,15 +558,20 @@ throughput(A, B, Prefix, HA, HB, Packets, Size) -> + byte_size(erlang:term_to_binary([0|<<>>])), % Benchmark overhead Bytes = Packets * (Size + Overhead), io:format("~w bytes, ~.4g s~n", [Bytes,Time/1000000]), + SizeString = integer_to_list(Size), ClientMsaccStats =:= undefined orelse - io:format( - "Sender core usage ratio: ~.4g ns/byte~n", - [msacc:stats(system_runtime, ClientMsaccStats)*1000/Bytes]), + report( + Prefix ++ " Sender_RelativeCoreLoad_" ++ SizeString, + round(msacc:stats(system_runtime, ClientMsaccStats) + * 1000000 / Bytes), + "ps/byte"), ServerMsaccStats =:= undefined orelse begin - io:format( - "Receiver core usage ratio: ~.4g ns/byte~n", - [msacc:stats(system_runtime, ServerMsaccStats)*1000/Bytes]), + report( + Prefix ++ " Receiver_RelativeCoreLoad_" ++ SizeString, + round(msacc:stats(system_runtime, ServerMsaccStats) + * 1000000 / Bytes), + "ps/byte"), msacc:print(ServerMsaccStats) end, io:format("******* ClientProf:~n", []), prof_print(ClientProf), @@ -441,7 +579,7 @@ throughput(A, B, Prefix, HA, HB, Packets, Size) -> io:format("******* Server GC Before:~n~p~n", [Server_GC_Before]), io:format("******* Server GC After:~n~p~n", [Server_GC_After]), Speed = round((Bytes * 1000000) / (1024 * Time)), - report(Prefix++" Throughput_"++integer_to_list(Size), Speed, "kB/s"). + report(Prefix ++ " Throughput_" ++ SizeString, Speed, "kB/s"). %% Runs on node A and spawns a server on node B throughput_runner(A, B, Rounds, Size) -> @@ -449,11 +587,12 @@ throughput_runner(A, B, Rounds, Size) -> [A] = rpc:call(B, erlang, nodes, []), ClientPid = self(), ServerPid = - erlang:spawn( + erlang:spawn_opt( B, - fun () -> throughput_server(ClientPid, Rounds) end), + fun () -> throughput_server(ClientPid, Rounds) end, + [{message_queue_data, off_heap}]), ServerMon = erlang:monitor(process, ServerPid), - msacc:available() andalso + msacc_available() andalso begin msacc:stop(), msacc:reset(), @@ -465,7 +604,7 @@ throughput_runner(A, B, Rounds, Size) -> throughput_client(ServerPid, ServerMon, Payload, Rounds), prof_stop(), MsaccStats = - case msacc:available() of + case msacc_available() of true -> MStats = msacc:stats(), msacc:stop(), @@ -505,7 +644,7 @@ throughput_server(Pid, N) -> GC_Before = get_server_gc_info(), %% dbg:tracer(port, dbg:trace_port(file, "throughput_server_gc.log")), %% dbg:p(TLSDistReceiver, garbage_collection), - msacc:available() andalso + msacc_available() andalso begin msacc:stop(), msacc:reset(), @@ -518,7 +657,7 @@ throughput_server(Pid, N) -> throughput_server_loop(_Pid, GC_Before, 0) -> prof_stop(), MsaccStats = - case msacc:available() of + case msacc_available() of true -> msacc:stop(), MStats = msacc:stats(), @@ -535,8 +674,13 @@ throughput_server_loop(_Pid, GC_Before, 0) -> server_gc_after => get_server_gc_info()}); throughput_server_loop(Pid, GC_Before, N) -> receive - {Pid, N, _} -> - throughput_server_loop(Pid, GC_Before, N-1) + Msg -> + case Msg of + {Pid, N, _} -> + throughput_server_loop(Pid, GC_Before, N - 1); + Other -> + erlang:error({self(),?FUNCTION_NAME,Other}) + end end. get_server_gc_info() -> @@ -676,7 +820,7 @@ get_node_args(Tag, Config) -> true -> proplists:get_value(Tag, Config); false -> - "" + proplists:get_value(ssl_dist_args, Config, "") end. @@ -731,3 +875,6 @@ report(Name, Value, Unit) -> term_to_string(Term) -> unicode:characters_to_list( io_lib:write(Term, [{encoding, unicode}])). + +msacc_available() -> + msacc:available(). diff --git a/lib/ssl/test/ssl_dist_test_lib.erl b/lib/ssl/test/ssl_dist_test_lib.erl index 1b9c853fc4..b8b805b2c4 100644 --- a/lib/ssl/test/ssl_dist_test_lib.erl +++ b/lib/ssl/test/ssl_dist_test_lib.erl @@ -144,6 +144,8 @@ mk_node_cmdline(ListenPort, Name, Args) -> ++ integer_to_list(ListenPort) ++ " " ++ Args ++ " " ++ "-env ERL_CRASH_DUMP " ++ Pwd ++ "/erl_crash_dump." ++ Name ++ " " + ++ "-kernel inet_dist_connect_options \"[{recbuf,12582912},{sndbuf,12582912},{high_watermark,8388608},{low_watermark,4194304}]\" " + ++ "-kernel inet_dist_listen_options \"[{recbuf,12582912},{sndbuf,12582912},{high_watermark,8388608},{low_watermark,4194304}]\" " ++ "-kernel error_logger \"{file,\\\"" ++ Pwd ++ "/error_log." ++ Name ++ "\\\"}\" " ++ "-setcookie " ++ atom_to_list(erlang:get_cookie()). @@ -179,8 +181,9 @@ check_ssl_node_up(Socket, Name, Bin) -> Parent = self(), Go = make_ref(), %% Spawn connection handler on test server side - Pid = spawn_link( + Pid = spawn( fun () -> + link(group_leader()), receive Go -> ok end, process_flag(trap_exit, true), tstsrvr_con_loop(Name, Socket, Parent) diff --git a/lib/ssl/test/ssl_eqc_SUITE.erl b/lib/ssl/test/ssl_eqc_SUITE.erl new file mode 100644 index 0000000000..bd36d35c02 --- /dev/null +++ b/lib/ssl/test/ssl_eqc_SUITE.erl @@ -0,0 +1,58 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015-2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% + +-module(ssl_eqc_SUITE). + +-compile(export_all). +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- + +all() -> + [ + tls_handshake_encoding + ]. + +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + ct_property_test:init_per_suite(Config). +end_per_suite(Config) -> + Config. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_,Config) -> + Config. + +init_per_testcase(_, Config0) -> + Config0. + +end_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- + +tls_handshake_encoding(Config) when is_list(Config) -> + %% manual test: proper:quickcheck(ssl_eqc_handshake:prop_tls_hs_encode_decode()). + true = ct_property_test:quickcheck(ssl_eqc_handshake:prop_tls_hs_encode_decode(), + Config). diff --git a/lib/ssl/test/ssl_handshake_SUITE.erl b/lib/ssl/test/ssl_handshake_SUITE.erl index 1fa6029963..1b432970b6 100644 --- a/lib/ssl/test/ssl_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_handshake_SUITE.erl @@ -25,6 +25,7 @@ -compile(export_all). -include_lib("common_test/include/ct.hrl"). +-include("ssl_alert.hrl"). -include("ssl_handshake.hrl"). -include("ssl_internal.hrl"). -include("tls_handshake.hrl"). @@ -42,7 +43,7 @@ all() -> [decode_hello_handshake, decode_empty_server_sni_correctly, select_proper_tls_1_2_rsa_default_hashsign, ignore_hassign_extension_pre_tls_1_2, - unorded_chain, + unorded_chain, signature_algorithms, encode_decode_srp]. %%-------------------------------------------------------------------- @@ -57,7 +58,9 @@ init_per_group(_GroupName, Config) -> end_per_group(_,Config) -> Config. -init_per_testcase(ignore_hassign_extension_pre_tls_1_2, Config0) -> +init_per_testcase(TC, Config0) when + TC =:= ignore_hassign_extension_pre_tls_1_2 orelse + TC =:= signature_algorithms -> catch crypto:stop(), try crypto:start() of ok -> @@ -106,15 +109,13 @@ decode_hello_handshake(_Config) -> #ssl_options{}), {Hello, _Data} = hd(Records), - #renegotiation_info{renegotiated_connection = <<0>>} - = (Hello#server_hello.extensions)#hello_extensions.renegotiation_info. - + Extensions = Hello#server_hello.extensions, + #{renegotiation_info := #renegotiation_info{renegotiated_connection = <<0>>}} = Extensions. decode_single_hello_extension_correctly(_Config) -> Renegotiation = <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(1), 0>>, - Extensions = ssl_handshake:decode_hello_extensions(Renegotiation), - #renegotiation_info{renegotiated_connection = <<0>>} - = Extensions#hello_extensions.renegotiation_info. + Extensions = ssl_handshake:decode_extensions(Renegotiation, {3,3}, undefined), + #{renegotiation_info := #renegotiation_info{renegotiated_connection = <<0>>}} = Extensions. decode_supported_elliptic_curves_hello_extension_correctly(_Config) -> % List of supported and unsupported curves (RFC4492:S5.1.1) @@ -125,37 +126,34 @@ decode_supported_elliptic_curves_hello_extension_correctly(_Config) -> Len = ListLen + 2, Extension = <<?UINT16(?ELLIPTIC_CURVES_EXT), ?UINT16(Len), ?UINT16(ListLen), EllipticCurveList/binary>>, % after decoding we should see only valid curves - #hello_extensions{elliptic_curves = DecodedCurves} = ssl_handshake:decode_hello_extensions(Extension), - #elliptic_curves{elliptic_curve_list = [?sect233k1, ?sect193r2]} = DecodedCurves. + Extensions = ssl_handshake:decode_hello_extensions(Extension, {3,2}, {3,2}, client), + #{elliptic_curves := #elliptic_curves{elliptic_curve_list = [?sect233k1, ?sect193r2]}} = Extensions. decode_unknown_hello_extension_correctly(_Config) -> FourByteUnknown = <<16#CA,16#FE, ?UINT16(4), 3, 0, 1, 2>>, Renegotiation = <<?UINT16(?RENEGOTIATION_EXT), ?UINT16(1), 0>>, - Extensions = ssl_handshake:decode_hello_extensions(<<FourByteUnknown/binary, Renegotiation/binary>>), - #renegotiation_info{renegotiated_connection = <<0>>} - = Extensions#hello_extensions.renegotiation_info. + Extensions = ssl_handshake:decode_hello_extensions(<<FourByteUnknown/binary, Renegotiation/binary>>, {3,2}, {3,2}, client), + #{renegotiation_info := #renegotiation_info{renegotiated_connection = <<0>>}} = Extensions. + encode_single_hello_sni_extension_correctly(_Config) -> - Exts = #hello_extensions{sni = #sni{hostname = "test.com"}}, SNI = <<16#00, 16#00, 16#00, 16#0d, 16#00, 16#0b, 16#00, 16#00, 16#08, $t, $e, $s, $t, $., $c, $o, $m>>, ExtSize = byte_size(SNI), HelloExt = <<ExtSize:16/unsigned-big-integer, SNI/binary>>, - Encoded = ssl_handshake:encode_hello_extensions(Exts), + Encoded = ssl_handshake:encode_extensions([#sni{hostname = "test.com"}]), HelloExt = Encoded. decode_single_hello_sni_extension_correctly(_Config) -> - Exts = #hello_extensions{sni = #sni{hostname = "test.com"}}, SNI = <<16#00, 16#00, 16#00, 16#0d, 16#00, 16#0b, 16#00, 16#00, 16#08, $t, $e, $s, $t, $., $c, $o, $m>>, - Decoded = ssl_handshake:decode_hello_extensions(SNI), - Exts = Decoded. + Decoded = ssl_handshake:decode_hello_extensions(SNI, {3,3}, {3,3}, client), + #{sni := #sni{hostname = "test.com"}} = Decoded. decode_empty_server_sni_correctly(_Config) -> - Exts = #hello_extensions{sni = #sni{hostname = ""}}, SNI = <<?UINT16(?SNI_EXT),?UINT16(0)>>, - Decoded = ssl_handshake:decode_hello_extensions(SNI), - Exts = Decoded. + Decoded = ssl_handshake:decode_hello_extensions(SNI, {3,3}, {3,3}, server), + #{sni := #sni{hostname = ""}} = Decoded. select_proper_tls_1_2_rsa_default_hashsign(_Config) -> @@ -170,11 +168,11 @@ ignore_hassign_extension_pre_tls_1_2(Config) -> Opts = proplists:get_value(server_opts, Config), CertFile = proplists:get_value(certfile, Opts), [{_, Cert, _}] = ssl_test_lib:pem_to_der(CertFile), - HashSigns = #hash_sign_algos{hash_sign_algos = [{sha512, rsa}, {sha, dsa}]}, - {sha512, rsa} = ssl_handshake:select_hashsign(HashSigns, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,3}), {3,3}), + HashSigns = #hash_sign_algos{hash_sign_algos = [{sha512, rsa}, {sha, dsa}, {sha, rsa}]}, + {sha512, rsa} = ssl_handshake:select_hashsign({HashSigns, undefined}, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,3}), {3,3}), %%% Ignore - {md5sha, rsa} = ssl_handshake:select_hashsign(HashSigns, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,2}), {3,2}), - {md5sha, rsa} = ssl_handshake:select_hashsign(HashSigns, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,0}), {3,0}). + {md5sha, rsa} = ssl_handshake:select_hashsign({HashSigns, undefined}, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,2}), {3,2}), + {md5sha, rsa} = ssl_handshake:select_hashsign({HashSigns, undefined}, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,0}), {3,0}). unorded_chain(Config) when is_list(Config) -> DefConf = ssl_test_lib:default_cert_chain_conf(), @@ -195,30 +193,78 @@ unorded_chain(Config) when is_list(Config) -> ssl_certificate:certificate_chain(PeerCert, ets:new(foo, []), ExtractedCerts, UnordedChain). encode_decode_srp(_Config) -> - Exts = #hello_extensions{ - srp = #srp{username = <<"foo">>}, - sni = #sni{hostname = "bar"}, - renegotiation_info = undefined, - signature_algs = undefined, - alpn = undefined, - next_protocol_negotiation = undefined, - ec_point_formats = undefined, - elliptic_curves = undefined - }, - EncodedExts = <<0,20, % Length + Exts = #{srp => #srp{username = <<"foo">>}, + sni => #sni{hostname = "bar"}, + renegotiation_info => undefined, + signature_algs => undefined, + alpn => undefined, + next_protocol_negotiation => undefined, + ec_point_formats => undefined, + elliptic_curves => undefined + }, + EncodedExts0 = <<0,20, % Length + 0,12, % SRP extension + 0,4, % Length + 3, % srp_I length + 102,111,111, % username = "foo" 0,0, % SNI extension 0,8, % Length 0,6, % ServerNameLength 0, % NameType (host_name) 0,3, % HostNameLength - 98,97,114, % hostname = "bar" - 0,12, % SRP extension - 0,4, % Length - 3, % srp_I length - 102,111,111>>, % username = "foo" - EncodedExts = ssl_handshake:encode_hello_extensions(Exts), - Exts = ssl_handshake:decode_hello_extensions({client, EncodedExts}). + 98,97,114>>, % hostname = "bar" + EncodedExts0 = <<?UINT16(_),EncodedExts/binary>> = + ssl_handshake:encode_hello_extensions(Exts), + Exts = ssl_handshake:decode_hello_extensions(EncodedExts, {3,3}, {3,3}, client). +signature_algorithms(Config) -> + Opts = proplists:get_value(server_opts, Config), + CertFile = proplists:get_value(certfile, Opts), + io:format("Cert = ~p~n", [CertFile]), + [{_, Cert, _}] = ssl_test_lib:pem_to_der(CertFile), + HashSigns0 = #hash_sign_algos{ + hash_sign_algos = [{sha512, rsa}, + {sha, dsa}, + {sha, rsa}]}, + Schemes0 = #signature_algorithms_cert{ + signature_scheme_list = [rsa_pkcs1_sha1, + ecdsa_sha1]}, + {sha512, rsa} = ssl_handshake:select_hashsign( + {HashSigns0, Schemes0}, + Cert, ecdhe_rsa, + tls_v1:default_signature_algs({3,3}), + {3,3}), + HashSigns1 = #hash_sign_algos{ + hash_sign_algos = [{sha, dsa}, + {sha, rsa}]}, + {sha, rsa} = ssl_handshake:select_hashsign( + {HashSigns1, Schemes0}, + Cert, ecdhe_rsa, + tls_v1:default_signature_algs({3,3}), + {3,3}), + Schemes1 = #signature_algorithms_cert{ + signature_scheme_list = [rsa_pkcs1_sha256, + ecdsa_sha1]}, + %% Signature not supported + #alert{} = ssl_handshake:select_hashsign( + {HashSigns1, Schemes1}, + Cert, ecdhe_rsa, + tls_v1:default_signature_algs({3,3}), + {3,3}), + %% No scheme, hashsign is used + {sha, rsa} = ssl_handshake:select_hashsign( + {HashSigns1, undefined}, + Cert, ecdhe_rsa, + tls_v1:default_signature_algs({3,3}), + {3,3}), + HashSigns2 = #hash_sign_algos{ + hash_sign_algos = [{sha, dsa}]}, + %% Signature not supported + #alert{} = ssl_handshake:select_hashsign( + {HashSigns2, Schemes1}, + Cert, ecdhe_rsa, + tls_v1:default_signature_algs({3,3}), + {3,3}). %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ diff --git a/lib/ssl/test/ssl_npn_hello_SUITE.erl b/lib/ssl/test/ssl_npn_hello_SUITE.erl index 35af666e9e..46734ba180 100644 --- a/lib/ssl/test/ssl_npn_hello_SUITE.erl +++ b/lib/ssl/test/ssl_npn_hello_SUITE.erl @@ -71,44 +71,46 @@ encode_and_decode_client_hello_test(Config) -> Version = ssl_test_lib:protocol_version(Config), {[{DecodedHandshakeMessage, _Raw}], _} = tls_handshake:get_tls_handshake(Version, list_to_binary(HandShakeData), <<>>, #ssl_options{}), - NextProtocolNegotiation = (DecodedHandshakeMessage#client_hello.extensions)#hello_extensions.next_protocol_negotiation, - NextProtocolNegotiation = undefined. + Extensions = DecodedHandshakeMessage#client_hello.extensions, + #{next_protocol_negotiation := undefined} = Extensions. %%-------------------------------------------------------------------- encode_and_decode_npn_client_hello_test(Config) -> HandShakeData = create_client_handshake(#next_protocol_negotiation{extension_data = <<>>}), Version = ssl_test_lib:protocol_version(Config), {[{DecodedHandshakeMessage, _Raw}], _} = tls_handshake:get_tls_handshake(Version, list_to_binary(HandShakeData), <<>>, #ssl_options{}), - NextProtocolNegotiation = (DecodedHandshakeMessage#client_hello.extensions)#hello_extensions.next_protocol_negotiation, - NextProtocolNegotiation = #next_protocol_negotiation{extension_data = <<>>}. + Extensions = DecodedHandshakeMessage#client_hello.extensions, + #{next_protocol_negotiation := #next_protocol_negotiation{extension_data = <<>>}} = Extensions. %%-------------------------------------------------------------------- encode_and_decode_server_hello_test(Config) -> HandShakeData = create_server_handshake(undefined), Version = ssl_test_lib:protocol_version(Config), {[{DecodedHandshakeMessage, _Raw}], _} = tls_handshake:get_tls_handshake(Version, list_to_binary(HandShakeData), <<>>, #ssl_options{}), - NextProtocolNegotiation = (DecodedHandshakeMessage#server_hello.extensions)#hello_extensions.next_protocol_negotiation, - NextProtocolNegotiation = undefined. + Extensions = DecodedHandshakeMessage#server_hello.extensions, + #{next_protocol_negotiation := undefined} = Extensions. + %%-------------------------------------------------------------------- encode_and_decode_npn_server_hello_test(Config) -> HandShakeData = create_server_handshake(#next_protocol_negotiation{extension_data = <<6, "spdy/2">>}), Version = ssl_test_lib:protocol_version(Config), {[{DecodedHandshakeMessage, _Raw}], _} = tls_handshake:get_tls_handshake(Version, list_to_binary(HandShakeData), <<>>, #ssl_options{}), - NextProtocolNegotiation = (DecodedHandshakeMessage#server_hello.extensions)#hello_extensions.next_protocol_negotiation, - ct:log("~p ~n", [NextProtocolNegotiation]), - NextProtocolNegotiation = #next_protocol_negotiation{extension_data = <<6, "spdy/2">>}. + Extensions = DecodedHandshakeMessage#server_hello.extensions, + ct:log("~p ~n", [Extensions]), + #{next_protocol_negotiation := #next_protocol_negotiation{extension_data = <<6, "spdy/2">>}} = Extensions. %%-------------------------------------------------------------------- create_server_hello_with_no_advertised_protocols_test(_Config) -> - Hello = ssl_handshake:server_hello(<<>>, {3, 0}, create_connection_states(), #hello_extensions{}), - undefined = (Hello#server_hello.extensions)#hello_extensions.next_protocol_negotiation. + Hello = ssl_handshake:server_hello(<<>>, {3, 0}, create_connection_states(), #{}), + Extensions = Hello#server_hello.extensions, + #{} = Extensions. %%-------------------------------------------------------------------- create_server_hello_with_advertised_protocols_test(_Config) -> Hello = ssl_handshake:server_hello(<<>>, {3, 0}, create_connection_states(), - #hello_extensions{next_protocol_negotiation = [<<"spdy/1">>, <<"http/1.0">>, <<"http/1.1">>]}), - [<<"spdy/1">>, <<"http/1.0">>, <<"http/1.1">>] = - (Hello#server_hello.extensions)#hello_extensions.next_protocol_negotiation. + #{next_protocol_negotiation => [<<"spdy/1">>, <<"http/1.0">>, <<"http/1.1">>]}), + Extensions = Hello#server_hello.extensions, + #{next_protocol_negotiation := [<<"spdy/1">>, <<"http/1.0">>, <<"http/1.1">>]} = Extensions. %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -120,9 +122,8 @@ create_client_handshake(Npn) -> session_id = <<>>, cipher_suites = [?TLS_DHE_DSS_WITH_DES_CBC_SHA], compression_methods = "", - extensions = #hello_extensions{ - next_protocol_negotiation = Npn, - renegotiation_info = #renegotiation_info{}} + extensions = #{next_protocol_negotiation => Npn, + renegotiation_info => #renegotiation_info{}} }, Vsn). create_server_handshake(Npn) -> @@ -133,9 +134,8 @@ create_server_handshake(Npn) -> session_id = <<>>, cipher_suite = ?TLS_DHE_DSS_WITH_DES_CBC_SHA, compression_method = 1, - extensions = #hello_extensions{ - next_protocol_negotiation = Npn, - renegotiation_info = #renegotiation_info{}} + extensions = #{next_protocol_negotiation => Npn, + renegotiation_info => #renegotiation_info{}} }, Vsn). create_connection_states() -> @@ -146,5 +146,5 @@ create_connection_states() -> } }, current_read => #{secure_renegotiation => false - } + } }. diff --git a/lib/ssl/test/ssl_rfc_5869_SUITE.erl b/lib/ssl/test/ssl_rfc_5869_SUITE.erl new file mode 100644 index 0000000000..8b2d1c2082 --- /dev/null +++ b/lib/ssl/test/ssl_rfc_5869_SUITE.erl @@ -0,0 +1,316 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +-module(ssl_rfc_5869_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- +all() -> + [sha_256_basic, + sha_256_long, + sha_256_no_salt, + sha_basic, + sha_long, + sha_no_salt, + sha_default_salt + ]. + +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + catch crypto:stop(), + try crypto:start() of + ok -> + Config + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(_Config) -> + application:stop(crypto). + +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + ct:timetrap({seconds, 5}), + Config. + +end_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- + +sha_256_basic() -> + [{doc, "Basic test case with SHA-256"}]. +sha_256_basic(Config) when is_list(Config) -> + %% Hash = SHA-256 + %% IKM = 0x0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b (22 octets) + %% salt = 0x000102030405060708090a0b0c (13 octets) + %% info = 0xf0f1f2f3f4f5f6f7f8f9 (10 octets) + %% L = 42 + %% PRK = 0x077709362c2e32df0ddc3f0dc47bba63 + %% 90b6c73bb50f9c3122ec844ad7c2b3e5 (32 octets) + %% OKM = 0x3cb25f25faacd57a90434f64d0362f2a + %% 2d2d0a90cf1a5a4c5db02d56ecc4c5bf + %% 34007208d5b887185865 (42 octets) + IKM = hexstr2bin("0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b"), + Salt = hexstr2bin("000102030405060708090a0b0c"), + Info = hexstr2bin("f0f1f2f3f4f5f6f7f8f9"), + PRK = hexstr2bin("077709362c2e32df0ddc3f0dc47bba63" + "90b6c73bb50f9c3122ec844ad7c2b3e5"), + OKM = hexstr2bin("3cb25f25faacd57a90434f64d0362f2a" + "2d2d0a90cf1a5a4c5db02d56ecc4c5bf" + "34007208d5b887185865"), + hkdf_test(sha256, Salt, IKM, PRK, Info, 42, OKM). + +sha_256_long() -> + [{doc, "Test with SHA-256 and longer inputs/outputs"}]. +sha_256_long(Config) when is_list(Config) -> + %% Hash = SHA-256 + %% IKM = 0x000102030405060708090a0b0c0d0e0f + %% 101112131415161718191a1b1c1d1e1f + %% 202122232425262728292a2b2c2d2e2f + %% 303132333435363738393a3b3c3d3e3f + %% 404142434445464748494a4b4c4d4e4f (80 octets) + %% salt = 0x606162636465666768696a6b6c6d6e6f + %% 707172737475767778797a7b7c7d7e7f + %% 808182838485868788898a8b8c8d8e8f + %% 909192939495969798999a9b9c9d9e9f + %% a0a1a2a3a4a5a6a7a8a9aaabacadaeaf (80 octets) + %% info = 0xb0b1b2b3b4b5b6b7b8b9babbbcbdbebf + %% c0c1c2c3c4c5c6c7c8c9cacbcccdcecf + %% d0d1d2d3d4d5d6d7d8d9dadbdcdddedf + %% e0e1e2e3e4e5e6e7e8e9eaebecedeeef + %% f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff (80 octets) + %% L = 82 + + %% PRK = 0x06a6b88c5853361a06104c9ceb35b45c + %% ef760014904671014a193f40c15fc244 (32 octets) + %% OKM = 0xb11e398dc80327a1c8e7f78c596a4934 + %% 4f012eda2d4efad8a050cc4c19afa97c + %% 59045a99cac7827271cb41c65e590e09 + %% da3275600c2f09b8367793a9aca3db71 + %% cc30c58179ec3e87c14c01d5c1f3434f + %% 1d87 (82 octets) + IKM = hexstr2bin("000102030405060708090a0b0c0d0e0f" + "101112131415161718191a1b1c1d1e1f" + "202122232425262728292a2b2c2d2e2f" + "303132333435363738393a3b3c3d3e3f" + "404142434445464748494a4b4c4d4e4f" + ), + Salt = hexstr2bin("606162636465666768696a6b6c6d6e6f" + "707172737475767778797a7b7c7d7e7f" + "808182838485868788898a8b8c8d8e8f" + "909192939495969798999a9b9c9d9e9f" + "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" + ), + Info = hexstr2bin("b0b1b2b3b4b5b6b7b8b9babbbcbdbebf" + "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf" + "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf" + "e0e1e2e3e4e5e6e7e8e9eaebecedeeef" + "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff" + ), + PRK = hexstr2bin("06a6b88c5853361a06104c9ceb35b45c" + "ef760014904671014a193f40c15fc244"), + OKM = hexstr2bin("b11e398dc80327a1c8e7f78c596a4934" + "4f012eda2d4efad8a050cc4c19afa97c" + "59045a99cac7827271cb41c65e590e09" + "da3275600c2f09b8367793a9aca3db71" + "cc30c58179ec3e87c14c01d5c1f3434f" + "1d87" + ), + hkdf_test(sha256, Salt, IKM, PRK, Info, 82, OKM). +sha_256_no_salt() -> + [{doc, "Test with SHA-256 and zero-length salt/info"}]. +sha_256_no_salt(Config) when is_list(Config) -> + %% Hash = SHA-256 + %% IKM = 0x0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b (22 octets) + %% salt = (0 octets) + %% info = (0 octets) + %% L = 42 + + %% PRK = 0x19ef24a32c717b167f33a91d6f648bdf + %% 96596776afdb6377ac434c1c293ccb04 (32 octets) + %% OKM = 0x8da4e775a563c18f715f802a063c5a31 + %% b8a11f5c5ee1879ec3454e5f3c738d2d + %% 9d201395faa4b61a96c8 (42 octets) + IKM = hexstr2bin("0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b"), + Salt = <<>>, + Info = <<>>, + PRK = hexstr2bin("19ef24a32c717b167f33a91d6f648bdf" + "96596776afdb6377ac434c1c293ccb04"), + OKM = hexstr2bin("8da4e775a563c18f715f802a063c5a31" + "b8a11f5c5ee1879ec3454e5f3c738d2d" + "9d201395faa4b61a96c8"), + hkdf_test(sha256, Salt, IKM, PRK, Info, 42, OKM). + +sha_basic() -> + [{doc, " Basic test case with SHA-1"}]. +sha_basic(Config) when is_list(Config) -> + %% Hash = SHA-1 + %% IKM = 0x0b0b0b0b0b0b0b0b0b0b0b (11 octets) + %% salt = 0x000102030405060708090a0b0c (13 octets) + %% info = 0xf0f1f2f3f4f5f6f7f8f9 (10 octets) + %% L = 42 + + %% PRK = 0x9b6c18c432a7bf8f0e71c8eb88f4b30baa2ba243 (20 octets) + %% OKM = 0x085a01ea1b10f36933068b56efa5ad81 + %% a4f14b822f5b091568a9cdd4f155fda2 + %% c22e422478d305f3f896 (42 octets) + IKM = hexstr2bin("0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b"), + Salt = hexstr2bin("000102030405060708090a0b0c"), + Info = hexstr2bin("f0f1f2f3f4f5f6f7f8f9"), + PRK = hexstr2bin("077709362c2e32df0ddc3f0dc47bba63" + "90b6c73bb50f9c3122ec844ad7c2b3e5"), + OKM = hexstr2bin("3cb25f25faacd57a90434f64d0362f2a" + "2d2d0a90cf1a5a4c5db02d56ecc4c5bf" + "34007208d5b887185865"), + hkdf_test(sha256, Salt, IKM, PRK, Info, 42, OKM). + +sha_long() -> + [{doc, "Test with SHA-1 and longer inputs/outputs"}]. +sha_long(Config) when is_list(Config) -> + %% Hash = SHA-1 + %% IKM = 0x000102030405060708090a0b0c0d0e0f + %% 101112131415161718191a1b1c1d1e1f + %% 202122232425262728292a2b2c2d2e2f + %% 303132333435363738393a3b3c3d3e3f + %% 404142434445464748494a4b4c4d4e4f (80 octets) + %% salt = 0x606162636465666768696a6b6c6d6e6f + %% 707172737475767778797a7b7c7d7e7f + %% 808182838485868788898a8b8c8d8e8f + %% 909192939495969798999a9b9c9d9e9f + %% a0a1a2a3a4a5a6a7a8a9aaabacadaeaf (80 octets) + %% info = 0xb0b1b2b3b4b5b6b7b8b9babbbcbdbebf + %% c0c1c2c3c4c5c6c7c8c9cacbcccdcecf + %% d0d1d2d3d4d5d6d7d8d9dadbdcdddedf + %% e0e1e2e3e4e5e6e7e8e9eaebecedeeef + %% f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff (80 octets) + %% L = 82 + + %% PRK = 0x8adae09a2a307059478d309b26c4115a224cfaf6 (20 octets) + %% OKM = 0x0bd770a74d1160f7c9f12cd5912a06eb + %% ff6adcae899d92191fe4305673ba2ffe + %% 8fa3f1a4e5ad79f3f334b3b202b2173c + %% 486ea37ce3d397ed034c7f9dfeb15c5e + %% 927336d0441f4c4300e2cff0d0900b52 + %% d3b4 (82 octets) + IKM = hexstr2bin("000102030405060708090a0b0c0d0e0f" + "101112131415161718191a1b1c1d1e1f" + "202122232425262728292a2b2c2d2e2f" + "303132333435363738393a3b3c3d3e3f" + "404142434445464748494a4b4c4d4e4f" + ), + Salt = hexstr2bin("606162636465666768696a6b6c6d6e6f" + "707172737475767778797a7b7c7d7e7f" + "808182838485868788898a8b8c8d8e8f" + "909192939495969798999a9b9c9d9e9f" + "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" + ), + Info = hexstr2bin("b0b1b2b3b4b5b6b7b8b9babbbcbdbebf" + "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf" + "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf" + "e0e1e2e3e4e5e6e7e8e9eaebecedeeef" + "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff" + ), + PRK = hexstr2bin("8adae09a2a307059478d309b26c4115a224cfaf6"), + OKM = hexstr2bin("0bd770a74d1160f7c9f12cd5912a06eb" + "ff6adcae899d92191fe4305673ba2ffe" + "8fa3f1a4e5ad79f3f334b3b202b2173c" + "486ea37ce3d397ed034c7f9dfeb15c5e" + "927336d0441f4c4300e2cff0d0900b52" + "d3b4" + ), + hkdf_test(sha, Salt, IKM, PRK, Info, 82, OKM). + +sha_no_salt() -> + [{doc, "Test with SHA-1 and zero-length salt/info"}]. +sha_no_salt(Config) when is_list(Config) -> + %% Hash = SHA-1 + %% IKM = 0x0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b (22 octets) + %% salt = (0 octets) + %% info = (0 octets) + %% L = 42 + + %% PRK = 0xda8c8a73c7fa77288ec6f5e7c297786aa0d32d01 (20 octets) + %% OKM = 0x0ac1af7002b3d761d1e55298da9d0506 + %% b9ae52057220a306e07b6b87e8df21d0 + %% ea00033de03984d34918 (42 octets) + IKM = hexstr2bin("0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b"), + Salt = <<>>, + Info = <<>>, + PRK = hexstr2bin("da8c8a73c7fa77288ec6f5e7c297786aa0d32d01"), + OKM = hexstr2bin("0ac1af7002b3d761d1e55298da9d0506" + "b9ae52057220a306e07b6b87e8df21d0" + "ea00033de03984d34918"), + hkdf_test(sha, Salt, IKM, PRK, Info, 42, OKM). + + +sha_default_salt() -> + [{doc, "Test with SHA-1, salt not provided (defaults to HashLen zero octets), + zero-length info"}]. +sha_default_salt(Config) when is_list(Config) -> + %% Hash = SHA-1 + %% IKM = 0x0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c (22 octets) + %% salt = not provided (defaults to HashLen zero octets) + %% info = (0 octets) + %% L = 42 + + %% PRK = 0x2adccada18779e7c2077ad2eb19d3f3e731385dd (20 octets) + %% OKM = 0x2c91117204d745f3500d636a62f64f0a + %% b3bae548aa53d423b0d1f27ebba6f5e5 + %% 673a081d70cce7acfc48 (42 octets) + IKM = hexstr2bin("0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c"), + Salt = binary:copy(<<0>>, 20), + Info = <<>>, + PRK = hexstr2bin("2adccada18779e7c2077ad2eb19d3f3e731385dd"), + OKM = hexstr2bin("2c91117204d745f3500d636a62f64f0a" + "b3bae548aa53d423b0d1f27ebba6f5e5" + "673a081d70cce7acfc48"), + hkdf_test(sha, Salt, IKM, PRK, Info, 42, OKM). + +hkdf_test(HashAlg, Salt, KeyingMaterial, PsedoRandKey, ContextInfo, Length, Key) -> + PsedoRandKey = tls_v1:hkdf_extract(HashAlg, Salt, KeyingMaterial), + Key = tls_v1:hkdf_expand(PsedoRandKey, ContextInfo, Length, HashAlg). + +hexstr2bin(S) when is_binary(S) -> + list_to_binary(hexstr2list(binary_to_list(S))); +hexstr2bin(S) -> + list_to_binary(hexstr2list(S)). + +hexstr2list([$ |T]) -> + hexstr2list(T); +hexstr2list([X,Y|T]) -> + [mkint(X)*16 + mkint(Y) | hexstr2list(T)]; +hexstr2list([]) -> + []. +mkint(C) when $0 =< C, C =< $9 -> + C - $0; +mkint(C) when $A =< C, C =< $F -> + C - $A + 10; +mkint(C) when $a =< C, C =< $f -> + C - $a + 10. diff --git a/lib/ssl/test/ssl_session_cache_SUITE.erl b/lib/ssl/test/ssl_session_cache_SUITE.erl index 7f33fe3204..b71b15b028 100644 --- a/lib/ssl/test/ssl_session_cache_SUITE.erl +++ b/lib/ssl/test/ssl_session_cache_SUITE.erl @@ -186,7 +186,7 @@ session_cleanup() -> session_cleanup(Config) when is_list(Config) -> process_flag(trap_exit, true), ClientOpts = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index c3e64e62d6..3b161a0c8a 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -51,20 +51,20 @@ node_to_hostip(Node) -> Address. start_server(Args) -> - Result = spawn_link(?MODULE, run_server, [Args]), + Node = proplists:get_value(node, Args), + Result = spawn_link(Node, ?MODULE, run_server, [Args]), receive {listen, up} -> Result end. run_server(Opts) -> - Node = proplists:get_value(node, Opts), Port = proplists:get_value(port, Opts), Options = proplists:get_value(options, Opts), Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), ct:log("~p:~p~nssl:listen(~p, ~p)~n", [?MODULE,?LINE, Port, Options]), - {ok, ListenSocket} = rpc:call(Node, Transport, listen, [Port, Options]), + {ok, ListenSocket} = Transport:listen(Port, Options), Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), run_server(ListenSocket, Opts). @@ -90,13 +90,12 @@ do_run_server(_, ok = Result, Opts) -> Pid = proplists:get_value(from, Opts), Pid ! {self(), Result}; do_run_server(ListenSocket, AcceptSocket, Opts) -> - Node = proplists:get_value(node, Opts), Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), {Module, Function, Args} = proplists:get_value(mfa, Opts), ct:log("~p:~p~nServer: apply(~p,~p,~p)~n", [?MODULE,?LINE, Module, Function, [AcceptSocket | Args]]), - case rpc:call(Node, Module, Function, [AcceptSocket | Args]) of + case apply(Module, Function, [AcceptSocket | Args]) of no_result_msg -> ok; Msg -> @@ -110,8 +109,8 @@ do_run_server(ListenSocket, AcceptSocket, Opts) -> run_server(ListenSocket, [MFA | proplists:delete(mfa, Opts)]); close -> ct:log("~p:~p~nServer closing ~p ~n", [?MODULE,?LINE, self()]), - Result = rpc:call(Node, Transport, close, [AcceptSocket], 500), - Result1 = rpc:call(Node, Transport, close, [ListenSocket], 500), + Result = Transport:close(AcceptSocket), + Result1 = Transport:close(ListenSocket), ct:log("~p:~p~nResult ~p : ~p ~n", [?MODULE,?LINE, Result, Result1]); {ssl_closed, _} -> ok @@ -132,41 +131,37 @@ connect(#sslsocket{} = ListenSocket, Opts) -> remove_close_msg(ReconnectTimes), AcceptSocket end; -connect(ListenSocket, Opts) -> - Node = proplists:get_value(node, Opts), +connect(ListenSocket, _Opts) -> ct:log("~p:~p~ngen_tcp:accept(~p)~n", [?MODULE,?LINE, ListenSocket]), - {ok, AcceptSocket} = rpc:call(Node, gen_tcp, accept, - [ListenSocket]), + {ok, AcceptSocket} = gen_tcp:accept(ListenSocket), AcceptSocket. connect(_, _, 0, AcceptSocket, _, _, _) -> AcceptSocket; connect(ListenSocket, Node, _N, _, Timeout, SslOpts, cancel) -> ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), - {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, - [ListenSocket]), + {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), ct:log("~p:~p~nssl:handshake(~p,~p,~p)~n", [?MODULE,?LINE, AcceptSocket, SslOpts,Timeout]), - case rpc:call(Node, ssl, handshake, [AcceptSocket, SslOpts, Timeout]) of + case ssl:handshake(AcceptSocket, SslOpts, Timeout) of {ok, Socket0, Ext} -> ct:log("Ext ~p:~n", [Ext]), ct:log("~p:~p~nssl:handshake_cancel(~p)~n", [?MODULE,?LINE, Socket0]), - rpc:call(Node, ssl, handshake_cancel, [Socket0]); + ssl:handshake_cancel(Socket0); Result -> ct:log("~p:~p~nssl:handshake@~p ret ~p",[?MODULE,?LINE, Node,Result]), Result end; connect(ListenSocket, Node, N, _, Timeout, SslOpts, [_|_] =ContOpts) -> ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), - {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, - [ListenSocket]), + {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), ct:log("~p:~p~nssl:handshake(~p,~p,~p)~n", [?MODULE,?LINE, AcceptSocket, SslOpts,Timeout]), - case rpc:call(Node, ssl, handshake, [AcceptSocket, SslOpts, Timeout]) of + case ssl:handshake(AcceptSocket, SslOpts, Timeout) of {ok, Socket0, Ext} -> ct:log("Ext ~p:~n", [Ext]), ct:log("~p:~p~nssl:handshake_continue(~p,~p,~p)~n", [?MODULE,?LINE, Socket0, ContOpts,Timeout]), - case rpc:call(Node, ssl, handshake_continue, [Socket0, ContOpts, Timeout]) of + case ssl:handshake_continue(Socket0, ContOpts, Timeout) of {ok, Socket} -> connect(ListenSocket, Node, N-1, Socket, Timeout, SslOpts, ContOpts); Error -> @@ -179,35 +174,35 @@ connect(ListenSocket, Node, N, _, Timeout, SslOpts, [_|_] =ContOpts) -> end; connect(ListenSocket, Node, N, _, Timeout, [], ContOpts) -> ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), - {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, - [ListenSocket]), + {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), ct:log("~p:~p~nssl:ssl_accept(~p, ~p)~n", [?MODULE,?LINE, AcceptSocket, Timeout]), - case rpc:call(Node, ssl, ssl_accept, [AcceptSocket, Timeout]) of - ok -> - connect(ListenSocket, Node, N-1, AcceptSocket, Timeout, [], ContOpts); + case ssl:handshake(AcceptSocket, Timeout) of + {ok, Socket} -> + connect(ListenSocket, Node, N-1, Socket, Timeout, [], ContOpts); Result -> - ct:log("~p:~p~nssl:ssl_accept@~p ret ~p",[?MODULE,?LINE, Node,Result]), + ct:log("~p:~p~nssl:handshake@~p ret ~p",[?MODULE,?LINE, Node,Result]), Result end; -connect(ListenSocket, Node, _, _, Timeout, Opts, _) -> +connect(ListenSocket, _Node, _, _, Timeout, Opts, _) -> ct:log("ssl:transport_accept(~p)~n", [ListenSocket]), - {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, - [ListenSocket]), - ct:log("ssl:ssl_accept(~p,~p, ~p)~n", [AcceptSocket, Opts, Timeout]), - rpc:call(Node, ssl, ssl_accept, [AcceptSocket, Opts, Timeout]), + {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), + ct:log("ssl:handshake(~p,~p, ~p)~n", [AcceptSocket, Opts, Timeout]), + ssl:handshake(AcceptSocket, Opts, Timeout), AcceptSocket. start_server_transport_abuse_socket(Args) -> - Result = spawn_link(?MODULE, transport_accept_abuse, [Args]), + Node = proplists:get_value(node, Args), + Result = spawn_link(Node, ?MODULE, transport_accept_abuse, [Args]), receive {listen, up} -> Result end. start_server_transport_control(Args) -> - Result = spawn_link(?MODULE, transport_switch_control, [Args]), + Node = proplists:get_value(node, Args), + Result = spawn_link(Node, ?MODULE, transport_switch_control, [Args]), receive {listen, up} -> Result @@ -215,35 +210,31 @@ start_server_transport_control(Args) -> transport_accept_abuse(Opts) -> - Node = proplists:get_value(node, Opts), Port = proplists:get_value(port, Opts), Options = proplists:get_value(options, Opts), Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), ct:log("~p:~p~nssl:listen(~p, ~p)~n", [?MODULE,?LINE, Port, Options]), - {ok, ListenSocket} = rpc:call(Node, Transport, listen, [Port, Options]), + {ok, ListenSocket} = Transport:listen(Port, Options), Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), - {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, - [ListenSocket]), - {error, _} = rpc:call(Node, ssl, connection_information, [AcceptSocket]), - _ = rpc:call(Node, ssl, handshake, [AcceptSocket, infinity]), + {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), + {error, _} = ssl:connection_information(AcceptSocket), + _ = ssl:handshake(AcceptSocket, infinity), Pid ! {self(), ok}. transport_switch_control(Opts) -> - Node = proplists:get_value(node, Opts), Port = proplists:get_value(port, Opts), Options = proplists:get_value(options, Opts), Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), ct:log("~p:~p~nssl:listen(~p, ~p)~n", [?MODULE,?LINE, Port, Options]), - {ok, ListenSocket} = rpc:call(Node, Transport, listen, [Port, Options]), + {ok, ListenSocket} = Transport:listen(Port, Options), Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), - {ok, AcceptSocket} = rpc:call(Node, ssl, transport_accept, - [ListenSocket]), - ok = rpc:call(Node, ssl, controlling_process, [AcceptSocket, self()]), + {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), + ok = ssl:controlling_process(AcceptSocket, self()), Pid ! {self(), ok}. @@ -256,7 +247,8 @@ remove_close_msg(ReconnectTimes) -> end. start_client(Args) -> - Result = spawn_link(?MODULE, run_client_init, [lists:delete(return_socket, Args)]), + Node = proplists:get_value(node, Args), + Result = spawn_link(Node, ?MODULE, run_client_init, [lists:delete(return_socket, Args)]), receive {connected, Socket} -> case lists:member(return_socket, Args) of @@ -288,8 +280,8 @@ run_client(Opts) -> client_cont_loop(Node, Host, Port, Pid, Transport, Options, ContOpts, Opts) end. -client_loop(Node, Host, Port, Pid, Transport, Options, Opts) -> - case rpc:call(Node, Transport, connect, [Host, Port, Options]) of +client_loop(_Node, Host, Port, Pid, Transport, Options, Opts) -> + case Transport:connect(Host, Port, Options) of {ok, Socket} -> Pid ! {connected, Socket}, ct:log("~p:~p~nClient: connected~n", [?MODULE,?LINE]), @@ -299,7 +291,7 @@ client_loop(Node, Host, Port, Pid, Transport, Options, Opts) -> {Module, Function, Args} = proplists:get_value(mfa, Opts), ct:log("~p:~p~nClient: apply(~p,~p,~p)~n", [?MODULE,?LINE, Module, Function, [Socket | Args]]), - case rpc:call(Node, Module, Function, [Socket | Args]) of + case apply(Module, Function, [Socket | Args]) of no_result_msg -> ok; Msg -> @@ -309,7 +301,7 @@ client_loop(Node, Host, Port, Pid, Transport, Options, Opts) -> receive close -> ct:log("~p:~p~nClient closing~n", [?MODULE,?LINE]), - rpc:call(Node, Transport, close, [Socket]); + Transport:close(Socket); {ssl_closed, Socket} -> ok; {gen_tcp, closed} -> @@ -339,16 +331,13 @@ client_loop(Node, Host, Port, Pid, Transport, Options, Opts) -> end; {error, Reason} -> ct:log("~p:~p~nClient: connection failed: ~p ~n", [?MODULE,?LINE, Reason]), - Pid ! {connect_failed, Reason}; - {badrpc,BadRPC} -> - ct:log("~p:~p~nBad rpc: ~p",[?MODULE,?LINE, BadRPC]), - Pid ! {connect_failed, {badrpc,BadRPC}} + Pid ! {connect_failed, Reason} end. -client_cont_loop(Node, Host, Port, Pid, Transport, Options, cancel, _Opts) -> - case rpc:call(Node, Transport, connect, [Host, Port, Options]) of +client_cont_loop(_Node, Host, Port, Pid, Transport, Options, cancel, _Opts) -> + case Transport:connect(Host, Port, Options) of {ok, Socket, _} -> - Result = rpc:call(Node, Transport, handshake_cancel, [Socket]), + Result = Transport:handshake_cancel(Socket), ct:log("~p:~p~nClient: Cancel: ~p ~n", [?MODULE,?LINE, Result]), Pid ! {connect_failed, Result}; {error, Reason} -> @@ -356,17 +345,17 @@ client_cont_loop(Node, Host, Port, Pid, Transport, Options, cancel, _Opts) -> Pid ! {connect_failed, Reason} end; -client_cont_loop(Node, Host, Port, Pid, Transport, Options, ContOpts, Opts) -> - case rpc:call(Node, Transport, connect, [Host, Port, Options]) of +client_cont_loop(_Node, Host, Port, Pid, Transport, Options, ContOpts, Opts) -> + case Transport:connect(Host, Port, Options) of {ok, Socket0, _} -> ct:log("~p:~p~nClient: handshake_continue(~p, ~p, infinity) ~n", [?MODULE, ?LINE, Socket0, ContOpts]), - case rpc:call(Node, Transport, handshake_continue, [Socket0, ContOpts]) of + case Transport:handshake_continue(Socket0, ContOpts) of {ok, Socket} -> Pid ! {connected, Socket}, {Module, Function, Args} = proplists:get_value(mfa, Opts), ct:log("~p:~p~nClient: apply(~p,~p,~p)~n", [?MODULE,?LINE, Module, Function, [Socket | Args]]), - case rpc:call(Node, Module, Function, [Socket | Args]) of + case apply(Module, Function, [Socket | Args]) of no_result_msg -> ok; Msg -> @@ -514,13 +503,6 @@ wait_for_result(Pid, Msg) -> %% Unexpected end. -user_lookup(psk, _Identity, UserState) -> - {ok, UserState}; -user_lookup(srp, Username, _UserState) -> - Salt = ssl_cipher:random_bytes(16), - UserPassHash = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, <<"secret">>])]), - {ok, {srp_1024, Salt, UserPassHash}}. - cert_options(Config) -> ClientCaCertFile = filename:join([proplists:get_value(priv_dir, Config), "client", "cacerts.pem"]), @@ -649,6 +631,40 @@ make_rsa_cert_chains(UserConf, Config, Suffix) -> [{reuseaddr, true}, {verify, verify_peer} | ServerConf] }. +make_ecc_cert_chains(UserConf, Config, Suffix) -> + ClientChain = proplists:get_value(client_chain, UserConf, default_cert_chain_conf()), + ServerChain = proplists:get_value(server_chain, UserConf, default_cert_chain_conf()), + CertChainConf = gen_conf(ecdsa, ecdsa, ClientChain, ServerChain), + ClientFileBase = filename:join([proplists:get_value(priv_dir, Config), "ecdsa" ++ Suffix]), + ServerFileBase = filename:join([proplists:get_value(priv_dir, Config), "ecdsa" ++ Suffix]), + GenCertData = public_key:pkix_test_data(CertChainConf), + [{server_config, ServerConf}, + {client_config, ClientConf}] = + x509_test:gen_pem_config_files(GenCertData, ClientFileBase, ServerFileBase), + {[{verify, verify_peer} | ClientConf], + [{reuseaddr, true}, {verify, verify_peer} | ServerConf] + }. + + +make_dsa_cert_chains(UserConf, Config, Suffix) -> + CryptoSupport = crypto:supports(), + case proplists:get_bool(dss, proplists:get_value(public_keys, CryptoSupport)) of + true -> + ClientChain = proplists:get_value(client_chain, UserConf, default_cert_chain_conf()), + ServerChain = proplists:get_value(server_chain, UserConf, default_cert_chain_conf()), + CertChainConf = gen_conf(dsa, dsa, ClientChain, ServerChain), + ClientFileBase = filename:join([proplists:get_value(priv_dir, Config), "dsa" ++ Suffix]), + ServerFileBase = filename:join([proplists:get_value(priv_dir, Config), "dsa" ++ Suffix]), + GenCertData = public_key:pkix_test_data(CertChainConf), + [{server_config, ServerConf}, + {client_config, ClientConf}] = + x509_test:gen_pem_config_files(GenCertData, ClientFileBase, ServerFileBase), + {[{verify, verify_peer} | ClientConf], + [{reuseaddr, true}, {verify, verify_peer} | ServerConf]}; + false -> + Config + end. + make_ec_cert_chains(UserConf, ClientChainType, ServerChainType, Config) -> make_ec_cert_chains(UserConf, ClientChainType, ServerChainType, Config, ?DEFAULT_CURVE). %% @@ -903,14 +919,14 @@ make_ecdh_rsa_cert(Config) -> end. start_upgrade_server(Args) -> - Result = spawn_link(?MODULE, run_upgrade_server, [Args]), + Node = proplists:get_value(node, Args), + Result = spawn_link(Node, ?MODULE, run_upgrade_server, [Args]), receive {listen, up} -> Result end. run_upgrade_server(Opts) -> - Node = proplists:get_value(node, Opts), Port = proplists:get_value(port, Opts), TimeOut = proplists:get_value(timeout, Opts, infinity), TcpOptions = proplists:get_value(tcp_options, Opts), @@ -918,43 +934,41 @@ run_upgrade_server(Opts) -> Pid = proplists:get_value(from, Opts), ct:log("~p:~p~ngen_tcp:listen(~p, ~p)~n", [?MODULE,?LINE, Port, TcpOptions]), - {ok, ListenSocket} = rpc:call(Node, gen_tcp, listen, [Port, TcpOptions]), + {ok, ListenSocket} = gen_tcp:listen(Port, TcpOptions), Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), ct:log("~p:~p~ngen_tcp:accept(~p)~n", [?MODULE,?LINE, ListenSocket]), - {ok, AcceptSocket} = rpc:call(Node, gen_tcp, accept, [ListenSocket]), + {ok, AcceptSocket} = gen_tcp:accept(ListenSocket), try {ok, SslAcceptSocket} = case TimeOut of infinity -> - ct:log("~p:~p~nssl:ssl_accept(~p, ~p)~n", + ct:log("~p:~p~nssl:handshake(~p, ~p)~n", [?MODULE,?LINE, AcceptSocket, SslOptions]), - rpc:call(Node, ssl, ssl_accept, - [AcceptSocket, SslOptions]); + ssl:handshake(AcceptSocket, SslOptions); _ -> - ct:log("~p:~p~nssl:ssl_accept(~p, ~p, ~p)~n", + ct:log("~p:~p~nssl:handshake(~p, ~p, ~p)~n", [?MODULE,?LINE, AcceptSocket, SslOptions, TimeOut]), - rpc:call(Node, ssl, ssl_accept, - [AcceptSocket, SslOptions, TimeOut]) + ssl:handshake(AcceptSocket, SslOptions, TimeOut) end, {Module, Function, Args} = proplists:get_value(mfa, Opts), - Msg = rpc:call(Node, Module, Function, [SslAcceptSocket | Args]), + Msg = apply(Module, Function, [SslAcceptSocket | Args]), ct:log("~p:~p~nUpgrade Server Msg: ~p ~n", [?MODULE,?LINE, Msg]), Pid ! {self(), Msg}, receive close -> ct:log("~p:~p~nUpgrade Server closing~n", [?MODULE,?LINE]), - rpc:call(Node, ssl, close, [SslAcceptSocket]) + ssl:close(SslAcceptSocket) end catch error:{badmatch, Error} -> Pid ! {self(), Error} end. start_upgrade_client(Args) -> - spawn_link(?MODULE, run_upgrade_client, [Args]). + Node = proplists:get_value(node, Args), + spawn_link(Node, ?MODULE, run_upgrade_client, [Args]). run_upgrade_client(Opts) -> - Node = proplists:get_value(node, Opts), Host = proplists:get_value(host, Opts), Port = proplists:get_value(port, Opts), Pid = proplists:get_value(from, Opts), @@ -963,34 +977,34 @@ run_upgrade_client(Opts) -> ct:log("~p:~p~ngen_tcp:connect(~p, ~p, ~p)~n", [?MODULE,?LINE, Host, Port, TcpOptions]), - {ok, Socket} = rpc:call(Node, gen_tcp, connect, [Host, Port, TcpOptions]), + {ok, Socket} = gen_tcp:connect(Host, Port, TcpOptions), send_selected_port(Pid, Port, Socket), ct:log("~p:~p~nssl:connect(~p, ~p)~n", [?MODULE,?LINE, Socket, SslOptions]), - {ok, SslSocket} = rpc:call(Node, ssl, connect, [Socket, SslOptions]), + {ok, SslSocket} = ssl:connect(Socket, SslOptions), {Module, Function, Args} = proplists:get_value(mfa, Opts), ct:log("~p:~p~napply(~p, ~p, ~p)~n", [?MODULE,?LINE, Module, Function, [SslSocket | Args]]), - Msg = rpc:call(Node, Module, Function, [SslSocket | Args]), + Msg = apply(Module, Function, [SslSocket | Args]), ct:log("~p:~p~nUpgrade Client Msg: ~p ~n", [?MODULE,?LINE, Msg]), Pid ! {self(), Msg}, receive close -> ct:log("~p:~p~nUpgrade Client closing~n", [?MODULE,?LINE]), - rpc:call(Node, ssl, close, [SslSocket]) + ssl:close(SslSocket) end. start_upgrade_server_error(Args) -> - Result = spawn_link(?MODULE, run_upgrade_server_error, [Args]), + Node = proplists:get_value(node, Args), + Result = spawn_link(Node,?MODULE, run_upgrade_server_error, [Args]), receive {listen, up} -> Result end. run_upgrade_server_error(Opts) -> - Node = proplists:get_value(node, Opts), Port = proplists:get_value(port, Opts), TimeOut = proplists:get_value(timeout, Opts, infinity), TcpOptions = proplists:get_value(tcp_options, Opts), @@ -998,22 +1012,20 @@ run_upgrade_server_error(Opts) -> Pid = proplists:get_value(from, Opts), ct:log("~p:~p~ngen_tcp:listen(~p, ~p)~n", [?MODULE,?LINE, Port, TcpOptions]), - {ok, ListenSocket} = rpc:call(Node, gen_tcp, listen, [Port, TcpOptions]), + {ok, ListenSocket} = gen_tcp:listen(Port, TcpOptions), Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), ct:log("~p:~p~ngen_tcp:accept(~p)~n", [?MODULE,?LINE, ListenSocket]), - {ok, AcceptSocket} = rpc:call(Node, gen_tcp, accept, [ListenSocket]), + {ok, AcceptSocket} = gen_tcp:accept(ListenSocket), Error = case TimeOut of infinity -> - ct:log("~p:~p~nssl:ssl_accept(~p, ~p)~n", + ct:log("~p:~p~nssl:handshake(~p, ~p)~n", [?MODULE,?LINE, AcceptSocket, SslOptions]), - rpc:call(Node, ssl, ssl_accept, - [AcceptSocket, SslOptions]); + ssl:handshake(AcceptSocket, SslOptions); _ -> - ct:log("~p:~p~nssl:ssl_accept(~p, ~p, ~p)~n", + ct:log("~p:~p~nssl:ssl_handshake(~p, ~p, ~p)~n", [?MODULE,?LINE, AcceptSocket, SslOptions, TimeOut]), - rpc:call(Node, ssl, ssl_accept, - [AcceptSocket, SslOptions, TimeOut]) + ssl:handshake(AcceptSocket, SslOptions, TimeOut) end, Pid ! {self(), Error}. @@ -1025,32 +1037,31 @@ start_server_error(Args) -> end. run_server_error(Opts) -> - Node = proplists:get_value(node, Opts), Port = proplists:get_value(port, Opts), Options = proplists:get_value(options, Opts), Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), ct:log("~p:~p~nssl:listen(~p, ~p)~n", [?MODULE,?LINE, Port, Options]), - case rpc:call(Node, Transport, listen, [Port, Options]) of + case Transport:listen(Port, Options) of {ok, #sslsocket{} = ListenSocket} -> %% To make sure error_client will %% get {error, closed} and not {error, connection_refused} Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), ct:log("~p:~p~nssl:transport_accept(~p)~n", [?MODULE,?LINE, ListenSocket]), - case rpc:call(Node, Transport, transport_accept, [ListenSocket]) of + case Transport:transport_accept(ListenSocket) of {error, _} = Error -> Pid ! {self(), Error}; {ok, AcceptSocket} -> ct:log("~p:~p~nssl:ssl_accept(~p)~n", [?MODULE,?LINE, AcceptSocket]), - Error = rpc:call(Node, ssl, ssl_accept, [AcceptSocket]), + Error = ssl:handshake(AcceptSocket), Pid ! {self(), Error} end; {ok, ListenSocket} -> Pid ! {listen, up}, send_selected_port(Pid, Port, ListenSocket), ct:log("~p:~p~n~p:accept(~p)~n", [?MODULE,?LINE, Transport, ListenSocket]), - case rpc:call(Node, Transport, accept, [ListenSocket]) of + case Transport:accept(ListenSocket) of {error, _} = Error -> Pid ! {self(), Error} end; @@ -1062,17 +1073,17 @@ run_server_error(Opts) -> end. start_client_error(Args) -> - spawn_link(?MODULE, run_client_error, [Args]). + Node = proplists:get_value(node, Args), + spawn_link(Node, ?MODULE, run_client_error, [Args]). run_client_error(Opts) -> - Node = proplists:get_value(node, Opts), Host = proplists:get_value(host, Opts), Port = proplists:get_value(port, Opts), Pid = proplists:get_value(from, Opts), Transport = proplists:get_value(transport, Opts, ssl), Options = proplists:get_value(options, Opts), ct:log("~p:~p~nssl:connect(~p, ~p, ~p)~n", [?MODULE,?LINE, Host, Port, Options]), - Error = rpc:call(Node, Transport, connect, [Host, Port, Options]), + Error = Transport:connect(Host, Port, Options), Pid ! {self(), Error}. accepters(N) -> @@ -1090,7 +1101,7 @@ accepters(Acc, N) -> basic_test(COpts, SOpts, Config) -> SType = proplists:get_value(server_type, Config), CType = proplists:get_value(client_type, Config), - {Server, Port} = start_server(SType, SOpts, Config), + {Server, Port} = start_server(SType, COpts, SOpts, Config), Client = start_client(CType, Port, COpts, Config), gen_check_result(Server, SType, Client, CType), stop(Server, Client). @@ -1106,6 +1117,35 @@ ecc_test_error(COpts, SOpts, CECCOpts, SECCOpts, Config) -> Client = start_client_ecc_error(erlang, Port, COpts, CECCOpts, Config), check_server_alert(Server, Client, insufficient_security). +start_basic_client(openssl, Version, Port, ClientOpts) -> + Cert = proplists:get_value(certfile, ClientOpts), + Key = proplists:get_value(keyfile, ClientOpts), + CA = proplists:get_value(cacertfile, ClientOpts), + Groups0 = proplists:get_value(groups, ClientOpts), + Exe = "openssl", + Args0 = ["s_client", "-verify", "2", "-port", integer_to_list(Port), + ssl_test_lib:version_flag(Version), + "-CAfile", CA, "-host", "localhost", "-msg", "-debug"], + Args1 = + case Groups0 of + undefined -> + Args0; + G -> + Args0 ++ ["-groups", G] + end, + Args = + case {Cert, Key} of + {C, K} when C =:= undefined orelse + K =:= undefined -> + Args1; + {C, K} -> + Args1 ++ ["-cert", C, "-key", K] + end, + + OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), + true = port_command(OpenSslPort, "Hello world"), + OpenSslPort. + start_client(openssl, Port, ClientOpts, Config) -> Cert = proplists:get_value(certfile, ClientOpts), Key = proplists:get_value(keyfile, ClientOpts), @@ -1128,7 +1168,7 @@ start_client(erlang, Port, ClientOpts, Config) -> {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, check_key_exchange_send_active, [KeyEx]}}, - {options, [{verify, verify_peer} | ClientOpts]}]). + {options, ClientOpts}]). %% Workaround for running tests on machines where openssl %% s_client would use an IPv6 address with localhost. As @@ -1163,20 +1203,19 @@ start_client_ecc_error(erlang, Port, ClientOpts, ECCOpts, Config) -> [{verify, verify_peer} | ClientOpts]}]). -start_server(openssl, ServerOpts, Config) -> - Cert = proplists:get_value(certfile, ServerOpts), - Key = proplists:get_value(keyfile, ServerOpts), - CA = proplists:get_value(cacertfile, ServerOpts), +start_server(openssl, ClientOpts, ServerOpts, Config) -> Port = inet_port(node()), Version = protocol_version(Config), Exe = "openssl", - Args = ["s_server", "-accept", integer_to_list(Port), ssl_test_lib:version_flag(Version), - "-verify", "2", "-cert", Cert, "-CAfile", CA, - "-key", Key, "-msg", "-debug"], + CertArgs = openssl_cert_options(ServerOpts), + [Cipher|_] = proplists:get_value(ciphers, ClientOpts, ssl:cipher_suites(default,Version)), + Args = ["s_server", "-accept", integer_to_list(Port), "-cipher", + ssl_cipher_format:suite_map_to_openssl_str(Cipher), + ssl_test_lib:version_flag(Version)] ++ CertArgs ++ ["-msg", "-debug"], OpenSslPort = portable_open_port(Exe, Args), true = port_command(OpenSslPort, "Hello world"), {OpenSslPort, Port}; -start_server(erlang, ServerOpts, Config) -> +start_server(erlang, _, ServerOpts, Config) -> {_, ServerNode, _} = ssl_test_lib:run_where(Config), KeyEx = proplists:get_value(check_keyex, Config, false), Server = start_server([{node, ServerNode}, {port, 0}, @@ -1239,6 +1278,29 @@ stop(Client, Server) -> close(Server), close(Client). + +openssl_cert_options(ServerOpts) -> + Cert = proplists:get_value(certfile, ServerOpts, undefined), + Key = proplists:get_value(keyfile, ServerOpts, undefined), + CA = proplists:get_value(cacertfile, ServerOpts, undefined), + case CA of + undefined -> + case cert_option("-cert", Cert) ++ cert_option("-key", Key) of + [] -> + ["-nocert"]; + Other -> + Other + end; + _ -> + cert_option("-cert", Cert) ++ cert_option("-CAfile", CA) ++ + cert_option("-key", Key) ++ ["-verify", "2"] + end. + +cert_option(_, undefined) -> + []; +cert_option(Opt, Value) -> + [Opt, Value]. + supported_eccs(Opts) -> ToCheck = proplists:get_value(eccs, Opts, []), Supported = ssl:eccs(), @@ -1335,13 +1397,13 @@ common_ciphers(crypto) -> common_ciphers(openssl) -> OpenSslSuites = string:tokens(string:strip(os:cmd("openssl ciphers"), right, $\n), ":"), - [ssl_cipher_format:suite_definition(S) + [ssl_cipher_format:suite_bin_to_map(S) || S <- ssl_cipher:suites(tls_record:highest_protocol_version([])), - lists:member(ssl_cipher_format:openssl_suite_name(S), OpenSslSuites) + lists:member(ssl_cipher_format:suite_map_to_openssl_str(ssl_cipher_format:suite_bin_to_map(S)), OpenSslSuites) ]. available_suites(Version) -> - [ssl_cipher_format:suite_definition(Suite) || + [ssl_cipher_format:suite_bin_to_map(Suite) || Suite <- ssl_cipher:filter_suites(ssl_cipher:suites(Version))]. @@ -1414,7 +1476,7 @@ string_regex_filter(_Str, _Search) -> false. ecdh_dh_anonymous_suites(Version) -> - ssl:filter_cipher_suites([ssl_cipher_format:suite_definition(S) || S <- ssl_cipher:anonymous_suites(Version)], + ssl:filter_cipher_suites([ssl_cipher_format:suite_bin_to_map(S) || S <- ssl_cipher:anonymous_suites(Version)], [{key_exchange, fun(dh_anon) -> true; @@ -1424,7 +1486,7 @@ ecdh_dh_anonymous_suites(Version) -> false end}]). psk_suites({3,_} = Version) -> - ssl:filter_cipher_suites([ssl_cipher_format:suite_definition(S) || S <- ssl_cipher:psk_suites(Version)], []); + ssl:filter_cipher_suites([ssl_cipher_format:suite_bin_to_map(S) || S <- ssl_cipher:psk_suites(Version)], []); psk_suites(Version) -> ssl:filter_cipher_suites(psk_suites(dtls_v1:corresponding_tls_version(Version)), [{cipher, @@ -1435,7 +1497,7 @@ psk_suites(Version) -> end}]). psk_anon_suites({3,_} = Version) -> - ssl:filter_cipher_suites([ssl_cipher_format:suite_definition(S) || S <- ssl_cipher:psk_suites_anon(Version)], + ssl:filter_cipher_suites([ssl_cipher_format:suite_bin_to_map(S) || S <- ssl_cipher:psk_suites_anon(Version)], [{key_exchange, fun(psk) -> true; @@ -1458,7 +1520,7 @@ psk_anon_suites(Version) -> srp_suites() -> - ssl:filter_cipher_suites([ssl_cipher_format:suite_definition(S) || S <- ssl_cipher:srp_suites()], + ssl:filter_cipher_suites([ssl_cipher_format:suite_bin_to_map(S) || S <- ssl_cipher:srp_suites()], [{key_exchange, fun(srp_rsa) -> true; @@ -1466,10 +1528,10 @@ srp_suites() -> false end}]). srp_anon_suites() -> - ssl:filter_cipher_suites([ssl_cipher_format:suite_definition(S) || S <- ssl_cipher:srp_suites_anon()], + ssl:filter_cipher_suites([ssl_cipher_format:suite_bin_to_map(S) || S <- ssl_cipher:srp_suites_anon()], []). srp_dss_suites() -> - ssl:filter_cipher_suites([ssl_cipher_format:suite_definition(S) || S <- ssl_cipher:srp_suites()], + ssl:filter_cipher_suites([ssl_cipher_format:suite_bin_to_map(S) || S <- ssl_cipher:srp_suites()], [{key_exchange, fun(srp_dss) -> true; @@ -1477,14 +1539,14 @@ srp_dss_suites() -> false end}]). chacha_suites(Version) -> - [ssl_cipher_format:suite_definition(S) || S <- ssl_cipher:filter_suites(ssl_cipher:chacha_suites(Version))]. + [ssl_cipher_format:suite_bin_to_map(S) || S <- ssl_cipher:filter_suites(ssl_cipher:chacha_suites(Version))]. rc4_suites(Version) -> - ssl:filter_cipher_suites([ssl_cipher_format:suite_definition(S) || S <-ssl_cipher:rc4_suites(Version)], []). + ssl:filter_cipher_suites([ssl_cipher_format:suite_bin_to_map(S) || S <-ssl_cipher:rc4_suites(Version)], []). des_suites(Version) -> - ssl:filter_cipher_suites([ssl_cipher_format:suite_definition(S) || S <-ssl_cipher:des_suites(Version)], []). + ssl:filter_cipher_suites([ssl_cipher_format:suite_bin_to_map(S) || S <-ssl_cipher:des_suites(Version)], []). tuple_to_map({Kex, Cipher, Mac}) -> #{key_exchange => Kex, @@ -1511,10 +1573,13 @@ cipher_result(Socket, Result) -> ct:log("~p:~p~nSuccessfull connect: ~p~n", [?MODULE,?LINE, Result]), %% Importante to send two packets here %% to properly test "cipher state" handling - ssl:send(Socket, "Hello\n"), - "Hello\n" = active_recv(Socket, length( "Hello\n")), - ssl:send(Socket, " world\n"), - " world\n" = active_recv(Socket, length(" world\n")), + Hello = "Hello\n", + World = " world\n", + ssl:send(Socket, Hello), + ct:sleep(500), + ssl:send(Socket, World), + Expected = Hello ++ World, + Expected = active_recv(Socket, length(Expected)), ok. session_info_result(Socket) -> @@ -1588,7 +1653,12 @@ init_tls_version(Version, Config) -> clean_tls_version(Config) -> proplists:delete(protocol_opts, proplists:delete(protocol, Config)). - + +sufficient_crypto_support(Version) + when Version == 'tlsv1.3' -> + CryptoSupport = crypto:supports(), + lists:member(rsa_pkcs1_pss_padding, proplists:get_value(rsa_opts, CryptoSupport)) andalso + lists:member(x448, proplists:get_value(curves, CryptoSupport)); sufficient_crypto_support(Version) when Version == 'tlsv1.2'; Version == 'dtlsv1.2' -> CryptoSupport = crypto:supports(), @@ -1742,6 +1812,15 @@ is_sane_ecc(crypto) -> is_sane_ecc(_) -> sufficient_crypto_support(cipher_ec). +is_sane_oppenssl_client() -> + [{_,_, Bin}] = crypto:info_lib(), + case binary_to_list(Bin) of + "OpenSSL 0.9" ++ _ -> + false; + _ -> + true + end. + is_fips(openssl) -> VersionStr = os:cmd("openssl version"), case re:split(VersionStr, "fips") of @@ -1906,6 +1985,8 @@ version_flag('tlsv1.1') -> "-tls1_1"; version_flag('tlsv1.2') -> "-tls1_2"; +version_flag('tlsv1.3') -> + "-tls1_3"; version_flag(sslv3) -> "-ssl3"; version_flag(sslv2) -> @@ -1916,10 +1997,10 @@ version_flag('dtlsv1') -> "-dtls1". filter_suites([Cipher | _] = Ciphers, AtomVersion) when is_list(Cipher)-> - filter_suites([ssl_cipher_format:openssl_suite(S) || S <- Ciphers], + filter_suites([ssl_cipher_format:suite_openssl_str_to_map(S) || S <- Ciphers], AtomVersion); filter_suites([Cipher | _] = Ciphers, AtomVersion) when is_binary(Cipher)-> - filter_suites([ssl_cipher_format:suite_definition(S) || S <- Ciphers], + filter_suites([ssl_cipher_format:suite_bin_to_map(S) || S <- Ciphers], AtomVersion); filter_suites(Ciphers0, AtomVersion) -> Version = tls_version(AtomVersion), @@ -1931,7 +2012,7 @@ filter_suites(Ciphers0, AtomVersion) -> ++ ssl_cipher:srp_suites_anon() ++ ssl_cipher:rc4_suites(Version), Supported1 = ssl_cipher:filter_suites(Supported0), - Supported2 = [ssl_cipher_format:suite_definition(S) || S <- Supported1], + Supported2 = [ssl_cipher_format:suite_bin_to_map(S) || S <- Supported1], [Cipher || Cipher <- Ciphers0, lists:member(Cipher, Supported2)]. -define(OPENSSL_QUIT, "Q\n"). @@ -2342,4 +2423,23 @@ reuse_session(ClientOpts, ServerOpts, Config) -> ssl_test_lib:close(Server1), ssl_test_lib:close(Client3), ssl_test_lib:close(Client4). - + +user_lookup(psk, _Identity, UserState) -> + {ok, UserState}; +user_lookup(srp, Username, _UserState) -> + Salt = ssl_cipher:random_bytes(16), + UserPassHash = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, <<"secret">>])]), + {ok, {srp_1024, Salt, UserPassHash}}. + +test_cipher(TestCase, Config) -> + [{name, Group} |_] = proplists:get_value(tc_group_properties, Config), + list_to_atom(re:replace(atom_to_list(TestCase), atom_to_list(Group) ++ "_", "", [{return, list}])). + +digest() -> + case application:get_env(ssl, protocol_version, application:get_env(ssl, dtls_protocol_version)) of + Ver when Ver == 'tlsv1.2'; + Ver == 'dtlsv1.2' -> + {digest, sha256}; + _ -> + {digest, sha1} + end. diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index df84411b6d..07abddbcf7 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -39,16 +39,14 @@ all() -> case ssl_test_lib:openssl_sane_dtls() of true -> - [{group, basic}, - {group, 'tlsv1.2'}, + [{group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, {group, 'sslv3'}, {group, 'dtlsv1.2'}, {group, 'dtlsv1'}]; false -> - [{group, basic}, - {group, 'tlsv1.2'}, + [{group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, {group, 'sslv3'}] @@ -57,8 +55,7 @@ all() -> groups() -> case ssl_test_lib:openssl_sane_dtls() of true -> - [{basic, [], basic_tests()}, - {'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, + [{'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1.1', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1', [], all_versions_tests()++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'sslv3', [], all_versions_tests()}, @@ -66,20 +63,13 @@ groups() -> {'dtlsv1', [], dtls_all_versions_tests()} ]; false -> - [{basic, [], basic_tests()}, - {'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, + [{'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1.1', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1', [], all_versions_tests()++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'sslv3', [], all_versions_tests()} ] end. - -basic_tests() -> - [basic_erlang_client_openssl_server, - basic_erlang_server_openssl_client, - expired_session - ]. - + all_versions_tests() -> [ erlang_client_openssl_server, @@ -191,16 +181,6 @@ end_per_suite(_Config) -> ssl:stop(), application:stop(crypto). -init_per_group(basic, Config0) -> - case ssl_test_lib:supports_ssl_tls_version('tlsv1.2') - orelse ssl_test_lib:supports_ssl_tls_version('tlsv1.1') - orelse ssl_test_lib:supports_ssl_tls_version('tlsv1') - of - true -> - ssl_test_lib:clean_tls_version(Config0); - false -> - {skip, "only sslv3 supported by OpenSSL"} - end; init_per_group(GroupName, Config) -> case ssl_test_lib:is_tls_version(GroupName) of @@ -243,7 +223,7 @@ init_per_testcase(TestCase, Config) when TestCase == erlang_server_openssl_client_dsa_cert; TestCase == erlang_client_openssl_server_dsa_cert; TestCase == erlang_server_openssl_client_dsa_cert -> - case ssl_test_lib:openssl_dsa_support() of + case ssl_test_lib:openssl_dsa_support() andalso ssl_test_lib:is_sane_oppenssl_client() of true -> special_init(TestCase, Config); false -> @@ -344,7 +324,16 @@ special_init(TestCase, Config0) ]} ]}]} | Config0], check_openssl_sni_support(Config); - +special_init(TestCase, Config) + when TestCase == erlang_server_openssl_client; + TestCase == erlang_server_openssl_client_client_cert; + TestCase == erlang_server_openssl_client_reuse_session -> + case ssl_test_lib:is_sane_oppenssl_client() of + true -> + Config; + false -> + {skip, "Broken OpenSSL client"} + end; special_init(_, Config) -> Config. @@ -357,85 +346,7 @@ end_per_testcase(_, Config) -> %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -basic_erlang_client_openssl_server() -> - [{doc,"Test erlang client with openssl server"}]. -basic_erlang_client_openssl_server(Config) when is_list(Config) -> - process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), - ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), - - {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), - - Data = "From openssl to erlang", - - Port = ssl_test_lib:inet_port(node()), - CertFile = proplists:get_value(certfile, ServerOpts), - KeyFile = proplists:get_value(keyfile, ServerOpts), - - Exe = "openssl", - Args = ["s_server", "-accept", integer_to_list(Port), - "-cert", CertFile, "-key", KeyFile], - - OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - - - ssl_test_lib:wait_for_openssl_server(Port, tls), - - Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {mfa, {?MODULE, - erlang_ssl_receive, [Data]}}, - {options, ClientOpts}]), - true = port_command(OpensslPort, Data), - - ssl_test_lib:check_result(Client, ok), - - %% Clean close down! Server needs to be closed first !! - ssl_test_lib:close_port(OpensslPort), - ssl_test_lib:close(Client), - process_flag(trap_exit, false). - -%%-------------------------------------------------------------------- -basic_erlang_server_openssl_client() -> - [{doc,"Test erlang server with openssl client"}]. -basic_erlang_server_openssl_client(Config) when is_list(Config) -> - process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), - - {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - - Data = "From openssl to erlang", - Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, - {from, self()}, - {mfa, {?MODULE, erlang_ssl_receive, [Data]}}, - {options,ServerOpts}]), - - Port = ssl_test_lib:inet_port(Server), - - Exe = "openssl", - Args = case no_low_flag("-no_ssl2") of - [] -> - ["s_client", "-connect", hostname_format(Hostname) ++ - ":" ++ integer_to_list(Port), no_low_flag("-no_ssl3") - | workaround_openssl_s_clinent()]; - Flag -> - ["s_client", "-connect", hostname_format(Hostname) ++ - ":" ++ integer_to_list(Port), no_low_flag("-no_ssl3"), Flag - | workaround_openssl_s_clinent()] - end, - - OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), - true = port_command(OpenSslPort, Data), - - ssl_test_lib:check_result(Server, ok), - - %% Clean close down! Server needs to be closed first !! - ssl_test_lib:close(Server), - ssl_test_lib:close_port(OpenSslPort), - process_flag(trap_exit, false). -%%-------------------------------------------------------------------- erlang_client_openssl_server() -> [{doc,"Test erlang client with openssl server"}]. erlang_client_openssl_server(Config) when is_list(Config) -> @@ -1161,7 +1072,7 @@ erlang_client_bad_openssl_server(Config) when is_list(Config) -> Client1 = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, - {mfa, {ssl_test_lib, no_result_msg, []}}, + {mfa, {ssl_test_lib, no_result, []}}, {options, [{versions, [Version]} | ClientOpts]}]), @@ -1249,7 +1160,7 @@ ssl2_erlang_server_openssl_client(Config) when is_list(Config) -> ct:log("Ports ~p~n", [[erlang:port_info(P) || P <- erlang:ports()]]), ssl_test_lib:consume_port_exit(OpenSslPort), - ssl_test_lib:check_server_alert(Server, bad_record_mac), + ssl_test_lib:check_server_alert(Server, unexpected_message), process_flag(trap_exit, false). %%-------------------------------------------------------------------- @@ -1550,6 +1461,7 @@ send_and_hostname(SSLSocket) -> end. erlang_server_openssl_client_sni_test(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> + Version = ssl_test_lib:protocol_version(Config), ct:log("Start running handshake, Config: ~p, SNIHostname: ~p, ExpectedSNIHostname: ~p, ExpectedCN: ~p", [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), ServerOptions = proplists:get_value(sni_server_opts, Config) ++ proplists:get_value(server_rsa_opts, Config), {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), @@ -1560,9 +1472,9 @@ erlang_server_openssl_client_sni_test(Config, SNIHostname, ExpectedSNIHostname, Exe = "openssl", ClientArgs = case SNIHostname of undefined -> - openssl_client_args(ssl_test_lib:supports_ssl_tls_version(sslv2), Hostname,Port); + openssl_client_args(Version, Hostname,Port); _ -> - openssl_client_args(ssl_test_lib:supports_ssl_tls_version(sslv2), Hostname, Port, SNIHostname) + openssl_client_args(Version, Hostname, Port, SNIHostname) end, ClientPort = ssl_test_lib:portable_open_port(Exe, ClientArgs), @@ -1573,6 +1485,7 @@ erlang_server_openssl_client_sni_test(Config, SNIHostname, ExpectedSNIHostname, erlang_server_openssl_client_sni_test_sni_fun(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> + Version = ssl_test_lib:protocol_version(Config), ct:log("Start running handshake for sni_fun, Config: ~p, SNIHostname: ~p, ExpectedSNIHostname: ~p, ExpectedCN: ~p", [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), [{sni_hosts, ServerSNIConf}] = proplists:get_value(sni_server_opts, Config), SNIFun = fun(Domain) -> proplists:get_value(Domain, ServerSNIConf, undefined) end, @@ -1585,9 +1498,9 @@ erlang_server_openssl_client_sni_test_sni_fun(Config, SNIHostname, ExpectedSNIHo Exe = "openssl", ClientArgs = case SNIHostname of undefined -> - openssl_client_args(ssl_test_lib:supports_ssl_tls_version(sslv2), Hostname,Port); + openssl_client_args(Version, Hostname,Port); _ -> - openssl_client_args(ssl_test_lib:supports_ssl_tls_version(sslv2), Hostname, Port, SNIHostname) + openssl_client_args(Version, Hostname, Port, SNIHostname) end, ClientPort = ssl_test_lib:portable_open_port(Exe, ClientArgs), @@ -1998,13 +1911,19 @@ send_wait_send(Socket, [ErlData, OpenSslData]) -> check_openssl_sni_support(Config) -> HelpText = os:cmd("openssl s_client --help"), - case string:str(HelpText, "-servername") of - 0 -> - {skip, "Current openssl doesn't support SNI"}; - _ -> - Config + case ssl_test_lib:is_sane_oppenssl_client() of + true -> + case string:str(HelpText, "-servername") of + 0 -> + {skip, "Current openssl doesn't support SNI"}; + _ -> + Config + end; + false -> + {skip, "Current openssl doesn't support SNI or extension handling is flawed"} end. + check_openssl_npn_support(Config) -> HelpText = os:cmd("openssl s_client --help"), case string:str(HelpText, "nextprotoneg") of @@ -2070,17 +1989,13 @@ workaround_openssl_s_clinent() -> [] end. -openssl_client_args(false, Hostname, Port) -> - ["s_client", "-connect", Hostname ++ ":" ++ integer_to_list(Port)]; -openssl_client_args(true, Hostname, Port) -> - ["s_client", "-no_ssl2", "-connect", Hostname ++ ":" ++ integer_to_list(Port)]. +openssl_client_args(Version, Hostname, Port) -> + ["s_client", "-connect", Hostname ++ ":" ++ integer_to_list(Port), ssl_test_lib:version_flag(Version)]. -openssl_client_args(false, Hostname, Port, ServerName) -> +openssl_client_args(Version, Hostname, Port, ServerName) -> ["s_client", "-connect", Hostname ++ ":" ++ - integer_to_list(Port), "-servername", ServerName]; -openssl_client_args(true, Hostname, Port, ServerName) -> - ["s_client", "-no_ssl2", "-connect", Hostname ++ ":" ++ - integer_to_list(Port), "-servername", ServerName]. + integer_to_list(Port), ssl_test_lib:version_flag(Version), "-servername", ServerName]. + hostname_format(Hostname) -> case lists:member($., Hostname) of @@ -2090,22 +2005,12 @@ hostname_format(Hostname) -> "localhost" end. -no_low_flag("-no_ssl2" = Flag) -> - case ssl_test_lib:supports_ssl_tls_version(sslv2) of - true -> - Flag; - false -> - "" - end; -no_low_flag(Flag) -> - Flag. - openssl_has_common_ciphers(Ciphers) -> OCiphers = ssl_test_lib:common_ciphers(openssl), has_common_ciphers(Ciphers, OCiphers). -has_common_ciphers([], OCiphers) -> +has_common_ciphers([], _) -> false; has_common_ciphers([Cipher | Rest], OCiphers) -> case lists:member(Cipher, OCiphers) of diff --git a/lib/ssl/test/ssl_upgrade_SUITE.erl b/lib/ssl/test/ssl_upgrade_SUITE.erl index 875399db76..ead18aeb73 100644 --- a/lib/ssl/test/ssl_upgrade_SUITE.erl +++ b/lib/ssl/test/ssl_upgrade_SUITE.erl @@ -47,10 +47,7 @@ init_per_suite(Config0) -> {skip, Reason} -> {skip, Reason}; Config -> - Result = - {ok, _} = make_certs:all(proplists:get_value(data_dir, Config), - proplists:get_value(priv_dir, Config)), - ssl_test_lib:cert_options(Config) + ssl_test_lib:make_rsa_cert(Config) end catch _:_ -> {skip, "Crypto did not start"} @@ -149,8 +146,8 @@ use_connection(Socket) -> end. soft_start_connection(Config, ResulProxy) -> - ClientOpts = proplists:get_value(client_verification_opts, Config), - ServerOpts = proplists:get_value(server_verification_opts, Config), + ClientOpts = proplists:get_value(client_rsa_verify_opts, Config), + ServerOpts = proplists:get_value(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = start_server([{node, ServerNode}, {port, 0}, {from, ResulProxy}, @@ -166,8 +163,8 @@ soft_start_connection(Config, ResulProxy) -> {Server, Client}. restart_start_connection(Config, ResulProxy) -> - ClientOpts = proplists:get_value(client_verification_opts, Config), - ServerOpts = proplists:get_value(server_verification_opts, Config), + ClientOpts = proplists:get_value(client_rsa_verify_opts, Config), + ServerOpts = proplists:get_value(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = start_server([{node, ServerNode}, {port, 0}, {from, ResulProxy}, diff --git a/lib/ssl/test/x509_test.erl b/lib/ssl/test/x509_test.erl index fea01efdaf..faf223ae35 100644 --- a/lib/ssl/test/x509_test.erl +++ b/lib/ssl/test/x509_test.erl @@ -22,7 +22,7 @@ -module(x509_test). - -include_lib("public_key/include/public_key.hrl"). +-include_lib("public_key/include/public_key.hrl"). -export([extensions/1, gen_pem_config_files/3]). diff --git a/lib/stdlib/doc/src/beam_lib.xml b/lib/stdlib/doc/src/beam_lib.xml index 8bb4cf9101..bb44ca3201 100644 --- a/lib/stdlib/doc/src/beam_lib.xml +++ b/lib/stdlib/doc/src/beam_lib.xml @@ -470,6 +470,18 @@ CryptoKeyFun(clear) -> term()</code> </func> <func> + <name name="strip" arity="2" since=""/> + <fsummary>Remove chunks not needed by the loader from a BEAM file. + </fsummary> + <desc> + <p>Removes all chunks from a BEAM + file except those needed by the loader or passed in. In particular, + the debug information (chunk <c>debug_info</c> and <c>abstract_code</c>) + is removed.</p> + </desc> + </func> + + <func> <name name="strip_files" arity="1" since=""/> <fsummary>Removes chunks not needed by the loader from BEAM files. </fsummary> @@ -483,6 +495,19 @@ CryptoKeyFun(clear) -> term()</code> </func> <func> + <name name="strip_files" arity="2" since=""/> + <fsummary>Removes chunks not needed by the loader from BEAM files. + </fsummary> + <desc> + <p>Removes all chunks except + those needed by the loader or passed in from BEAM files. In particular, + the debug information (chunk <c>debug_info</c> and <c>abstract_code</c>) + is removed. The returned list contains one element for each + specified filename, in the same order as in <c>Files</c>.</p> + </desc> + </func> + + <func> <name name="strip_release" arity="1" since=""/> <fsummary>Remove chunks not needed by the loader from all BEAM files of a release.</fsummary> @@ -497,6 +522,20 @@ CryptoKeyFun(clear) -> term()</code> </func> <func> + <name name="strip_release" arity="2" since=""/> + <fsummary>Remove chunks not needed by the loader from all BEAM files of + a release.</fsummary> + <desc> + <p>Removes all chunks + except those needed by the loader or passed in from the BEAM files of a + release. <c><anno>Dir</anno></c> is to be the installation root + directory. For example, the current OTP release can be + stripped with the call + <c>beam_lib:strip_release(code:root_dir())</c>.</p> + </desc> + </func> + + <func> <name name="version" arity="1" since=""/> <fsummary>Read the module version of the BEAM file.</fsummary> <desc> diff --git a/lib/stdlib/doc/src/calendar.xml b/lib/stdlib/doc/src/calendar.xml index 518a085c89..6308420c52 100644 --- a/lib/stdlib/doc/src/calendar.xml +++ b/lib/stdlib/doc/src/calendar.xml @@ -513,7 +513,7 @@ <title>Date and Time Source</title> <p>Local time is obtained from the Erlang BIF <c>localtime/0</c>. Universal time is computed from the BIF <c>universaltime/0</c>.</p> - <p>The following fapply:</p> + <p>The following apply:</p> <list type="bulleted"> <item>There are 86400 seconds in a day.</item> <item>There are 365 days in an ordinary year.</item> diff --git a/lib/stdlib/doc/src/dets.xml b/lib/stdlib/doc/src/dets.xml index 8e4e002000..8b9502a3b1 100644 --- a/lib/stdlib/doc/src/dets.xml +++ b/lib/stdlib/doc/src/dets.xml @@ -1090,8 +1090,8 @@ ok </item> <item> <p><c>select</c> - The table is traversed by calling - <seealso marker="dets:select/3"><c>dets:select/3</c></seealso> and - <seealso marker="dets:select/1"><c>dets:select/1</c></seealso>. + <seealso marker="dets#select/3"><c>dets:select/3</c></seealso> and + <seealso marker="dets#select/1"><c>dets:select/1</c></seealso>. Option <c>n_objects</c> determines the number of objects returned (the third argument of <c>select/3</c>). The match specification (the second argument of diff --git a/lib/stdlib/doc/src/digraph_utils.xml b/lib/stdlib/doc/src/digraph_utils.xml index 13b0aaad9e..a23b02c6c1 100644 --- a/lib/stdlib/doc/src/digraph_utils.xml +++ b/lib/stdlib/doc/src/digraph_utils.xml @@ -371,7 +371,7 @@ the default, the type of <c><anno>Digraph</anno></c> is used for the subgraph as well. Otherwise the option value of <c>type</c> is used as argument to - <seealso marker="digraph:new/1"><c>digraph:new/1</c></seealso>.</p> + <seealso marker="digraph#new/1"><c>digraph:new/1</c></seealso>.</p> <p>If the value of option <c>keep_labels</c> is <c>true</c>, which is the default, the <seealso marker="#label">labels</seealso> of vertices and edges diff --git a/lib/stdlib/doc/src/erl_pp.xml b/lib/stdlib/doc/src/erl_pp.xml index f1c3aa5a41..0a46139db6 100644 --- a/lib/stdlib/doc/src/erl_pp.xml +++ b/lib/stdlib/doc/src/erl_pp.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>1996</year> - <year>2016</year> + <year>2019</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -65,6 +65,10 @@ </datatype> <datatype> <name name="option"/> + <desc> + <p>The option <c>quote_singleton_atom_types</c> + is used to add quotes to all singleton atom types.</p> + </desc> </datatype> <datatype> <name name="options"/> diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index b9ceaa1d69..2cb677785d 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -188,6 +188,31 @@ is used to keep the table fixated during the entire traversal.</p> </item> </list> + <note> + <p>Even though the access of a single object is always guaranteed to be + <seealso marker="#concurrency">atomic and isolated</seealso>, each traversal + through a table to find the next key is not done with such guarantees. This is often + not a problem, but may cause rare subtle "unexpected" effects if a concurrent + process inserts objects during a traversal. For example, consider one + process doing</p> +<pre> +ets:new(t, [ordered_set, named_table]), +ets:insert(t, {1}), +ets:insert(t, {2}), +ets:insert(t, {3}), +</pre> + <p>A concurrent call to <c>ets:first(t)</c>, done by another + process, may then in rare cases return <c>2</c> even though + <c>2</c> has never existed in the table ordered as the first key. In + the same way, a concurrent call to <c>ets:next(t, 1)</c> may return + <c>3</c> even though <c>3</c> never existed in the table + ordered directly after <c>1</c>.</p> + <p>Effects like this are improbable but possible. The probability will + further be reduced (if not vanish) if table option + <seealso marker="#new_2_write_concurrency"><c>write_concurrency</c></seealso> + is not enabled. This can also only be a potential concern for + <c>ordered_set</c> where the traversal order is defined.</p> + </note> <p>Traversals using <c>match</c> and <c>select</c> functions may not need to scan the entire table depending on how the key is specified. A match pattern with a <em>fully bound key</em> (without any match variables) will @@ -617,12 +642,11 @@ Error: fun containing local Erlang function calls <p><marker id="info_2_safe_fixed_monotonic_time"/></p> <p><c>Item=safe_fixed|safe_fixed_monotonic_time, Value={FixationTime,Info}|false</c></p> - <p>If the table has been fixed using + <p>If the table is fixed using <seealso marker="#safe_fixtable/2"> <c>safe_fixtable/2</c></seealso>, the call returns a tuple where <c>FixationTime</c> is the - time when the table was first fixed by a process, which either - is or is not one of the processes it is fixed by now.</p> + last time when the table changed from unfixed to fixed.</p> <p>The format and value of <c>FixationTime</c> depends on <c>Item</c>:</p> <taglist> @@ -654,14 +678,20 @@ Error: fun containing local Erlang function calls table is fixed by now. <c>RefCount</c> is the value of the reference counter and it keeps track of how many times the table has been fixed by the process.</p> - <p>If the table never has been fixed, the call returns - <c>false</c>.</p> + <p>Table fixations are not limited to <seealso marker="#safe_fixtable/2"> + <c>safe_fixtable/2</c></seealso>. Temporary fixations may also + be done by for example <seealso marker="#traversal">traversing + functions</seealso> like <c>select</c> and <c>match</c>. Such + table fixations are automatically released before the + corresponding functions returns, but they may be seen by a + concurrent call to + <c>ets:info(T,safe_fixed|safe_fixed_monotonic_time)</c>.</p> + <p>If the table is not fixed at all, the call returns <c>false</c>.</p> </item> <item> <p><c>Item=stats, Value=tuple()</c></p> - <p>Returns internal statistics about <c>set</c>, <c>bag</c>, and - <c>duplicate_bag</c> tables on an internal format used by OTP - test suites. Not for production use.</p></item> + <p>Returns internal statistics about tables on an internal format + used by OTP test suites. Not for production use.</p></item> </list> </desc> </func> @@ -1190,11 +1220,17 @@ ets:select(Table, MatchSpec),</code> Functions that makes such promises over many objects (like <seealso marker="#insert/2"><c>insert/2</c></seealso>) gain less (or nothing) from this option.</p> - <p>Table type <c>ordered_set</c> is not affected by this option. - Also, the memory consumption inflicted by - both <c>write_concurrency</c> and <c>read_concurrency</c> is a - constant overhead per table. This overhead can be especially - large when both options are combined.</p> + <p>The memory consumption inflicted by both <c>write_concurrency</c> + and <c>read_concurrency</c> is a constant overhead per table for + <c>set</c>, <c>bag</c> and <c>duplicate_bag</c>. For + <c>ordered_set</c> the memory overhead depends on the number + of inserted objects and the amount of actual detected + concurrency in runtime. The memory overhead can be especially + large when both options are combined.</p> + <note> + <p>Prior to stdlib-3.7 (OTP-22.0) <c>write_concurrency</c> had no + effect on <c>ordered_set</c>.</p> + </note> <marker id="new_2_read_concurrency"></marker> </item> <tag><c>{read_concurrency,boolean()}</c></tag> diff --git a/lib/stdlib/doc/src/filename.xml b/lib/stdlib/doc/src/filename.xml index ae42846c6b..60b7eb3436 100644 --- a/lib/stdlib/doc/src/filename.xml +++ b/lib/stdlib/doc/src/filename.xml @@ -546,7 +546,7 @@ true <p><em>Examples:</em></p> <pre> 20> <input>filename:rootname("/beam.src/kalle").</input> -/beam.src/kalle" +"/beam.src/kalle" 21> <input>filename:rootname("/beam.src/foo.erl").</input> "/beam.src/foo" 22> <input>filename:rootname("/beam.src/foo.erl", ".erl").</input> diff --git a/lib/stdlib/doc/src/gb_trees.xml b/lib/stdlib/doc/src/gb_trees.xml index 570c9c7cb6..08aa1865e8 100644 --- a/lib/stdlib/doc/src/gb_trees.xml +++ b/lib/stdlib/doc/src/gb_trees.xml @@ -42,11 +42,8 @@ <section> <title>Data Structure</title> - <code type="none"> -{Size, Tree}</code> - - <p><c>Tree</c> is composed of nodes of the form <c>{Key, Value, Smaller, - Bigger}</c> and the "empty tree" node <c>nil</c>.</p> + <p>Trees and iterators are built using opaque data structures that should + not be pattern-matched from outside this module.</p> <p>There is no attempt to balance trees after deletions. As deletions do not increase the height of a tree, this should be OK.</p> diff --git a/lib/stdlib/doc/src/gen_statem.xml b/lib/stdlib/doc/src/gen_statem.xml index aaa26df18d..6f6849a19d 100644 --- a/lib/stdlib/doc/src/gen_statem.xml +++ b/lib/stdlib/doc/src/gen_statem.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2016</year><year>2018</year> + <year>2016</year><year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -167,7 +167,7 @@ erlang:'!' -----> Module:StateName/3 </p> <marker id="state callback"/> <p> - The "<em>state callback</em>" for a specific + The <em>state callback</em> for a specific <seealso marker="#type-state">state</seealso> in a <c>gen_statem</c> is the callback function that is called for all events in this state. It is selected depending on which @@ -179,7 +179,7 @@ erlang:'!' -----> Module:StateName/3 When the <seealso marker="#type-callback_mode"><em>callback mode</em></seealso> is <c>state_functions</c>, the state must be an atom and - is used as the state callback name; see + is used as the <em>state callback</em> name; see <seealso marker="#Module:StateName/3"><c>Module:StateName/3</c></seealso>. This co-locates all code for a specific state in one function as the <c>gen_statem</c> engine @@ -192,7 +192,7 @@ erlang:'!' -----> Module:StateName/3 When the <seealso marker="#type-callback_mode"><em>callback mode</em></seealso> is <c>handle_event_function</c>, the state can be any term - and the state callback name is + and the <em>state callback</em> name is <seealso marker="#Module:handle_event/4"><c>Module:handle_event/4</c></seealso>. This makes it easy to branch depending on state or event as you desire. Be careful about which events you handle in which @@ -200,12 +200,36 @@ erlang:'!' -----> Module:StateName/3 forever creating an infinite busy loop. </p> <p> - The <c>gen_statem</c> enqueues incoming events in order of arrival - and presents these to the - <seealso marker="#state callback">state callback</seealso> - in that order. The state callback can postpone an event - so it is not retried in the current state. - After a state change the queue restarts with the postponed events. + When <c>gen_statem</c> receives a process message it is + converted into an event and the + <seealso marker="#state callback"><em>state callback</em></seealso> + is called with the event as two arguments: type and content. + When the + <seealso marker="#state callback"><em>state callback</em></seealso> + has processed the event it returns to <c>gen_statem</c> + which does a <em>state transition</em>. + If this <em>state transition</em> is to a different state, + that is: <c>NextState =/= State</c>, it is a <em>state change</em>. + </p> + <p> + The + <seealso marker="#state callback"><em>state callback</em></seealso> + may return + <seealso marker="#type-action"><em>transition actions</em></seealso> + for <c>gen_statem</c> + to execute during the <em>state transition</em>, + for example to reply to a + <seealso marker="#call/2"><c>gen_statem:call/2,3</c></seealso>. + </p> + <p> + One of the possible <em>transition actions</em> + is to postpone the current event. + Then it is not retried in the current state. + The <c>gen_statem</c> engine keeps a queue of events + divided into the postponed events + and the events still to process. + After a <em>state change</em> the queue restarts + with the postponed events. </p> <p> The <c>gen_statem</c> event queue model is sufficient @@ -215,13 +239,17 @@ erlang:'!' -----> Module:StateName/3 to entering a new receive statement. </p> <p> - The <seealso marker="#state callback">state callback</seealso> + The + <seealso marker="#state callback"><em>state callback</em></seealso> can insert events using the - <seealso marker="#type-action"><c>action()</c></seealso> + <seealso marker="#type-action"><em>transition actions</em></seealso> <c>next_event</c> - and such an event is inserted as the next to present - to the state callback. That is, as if it is - the oldest incoming event. A dedicated + and such an event is inserted in the event queue + as the next to call the + <seealso marker="#state callback"><em>state callback</em></seealso> + with. + That is, as if it is the oldest incoming event. + A dedicated <seealso marker="#type-event_type"><c>event_type()</c></seealso> <c>internal</c> can be used for such events making them impossible to mistake for external events. @@ -236,24 +264,26 @@ erlang:'!' -----> Module:StateName/3 <p> The <c>gen_statem</c> engine can automatically make a specialized call to the - <seealso marker="#state callback">state callback</seealso> + <seealso marker="#state callback"><em>state callback</em></seealso> whenever a new state is entered; see <seealso marker="#type-state_enter"><c>state_enter()</c></seealso>. This is for writing code common to all state entries. - Another way to do it is to insert an event at the state transition, - and/or to use a dedicated state transition function, + Another way to do it is to explicitly insert an event + at the <em>state transition</em>, + and/or to use a dedicated <em>state transition</em> function, but that is something you will have to remember - at every state transition to the state(s) that need it. + at every <em>state transition</em> to the state(s) that need it. </p> <note> <p>If you in <c>gen_statem</c>, for example, postpone - an event in one state and then call another state callback - of yours, you have not changed states and hence the postponed event - is not retried, which is logical but can be confusing. + an event in one state and then call another <em>state callback</em> + of yours, you have not done a <em>state change</em> + and hence the postponed event is not retried, + which is logical but can be confusing. </p> </note> <p> - For the details of a state transition, see type + For the details of a <em>state transition</em>, see type <seealso marker="#type-transition_option"><c>transition_option()</c></seealso>. </p> <p> @@ -276,7 +306,8 @@ erlang:'!' -----> Module:StateName/3 The <c>gen_statem</c> process can go into hibernation; see <seealso marker="proc_lib#hibernate/3"><c>proc_lib:hibernate/3</c></seealso>. It is done when a - <seealso marker="#state callback">state callback</seealso> or + <seealso marker="#state callback"><em>state callback</em></seealso> + or <seealso marker="#Module:init/1"><c>Module:init/1</c></seealso> specifies <c>hibernate</c> in the returned <seealso marker="#type-action"><c>Actions</c></seealso> @@ -289,12 +320,13 @@ erlang:'!' -----> Module:StateName/3 </p> <p> There is also a server start option - <seealso marker="#type-hibernate_after_opt"> + <seealso marker="#type-enter_loop_opt"> <c>{hibernate_after, Timeout}</c> </seealso> for - <seealso marker="#start/3"><c>start/3,4</c></seealso> or - <seealso marker="#start_link/3"><c>start_link/3,4</c></seealso> + <seealso marker="#start/3"><c>start/3,4</c></seealso>, + <seealso marker="#start_link/3"><c>start_link/3,4</c></seealso> or + <seealso marker="#enter_loop/4"><c>enter_loop/4,5,6</c></seealso>, that may be used to automatically hibernate the server. </p> </description> @@ -483,36 +515,6 @@ handle_event(_, _, State, Data) -> </desc> </datatype> <datatype> - <name name="debug_opt"/> - <desc> - <p> - Debug option that can be used when starting - a <c>gen_statem</c> server through, - <seealso marker="#enter_loop/4"><c>enter_loop/4-6</c></seealso>. - </p> - <p> - For every entry in <c><anno>Dbgs</anno></c>, - the corresponding function in - <seealso marker="sys"><c>sys</c></seealso> is called. - </p> - </desc> - </datatype> - <datatype> - <name name="hibernate_after_opt"/> - <desc> - <p> - hibernate_after option that can be used when starting - a <c>gen_statem</c> server through, - <seealso marker="#enter_loop/4"><c>enter_loop/4-6</c></seealso>. - </p> - <p>If option<seealso marker="#type-hibernate_after_opt"><c>{hibernate_after,HibernateAfterTimeout}</c></seealso> is present, the <c>gen_statem</c> - process awaits any message for <c>HibernateAfterTimeout</c> milliseconds and - if no message is received, the process goes into hibernation automatically - (by calling <seealso marker="proc_lib#hibernate/3"><c>proc_lib:hibernate/3</c></seealso>). - </p> - </desc> - </datatype> - <datatype> <name name="start_opt"/> <desc> <p> @@ -532,6 +534,37 @@ handle_event(_, _, State, Data) -> </desc> </datatype> <datatype> + <name name="enter_loop_opt"/> + <desc> + <p> + Options that can be used when starting + a <c>gen_statem</c> server through, + <seealso marker="#enter_loop/4"><c>enter_loop/4-6</c></seealso>. + </p> + <taglist> + <tag><c>hibernate_after</c></tag> + <item> + <p> + <c>HibernateAfterTimeout</c> + specifies that the <c>gen_statem</c> process awaits + any message for <c>HibernateAfterTimeout</c> milliseconds and + if no message is received, the process goes into hibernation + automatically (by calling + <seealso marker="proc_lib#hibernate/3"><c>proc_lib:hibernate/3</c></seealso>). + </p> + </item> + <tag><c>debug</c></tag> + <item> + <p> + For every entry in <c><anno>Dbgs</anno></c>, + the corresponding function in + <seealso marker="sys"><c>sys</c></seealso> is called. + </p> + </item> + </taglist> + </desc> + </datatype> + <datatype> <name name="from"/> <desc> <p> @@ -551,7 +584,7 @@ handle_event(_, _, State, Data) -> <seealso marker="#type-callback_mode"><em>callback mode</em></seealso> is <c>handle_event_function</c>, the state can be any term. - After a state change (<c>NextState =/= State</c>), + After a <em>state change</em> (<c>NextState =/= State</c>), all postponed events are retried. </p> </desc> @@ -564,7 +597,7 @@ handle_event(_, _, State, Data) -> <seealso marker="#type-callback_mode"><em>callback mode</em></seealso> is <c>state_functions</c>, the state must be of this type. - After a state change (<c>NextState =/= State</c>), + After a <em>state change</em> (<c>NextState =/= State</c>), all postponed events are retried. </p> </desc> @@ -595,7 +628,7 @@ handle_event(_, _, State, Data) -> </p> <p> <c>internal</c> events can only be generated by the - state machine itself through the state transition action + state machine itself through the <em>transition action</em> <seealso marker="#type-action"><c>next_event</c></seealso>. </p> </desc> @@ -633,9 +666,9 @@ handle_event(_, _, State, Data) -> This is the return type from <seealso marker="#Module:callback_mode/0"><c>Module:callback_mode/0</c></seealso> and selects - <seealso marker="#type-callback_mode">callback mode</seealso> + <seealso marker="#type-callback_mode"><em>callback mode</em></seealso> and whether to do - <seealso marker="#type-state_enter">state enter calls</seealso>, + <seealso marker="#type-state_enter"><em>state enter calls</em></seealso>, or not. </p> </desc> @@ -684,13 +717,15 @@ handle_event(_, _, State, Data) -> If <seealso marker="#Module:callback_mode/0"><c>Module:callback_mode/0</c></seealso> returns a list containing <c>state_enter</c>, - the <c>gen_statem</c> engine will, at every state change, + the <c>gen_statem</c> engine will, at every <em>state change</em>, call the <seealso marker="#state callback">state callback</seealso> with arguments <c>(enter, OldState, Data)</c>. This may look like an event but is really a call - performed after the previous state callback returned - and before any event is delivered to the new state callback. + performed after the previous + <seealso marker="#state callback"><em>state callback</em></seealso> + returned and before any event is delivered to the new + <seealso marker="#state callback"><em>state callback</em></seealso>. See <seealso marker="#Module:StateName/3"><c>Module:StateName/3</c></seealso> and @@ -703,27 +738,27 @@ handle_event(_, _, State, Data) -> <seealso marker="#type-state_callback_result"> <c>repeat_state_and_data</c> </seealso> - tuple from the state callback. + tuple from the <em>state callback</em>. </p> <p> If <seealso marker="#Module:callback_mode/0"><c>Module:callback_mode/0</c></seealso> - does not return such a list, no state enter calls are done. + does not return such a list, no <em>state enter calls</em> are done. </p> <p> If <seealso marker="#Module:code_change/4"><c>Module:code_change/4</c></seealso> should transform the state, - it is regarded as a state rename and not a state change, - which will not cause a state enter call. + it is regarded as a state rename and not a <em>state change</em>, + which will not cause a <em>state enter call</em>. </p> <p> - Note that a state enter call <em>will</em> be done + Note that a <em>state enter call</em> <em>will</em> be done right before entering the initial state even though this - formally is not a state change. - In this case <c>OldState</c> will be the same as <c>State</c>, - which can not happen for a subsequent state change, - but will happen when repeating the state enter call. + actually is not a <em>state change</em>. + In this case <c>OldState =:= State</c>, + which cannot happen for a subsequent state change, + but will happen when repeating the <em>state enter call</em>. </p> </desc> </datatype> @@ -733,8 +768,11 @@ handle_event(_, _, State, Data) -> <p> Transition options can be set by <seealso marker="#type-action">actions</seealso> - and modify the state transition. - Here are the sequence of steps for a state transition: + and modify the <em>state transition</em>. + The <em>state transition</em> takes place when the + <seealso marker="#state callback"><em>state callback</em></seealso> + has processed an event and returns. + Here are the sequence of steps for a <em>state transition</em>: </p> <list type="ordered"> <item> @@ -765,7 +803,7 @@ handle_event(_, _, State, Data) -> returned by the state callback that caused the state entry. </p> <p> - Should this state enter call return any of + Should this <em>state enter call</em> return any of the mentioned <c>repeat_*</c> callback results it is repeated again, with the updated <c>Data</c>. </p> @@ -787,7 +825,8 @@ handle_event(_, _, State, Data) -> </item> <item> <p> - If the state changes, the queue of incoming events + If this is a <em>state change</em>, + the queue of incoming events is reset to start with the oldest postponed. </p> </item> @@ -821,7 +860,7 @@ handle_event(_, _, State, Data) -> if the event queue is empty. </p> <p> - A state change cancels a + A <em>state change</em> cancels a <seealso marker="#type-state_timeout"><c>state_timeout()</c></seealso> and any new transition option of this type belongs to the new state. @@ -830,7 +869,7 @@ handle_event(_, _, State, Data) -> <item> <p> If there are enqueued events the - <seealso marker="#state callback">state callback</seealso> + <seealso marker="#state callback"><em>state callback</em></seealso> for the possibly new state is called with the oldest enqueued event, and we start again from the top of this list. @@ -848,7 +887,7 @@ handle_event(_, _, State, Data) -> the next incoming message awakens the <c>gen_statem</c>, but if it is a system event it goes right back into hibernation. When a new message arrives the - <seealso marker="#state callback">state callback</seealso> + <seealso marker="#state callback"><em>state callback</em></seealso> is called with the corresponding event, and we start again from the top of this sequence. </p> @@ -861,7 +900,7 @@ handle_event(_, _, State, Data) -> <desc> <p> If <c>true</c>, postpones the current event and retries - it when the state changes + it after a <em>state change</em> (<c>NextState =/= State</c>). </p> </desc> @@ -1021,9 +1060,9 @@ handle_event(_, _, State, Data) -> <name name="action"/> <desc> <p> - These state transition actions can be invoked by + These <em>transition actions</em> can be invoked by returning them from the - <seealso marker="#state callback">state callback</seealso> + <seealso marker="#state callback"><em>state callback</em></seealso> when it is called with an <seealso marker="#type-event_type">event</seealso>, from @@ -1054,7 +1093,7 @@ handle_event(_, _, State, Data) -> <c>transition_option()</c> </seealso> <seealso marker="#type-postpone"><c>postpone()</c></seealso> - for this state transition. + for this <em>state transition</em>. This action is ignored when returned from <seealso marker="#Module:init/1"><c>Module:init/1</c></seealso> or given to @@ -1093,9 +1132,9 @@ handle_event(_, _, State, Data) -> <name name="enter_action"/> <desc> <p> - These state transition actions can be invoked by + These <em>transition actions</em> can be invoked by returning them from the - <seealso marker="#state callback">state callback</seealso>, from + <seealso marker="#state callback"><em>state callback</em></seealso>, from <seealso marker="#Module:init/1"><c>Module:init/1</c></seealso> or by giving them to <seealso marker="#enter_loop/5"><c>enter_loop/5,6</c></seealso>. @@ -1119,7 +1158,7 @@ handle_event(_, _, State, Data) -> Sets the <seealso marker="#type-transition_option"><c>transition_option()</c></seealso> <seealso marker="#type-hibernate"><c>hibernate()</c></seealso> - for this state transition. + for this <em>state transition</em>. </p> </item> </taglist> @@ -1129,9 +1168,9 @@ handle_event(_, _, State, Data) -> <name name="timeout_action"/> <desc> <p> - These state transition actions can be invoked by + These <em>transition actions</em> can be invoked by returning them from the - <seealso marker="#state callback">state callback</seealso>, from + <seealso marker="#state callback"><em>state callback</em></seealso>, from <seealso marker="#Module:init/1"><c>Module:init/1</c></seealso> or by giving them to <seealso marker="#enter_loop/5"><c>enter_loop/5,6</c></seealso>. @@ -1147,7 +1186,7 @@ handle_event(_, _, State, Data) -> Short for <c>{timeout,Time,Time}</c>, that is, the time-out message is the time-out time. This form exists to make the - <seealso marker="#state callback">state callback</seealso> + <seealso marker="#state callback"><em>state callback</em></seealso> return value <c>{next_state,NextState,NewData,Time}</c> allowed like for <c>gen_fsm</c>. </p> @@ -1193,9 +1232,9 @@ handle_event(_, _, State, Data) -> <name name="reply_action"/> <desc> <p> - This state transition action can be invoked by + This <em>transition action</em> can be invoked by returning it from the - <seealso marker="#state callback">state callback</seealso>, from + <seealso marker="#state callback"><em>state callback</em></seealso>, from <seealso marker="#Module:init/1"><c>Module:init/1</c></seealso> or by giving it to <seealso marker="#enter_loop/5"><c>enter_loop/5,6</c></seealso>. @@ -1210,7 +1249,7 @@ handle_event(_, _, State, Data) -> <c><anno>From</anno></c> must be the term from argument <seealso marker="#type-event_type"><c>{call,<anno>From</anno>}</c></seealso> in a call to a - <seealso marker="#state callback">state callback</seealso>. + <seealso marker="#state callback"><em>state callback</em></seealso>. </p> <p> Note that using this action from @@ -1219,7 +1258,7 @@ handle_event(_, _, State, Data) -> <seealso marker="#enter_loop/5"><c>enter_loop/5,6</c></seealso> would be weird on the border of witchcraft since there has been no earlier call to a - <seealso marker="#state callback">state callback</seealso> + <seealso marker="#state callback"><em>state callback</em></seealso> in this server. </p> </desc> @@ -1239,7 +1278,7 @@ handle_event(_, _, State, Data) -> The <seealso marker="#type-action"><c>Actions</c></seealso> are executed when entering the first <seealso marker="#type-state">state</seealso> just as for a - <seealso marker="#state callback">state callback</seealso>, + <seealso marker="#state callback"><em>state callback</em></seealso>, except that the action <c>postpone</c> is forced to <c>false</c> since there is no event to postpone. </p> @@ -1256,7 +1295,7 @@ handle_event(_, _, State, Data) -> <desc> <p> <c><anno>State</anno></c> is the current state - and it can not be changed since the state callback + and it cannot be changed since the state callback was called with a <seealso marker="#type-state_enter"><em>state enter call</em></seealso>. </p> @@ -1292,11 +1331,13 @@ handle_event(_, _, State, Data) -> <tag><c>next_state</c></tag> <item> <p> - The <c>gen_statem</c> does a state transition to + The <c>gen_statem</c> does a <em>state transition</em> to <c><anno>NextState</anno></c> (which can be the same as the current state), sets <c><anno>NewData</anno></c>, and executes all <c><anno>Actions</anno></c>. + If <c><anno>NextState</anno> =/= CurrentState</c> + the <em>state transition</em> is a <em>state change</em>. </p> </item> </taglist> @@ -1318,54 +1359,33 @@ handle_event(_, _, State, Data) -> <tag><c>keep_state</c></tag> <item> <p> - The <c>gen_statem</c> keeps the current state, or - does a state transition to the current state if you like, - sets <c><anno>NewData</anno></c>, - and executes all <c><anno>Actions</anno></c>. - This is the same as + The same as <c>{next_state,CurrentState,<anno>NewData</anno>,<anno>Actions</anno>}</c>. </p> </item> <tag><c>keep_state_and_data</c></tag> <item> <p> - The <c>gen_statem</c> keeps the current state or - does a state transition to the current state if you like, - keeps the current server data, - and executes all <c><anno>Actions</anno></c>. - This is the same as - <c>{next_state,CurrentState,CurrentData,<anno>Actions</anno>}</c>. + The same as + <c>{keep_state,CurrentData,<anno>Actions</anno>}</c>. </p> </item> <tag><c>repeat_state</c></tag> <item> <p> - The <c>gen_statem</c> keeps the current state, or - does a state transition to the current state if you like, - sets <c><anno>NewData</anno></c>, - and executes all <c><anno>Actions</anno></c>. If the <c>gen_statem</c> runs with <seealso marker="#type-state_enter"><em>state enter calls</em></seealso>, - the state enter call is repeated, see type + the <em>state enter call</em> is repeated, see type <seealso marker="#type-transition_option"><c>transition_option()</c></seealso>, - otherwise <c>repeat_state</c> is the same as + other than that <c>repeat_state</c> is the same as <c>keep_state</c>. </p> </item> <tag><c>repeat_state_and_data</c></tag> <item> <p> - The <c>gen_statem</c> keeps the current state and data, or - does a state transition to the current state if you like, - and executes all <c><anno>Actions</anno></c>. - This is the same as + The same as <c>{repeat_state,CurrentData,<anno>Actions</anno>}</c>. - If the <c>gen_statem</c> runs with - <seealso marker="#type-state_enter"><em>state enter calls</em></seealso>, - the state enter call is repeated, see type - <seealso marker="#type-transition_option"><c>transition_option()</c></seealso>, - otherwise <c>repeat_state_and_data</c> is the same as - <c>keep_state_and_data</c>. </p> </item> <tag><c>stop</c></tag> @@ -1408,14 +1428,15 @@ handle_event(_, _, State, Data) -> by sending a request and waiting until its reply arrives. The <c>gen_statem</c> calls the - <seealso marker="#state callback">state callback</seealso> with + <seealso marker="#state callback"><em>state callback</em></seealso> + with <seealso marker="#type-event_type"><c>event_type()</c></seealso> <c>{call,From}</c> and event content <c><anno>Request</anno></c>. </p> <p> A <c><anno>Reply</anno></c> is generated when a - <seealso marker="#state callback">state callback</seealso> + <seealso marker="#state callback"><em>state callback</em></seealso> returns with <c>{reply,From,<anno>Reply</anno>}</c> as one <seealso marker="#type-action"><c>action()</c></seealso>, @@ -1484,7 +1505,8 @@ handle_event(_, _, State, Data) -> ignoring if the destination node or <c>gen_statem</c> does not exist. The <c>gen_statem</c> calls the - <seealso marker="#state callback">state callback</seealso> with + <seealso marker="#state callback"><em>state callback</em></seealso> + with <seealso marker="#type-event_type"><c>event_type()</c></seealso> <c>cast</c> and event content <c><anno>Msg</anno></c>. @@ -1598,18 +1620,18 @@ handle_event(_, _, State, Data) -> <seealso marker="#call/2"><c>call/2</c></seealso> when the reply cannot be defined in the return value of a - <seealso marker="#state callback">state callback</seealso>. + <seealso marker="#state callback"><em>state callback</em></seealso>. </p> <p> <c><anno>From</anno></c> must be the term from argument <seealso marker="#type-event_type"><c>{call,<anno>From</anno>}</c></seealso> to the - <seealso marker="#state callback">state callback</seealso>. + <seealso marker="#state callback"><em>state callback</em></seealso>. A reply or multiple replies canalso be sent using one or several <seealso marker="#type-reply_action"><c>reply_action()</c></seealso>s from a - <seealso marker="#state callback">state callback</seealso>. + <seealso marker="#state callback"><em>state callback</em></seealso>. </p> <note> <p> @@ -1681,7 +1703,11 @@ handle_event(_, _, State, Data) -> <list type="bulleted"> <item> <p> - If option <c>{timeout,Time}</c> is present in + If option + <seealso marker="#type-start_opt"> + <c>{timeout,Time}</c> + </seealso> + is present in <c><anno>Opts</anno></c>, the <c>gen_statem</c> is allowed to spend <c>Time</c> milliseconds initializing or it terminates and the start function returns @@ -1689,23 +1715,33 @@ handle_event(_, _, State, Data) -> </p> </item> <item> - <p>If option<seealso marker="#type-hibernate_after_opt"><c>{hibernate_after,HibernateAfterTimeout}</c></seealso> is present, the <c>gen_statem</c> - process awaits any message for <c>HibernateAfterTimeout</c> milliseconds and - if no message is received, the process goes into hibernation automatically - (by calling <seealso marker="proc_lib#hibernate/3"><c>proc_lib:hibernate/3</c></seealso>). + <p>If option + <seealso marker="#type-enter_loop_opt"> + <c>{hibernate_after,HibernateAfterTimeout}</c> + </seealso> + is present, the <c>gen_statem</c> + process awaits any message for <c>HibernateAfterTimeout</c> milliseconds and + if no message is received, the process goes into hibernation automatically + (by calling <seealso marker="proc_lib#hibernate/3"><c>proc_lib:hibernate/3</c></seealso>). </p> </item> <item> <p> If option - <seealso marker="#type-debug_opt"><c>{debug,Dbgs}</c></seealso> + <seealso marker="#type-enter_loop_opt"> + <c>{debug,Dbgs}</c> + </seealso> is present in <c><anno>Opts</anno></c>, debugging through <seealso marker="sys"><c>sys</c></seealso> is activated. </p> </item> <item> <p> - If option <c>{spawn_opt,SpawnOpts}</c> is present in + If option + <seealso marker="#type-start_opt"> + <c>{spawn_opt,SpawnOpts}</c> + </seealso> + is present in <c><anno>Opts</anno></c>, <c>SpawnOpts</c> is passed as option list to <seealso marker="erts:erlang#spawn_opt/2"><c>erlang:spawn_opt/2</c></seealso>, @@ -1826,7 +1862,7 @@ handle_event(_, _, State, Data) -> for efficiency reasons, so this function is only called once after server start and after code change, but before the first - <seealso marker="#state callback">state callback</seealso> + <seealso marker="#state callback"><em>state callback</em></seealso> in the current code version is called. More occasions may be added in future versions of <c>gen_statem</c>. @@ -1883,7 +1919,7 @@ handle_event(_, _, State, Data) -> <p> This callback is optional, so callback modules need not export it. If a release upgrade/downgrade with - <c>Change={advanced,Extra}</c> + <c>Change = {advanced,Extra}</c> specified in the <c>.appup</c> file is made when <c>code_change/4</c> is not implemented the process will crash with exit reason <c>undef</c>. @@ -1893,7 +1929,7 @@ handle_event(_, _, State, Data) -> This function is called by a <c>gen_statem</c> when it is to update its internal state during a release upgrade/downgrade, that is, when the instruction <c>{update,Module,Change,...}</c>, - where <c>Change={advanced,Extra}</c>, is specified in the + where <c>Change = {advanced,Extra}</c>, is specified in the <seealso marker="sasl:appup"><c>appup</c></seealso> file. For more information, see <seealso marker="doc/design_principles:release_handling#instr">OTP Design Principles</seealso>. @@ -1922,7 +1958,7 @@ handle_event(_, _, State, Data) -> <p> If the function returns a failure <c>Reason</c>, the ongoing upgrade fails and rolls back to the old release. - Note that <c>Reason</c> can not be an <c>{ok,_,_}</c> tuple + Note that <c>Reason</c> cannot be an <c>{ok,_,_}</c> tuple since that will be regarded as a <c>{ok,NewState,NewData}</c> tuple, and that a tuple matching <c>{ok,_}</c> @@ -1933,13 +1969,13 @@ handle_event(_, _, State, Data) -> <p> Also note when upgrading a <c>gen_statem</c>, this function and hence - the <c>Change={advanced,Extra}</c> parameter in the + the <c>Change = {advanced,Extra}</c> parameter in the <seealso marker="sasl:appup"><c>appup</c></seealso> file is not only needed to update the internal state or to act on the <c>Extra</c> argument. It is also needed if an upgrade or downgrade should change <seealso marker="#type-callback_mode"><em>callback mode</em></seealso>, - or else the callback mode after the code change + or else the <em>callback mode</em> after the code change will not be honoured, most probably causing a server crash. </p> @@ -2148,7 +2184,7 @@ init(Args) -> erlang:error(not_implemented, [Args]).</pre> <seealso marker="#type-event_type"><c>{call,From}</c></seealso>, the caller waits for a reply. The reply can be sent from this or from any other - <seealso marker="#state callback">state callback</seealso> + <seealso marker="#state callback"><em>state callback</em></seealso> by returning with <c>{reply,From,Reply}</c> in <seealso marker="#type-action"><c>Actions</c></seealso>, in <seealso marker="#type-reply_action"><c>Replies</c></seealso>, @@ -2173,9 +2209,9 @@ init(Args) -> erlang:error(not_implemented, [Args]).</pre> </p> <p> When the <c>gen_statem</c> runs with - <seealso marker="#type-state_enter">state enter calls</seealso>, + <seealso marker="#type-state_enter"><em>state enter calls</em></seealso>, these functions are also called with arguments - <c>(enter, OldState, ...)</c> whenever the state changes. + <c>(enter, OldState, ...)</c> during every <em>state change</em>. In this case there are some restrictions on the <seealso marker="#type-enter_action">actions</seealso> that may be returned: @@ -2208,7 +2244,7 @@ init(Args) -> erlang:error(not_implemented, [Args]).</pre> <seealso marker="erts:erlang#throw/1"><c>throw</c></seealso> to return the result, which can be useful. For example to bail out with <c>throw(keep_state_and_data)</c> - from deep within complex code that can not + from deep within complex code that cannot return <c>{next_state,State,Data}</c> because <c>State</c> or <c>Data</c> is no longer in scope. </p> diff --git a/lib/stdlib/doc/src/maps.xml b/lib/stdlib/doc/src/maps.xml index e2a62cb397..8e88882b56 100644 --- a/lib/stdlib/doc/src/maps.xml +++ b/lib/stdlib/doc/src/maps.xml @@ -35,9 +35,10 @@ <datatypes> <datatype> - <name name="iterator"/> + <name name="iterator" n_vars="2"/> <desc> - <p>An iterator representing the key value associations in a map.</p> + <p>An iterator representing the associations in a map with keys of type + <c><anno>Key</anno></c> and values of type <c><anno>Value</anno></c>.</p> <p>Created using <seealso marker="#iterator-1"><c>maps:iterator/1</c></seealso>.</p> <p>Consumed by <seealso marker="#next-1"><c>maps:next/1</c></seealso>, <seealso marker="#filter-2"><c>maps:filter/2</c></seealso>, @@ -45,6 +46,10 @@ <seealso marker="#map-2"><c>maps:map/2</c></seealso>.</p> </desc> </datatype> + + <datatype> + <name name="iterator" n_vars="0"/> + </datatype> </datatypes> <funcs> @@ -90,13 +95,13 @@ <name name="fold" arity="3" since="OTP 17.0"/> <fsummary></fsummary> <desc> - <p>Calls <c>F(K, V, AccIn)</c> for every <c><anno>K</anno></c> to value - <c><anno>V</anno></c> association in <c><anno>MapOrIter</anno></c> in - any order. Function <c>fun F/3</c> must return a new - accumulator, which is passed to the next successive call. - This function returns the final value of the accumulator. The initial - accumulator value <c><anno>Init</anno></c> is returned if the map is - empty.</p> + <p>Calls <c>F(Key, Value, AccIn)</c> for every <c><anno>Key</anno></c> + to value <c><anno>Value</anno></c> association in + <c><anno>MapOrIter</anno></c> in any order. Function <c>fun F/3</c> + must return a new accumulator, which is passed to the next successive + call. This function returns the final value of the accumulator. + The initial accumulator value <c><anno>Init</anno></c> is returned + if the map is empty.</p> <p>The call fails with a <c>{badmap,Map}</c> exception if <c><anno>MapOrIter</anno></c> is not a map or valid iterator, or with <c>badarg</c> if <c><anno>Fun</anno></c> is not a @@ -234,11 +239,12 @@ none</code> <fsummary></fsummary> <desc> <p>Produces a new map <c><anno>Map</anno></c> by calling function - <c>fun F(K, V1)</c> for every <c><anno>K</anno></c> to value - <c><anno>V1</anno></c> association in <c><anno>MapOrIter</anno></c> in - any order. Function <c>fun F/2</c> must return value - <c><anno>V2</anno></c> to be associated with key <c><anno>K</anno></c> - for the new map <c><anno>Map</anno></c>.</p> + <c>fun F(Key, Value1)</c> for every <c><anno>Key</anno></c> to value + <c><anno>Value1</anno></c> association in + <c><anno>MapOrIter</anno></c> in any order. Function <c>fun Fun/2</c> + must return value <c><anno>Value2</anno></c> to be associated with + key <c><anno>Key</anno></c> for the new map + <c><anno>Map</anno></c>.</p> <p>The call fails with a <c>{badmap,Map}</c> exception if <c><anno>MapOrIter</anno></c> is not a map or valid iterator, or with <c>badarg</c> if <c><anno>Fun</anno></c> is not a diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index f677438154..65650a25c7 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -3884,7 +3884,7 @@ you use erlang:halt/2 with an integer first argument and an option list containing {flush,false} as the second argument. Note that now is flushing not dependant of the - exit code, and you can not only flush async threads + exit code, and you cannot only flush async threads operations which we deemed as a strange behaviour anyway. </p> <p>Also, erlang:halt/1,2 has gotten a new feature: If the @@ -4398,9 +4398,9 @@ Supervisors should not save child-specs for temporary processes when they terminate as they should not be restarted. Saving the temporary child spec will result in - that you can not start a new temporary process with the + that you cannot start a new temporary process with the same child spec as an already terminated temporary - process. Since R14B02 you can not restart a temporary + process. Since R14B02 you cannot restart a temporary temporary process as arguments are no longer saved, it has however always been semantically incorrect to restart a temporary process. Thanks to Filipe David Manana for diff --git a/lib/stdlib/doc/src/proc_lib.xml b/lib/stdlib/doc/src/proc_lib.xml index aeb9f48735..bb983903a9 100644 --- a/lib/stdlib/doc/src/proc_lib.xml +++ b/lib/stdlib/doc/src/proc_lib.xml @@ -166,7 +166,7 @@ <fsummary>Hibernate a process until a message is sent to it.</fsummary> <desc> <p>This function does the same as (and does call) the - <seealso marker="erts:erlang#erlang:hibernate/3"> + <seealso marker="erts:erlang#hibernate/3"> <c>hibernate/3</c></seealso> BIF, but ensures that exception handling and logging continues to work as expected when the process wakes up.</p> diff --git a/lib/stdlib/doc/src/qlc.xml b/lib/stdlib/doc/src/qlc.xml index fe60c2e9bb..34f7c5bab9 100644 --- a/lib/stdlib/doc/src/qlc.xml +++ b/lib/stdlib/doc/src/qlc.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2004</year><year>2016</year> + <year>2004</year><year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -581,11 +581,13 @@ gb_iter(I0, N, EFun) -> <input>{K} <- ets:table(E1),</input> <input>K == 2.71 orelse K == a]),</input> <input>io:format("~s~n", [qlc:info(Q1)]).</input> -ets:match_spec_run(lists:flatmap(fun(V) -> - ets:lookup(20493, V) - end, - [a,2.71]), - ets:match_spec_compile([{{'$1'},[],['$1']}]))</pre> +ets:match_spec_run( + lists:flatmap(fun(V) -> + ets:lookup(#Ref<0.3098908599.2283929601.256025>, + V) + end, + [a, 2.71]), + ets:match_spec_compile([{{'$1'}, [], ['$1']}]))</pre> <p>In the example, operator <c>==/2</c> has been handled exactly as <c>=:=/2</c> would have been handled. However, @@ -607,9 +609,10 @@ ets:match_spec_run(lists:flatmap(fun(V) -> <input>end,</input> <input>Q2 = F2({2,2}),</input> <input>io:format("~s~n", [qlc:info(Q2)]).</input> -ets:table(53264, +ets:table(#Ref<0.3098908599.2283929601.256125>, [{traverse, - {select,[{{'$1','$2'},[{'==','$1',{const,{2,2}}}],['$2']}]}}]) + {select, + [{{'$1', '$2'}, [{'==', '$1', {const, {2, 2}}}], ['$2']}]}}]) 3> <input>lists:sort(qlc:e(Q2)).</input> [a,b,c]</pre> @@ -629,8 +632,9 @@ ets:table(53264, <input>end,</input> <input>Q3 = F3({2,2}),</input> <input>io:format("~s~n", [qlc:info(Q3)]).</input> -ets:match_spec_run(ets:lookup(86033, {2,2}), - ets:match_spec_compile([{{'$1','$2'},[],['$2']}])) +ets:match_spec_run(ets:lookup(#Ref<0.3098908599.2283929601.256211>, + {2, 2}), + ets:match_spec_compile([{{'$1', '$2'}, [], ['$2']}])) 5> <input>qlc:e(Q3).</input> [b]</pre> @@ -892,21 +896,21 @@ begin V1 = qlc:q([ SQV || - SQV <- [x,y] + SQV <- [x, y] ], - [{unique,true}]), + [{unique, true}]), V2 = qlc:q([ SQV || - SQV <- [a,b] + SQV <- [a, b] ], - [{unique,true}]), + [{unique, true}]), qlc:q([ {X,Y} || X <- V1, Y <- V2 ], - [{unique,true}]) + [{unique, true}]) end</pre> <p>In the following example QLC <c>V2</c> has been inserted to show the joined generators and the join @@ -927,19 +931,21 @@ begin V1 = qlc:q([ P0 || - P0 = {W,Y} <- ets:table(17) + P0 = {W, Y} <- + ets:table(#Ref<0.3098908599.2283929601.256549>) ]), V2 = qlc:q([ - [G1|G2] || + [G1 | G2] || G2 <- V1, - G1 <- ets:table(16), + G1 <- + ets:table(#Ref<0.3098908599.2283929601.256548>), element(2, G1) =:= element(1, G2) ], - [{join,lookup}]), + [{join, lookup}]), qlc:q([ - {X,Z,W} || - [{X,Z}|{W,Y}] <- V2 + {X, Z, W} || + [{X, Z} | {W, Y}] <- V2 ]) end</pre> </desc> @@ -1080,27 +1086,27 @@ begin V1 = qlc:q([ P0 || - P0 = {X,Z} <- - qlc:keysort(1, [{a,1},{b,4},{c,6}], []) + P0 = {X, Z} <- + qlc:keysort(1, [{a, 1}, {b, 4}, {c, 6}], []) ]), V2 = qlc:q([ P0 || - P0 = {W,Y} <- - qlc:keysort(2, [{2,a},{3,b},{4,c}], []) + P0 = {W, Y} <- + qlc:keysort(2, [{2, a}, {3, b}, {4, c}], []) ]), V3 = qlc:q([ - [G1|G2] || + [G1 | G2] || G1 <- V1, G2 <- V2, element(1, G1) == element(2, G2) ], - [{join,merge},{cache,list}]), + [{join, merge}, {cache, list}]), qlc:q([ - {A,X,Z,W} || - A <- [a,b,c], - [{X,Z}|{W,Y}] <- V3, + {A, X, Z, W} || + A <- [a, b, c], + [{X, Z} | {W, Y}] <- V3, X =:= Y ]) end</pre> @@ -1141,14 +1147,21 @@ ets:match_spec_run( gb_trees:lookup(K, gb_trees:from_orddict([])) of - {value,V} -> - [{K,V}]; + {value, V} -> + [{K, V}]; none -> [] end end, - [{1,a},{1,b},{1,c},{2,a},{2,b},{2,c}]), - ets:match_spec_compile([{{{'$1','$2'},'_'},[],['$1']}]))</pre> + [{1, a}, + {1, b}, + {1, c}, + {2, a}, + {2, b}, + {2, c}]), + ets:match_spec_compile([{{{'$1', '$2'}, '_'}, + [], + ['$1']}]))</pre> <p>Options:</p> <list type="bulleted"> <item> diff --git a/lib/stdlib/doc/src/queue.xml b/lib/stdlib/doc/src/queue.xml index 83a8afea81..89cce6d85b 100644 --- a/lib/stdlib/doc/src/queue.xml +++ b/lib/stdlib/doc/src/queue.xml @@ -168,7 +168,7 @@ <fsummary>Test if a queue is empty.</fsummary> <desc> <p>Tests if <c><anno>Q</anno></c> is empty and returns <c>true</c> if - so, otherwise otherwise.</p> + so, otherwise <c>false</c>.</p> </desc> </func> diff --git a/lib/stdlib/doc/src/rand.xml b/lib/stdlib/doc/src/rand.xml index 27d2d99f3c..b4737b48f8 100644 --- a/lib/stdlib/doc/src/rand.xml +++ b/lib/stdlib/doc/src/rand.xml @@ -38,25 +38,71 @@ <p> This module provides a pseudo random number generator. The module contains a number of algorithms. - The uniform distribution algorithms use the + The uniform distribution algorithms are based on the <url href="http://xorshift.di.unimi.it"> - xoroshiro116+ and xorshift1024* algorithms by Sebastiano Vigna. + Xoroshiro and Xorshift algorithms </url> + by Sebastiano Vigna. The normal distribution algorithm uses the <url href="http://www.jstatsoft.org/v05/i08"> Ziggurat Method by Marsaglia and Tsang </url> on top of the uniform distribution algorithm. </p> - <p>For some algorithms, jump functions are provided for generating - non-overlapping sequences for parallel computations. - The jump functions perform calculations - equivalent to perform a large number of repeated calls - for calculating new states. </p> + <p> + For most algorithms, jump functions are provided for generating + non-overlapping sequences for parallel computations. + The jump functions perform calculations + equivalent to perform a large number of repeated calls + for calculating new states. + </p> <p>The following algorithms are provided:</p> <taglist> + <tag><c>exsss</c></tag> + <item> + <p>Xorshift116**, 58 bits precision and period of 2^116-1</p> + <p>Jump function: equivalent to 2^64 calls</p> + <p> + This is the Xorshift116 generator combined with the StarStar scrambler + from the 2018 paper by David Blackman and Sebastiano Vigna: + <url href="http://vigna.di.unimi.it/ftp/papers/ScrambledLinear.pdf"> + Scrambled Linear Pseudorandom Number Generators + </url> + </p> + <p> + The generator does not need 58-bit rotates so it is faster + than the Xoroshiro116 generator, and when combined with + the StarStar scrambler it does not have any weak low bits + like <c>exrop</c> (Xoroshiro116+). + </p> + <p> + Alas, this combination is about 10% slower than <c>exrop</c>, + but is despite that the default algorithm thanks to its + statistical qualities. + </p> + </item> + <tag><c>exro928ss</c></tag> + <item> + <p>Xoroshiro928**, 58 bits precision and a period of 2^928-1</p> + <p>Jump function: equivalent to 2^512 calls</p> + <p> + This is a 58 bit version of Xoroshiro1024**, + from the 2018 paper by David Blackman and Sebastiano Vigna: + <url href="http://vigna.di.unimi.it/ftp/papers/ScrambledLinear.pdf"> + Scrambled Linear Pseudorandom Number Generators + </url> + that on a 64 bit Erlang system executes only about 40% slower than + the default <c>exsss</c> algorithm but with much longer period + and better statistical properties, and on the flip side + a larger state. + </p> + <p> + Many thanks to Sebastiano Vigna for his help with + the 58 bit adaption. + </p> + </item> <tag><c>exrop</c></tag> <item> <p>Xoroshiro116+, 58 bits precision and period of 2^116-1</p> @@ -83,7 +129,7 @@ </taglist> <p> - The default algorithm is <c>exrop</c> (Xoroshiro116+). + The default algorithm is <c>exsss</c> (Xorshift116**). If a specific algorithm is required, ensure to always use <seealso marker="#seed-1"> <c>seed/1</c></seealso> to initialize the state. @@ -154,19 +200,19 @@ R1 = rand:uniform(),</pre> <p>Use a specified algorithm:</p> <pre> -_ = rand:seed(exs1024s), +_ = rand:seed(exs928ss), R2 = rand:uniform(),</pre> <p>Use a specified algorithm with a constant seed:</p> <pre> -_ = rand:seed(exs1024s, {123, 123534, 345345}), +_ = rand:seed(exs928ss, {123, 123534, 345345}), R3 = rand:uniform(),</pre> <p>Use the functional API with a non-constant seed:</p> <pre> -S0 = rand:seed_s(exrop), +S0 = rand:seed_s(exsss), {R4, S1} = rand:uniform_s(S0),</pre> <p>Textbook basic form Box-Muller standard normal deviate</p> @@ -195,8 +241,9 @@ SND0 = math:sqrt(-2 * math:log(R5)) * math:cos(math:pi() * R6)</pre> </note> <p> - For all these generators the lowest bit(s) has got - a slightly less random behaviour than all other bits. + For all these generators except <c>exro928ss</c> and <c>exsss</c> + the lowest bit(s) has got a slightly less + random behaviour than all other bits. 1 bit for <c>exrop</c> (and <c>exsp</c>), and 3 bits for <c>exs1024s</c>. See for example the explanation in the @@ -211,7 +258,7 @@ up to (and included) 16TB, with the exception of binary rank tests, which fail due to the lowest bit being an LFSR; all other bits pass all tests. We suggest to use a sign test to extract a random Boolean value.</pre> <p> - If this is a problem; to generate a boolean + If this is a problem; to generate a boolean with these algorithms use something like this: </p> <pre>(rand:uniform(16) > 8)</pre> @@ -254,21 +301,50 @@ tests. We suggest to use a sign test to extract a random Boolean value.</pre> </desc> </datatype> <datatype> - <name name="exs64_state"/> - <desc><p>Algorithm specific internal state</p></desc> + <name name="seed"/> + <desc> + <p> + A seed value for the generator. + </p> + <p> + A list of integers sets the generator's internal state directly, + after algorithm-dependent checks of the value + and masking to the proper word size. + </p> + <p> + An integer is used as the initial state for a SplitMix64 generator. + The output values of that is then used for setting + the generator's internal state + after masking to the proper word size + and if needed avoiding zero values. + </p> + <p> + A traditional 3-tuple of integers seed is passed through + algorithm-dependent hashing functions to create + the generator's initial state. + </p> + </desc> </datatype> <datatype> <name name="exsplus_state"/> <desc><p>Algorithm specific internal state</p></desc> </datatype> <datatype> - <name name="exs1024_state"/> + <name name="exro928_state"/> <desc><p>Algorithm specific internal state</p></desc> </datatype> <datatype> <name name="exrop_state"/> <desc><p>Algorithm specific internal state</p></desc> </datatype> + <datatype> + <name name="exs1024_state"/> + <desc><p>Algorithm specific internal state</p></desc> + </datatype> + <datatype> + <name name="exs64_state"/> + <desc><p>Algorithm specific internal state</p></desc> + </datatype> </datatypes> <funcs> diff --git a/lib/stdlib/doc/src/slave.xml b/lib/stdlib/doc/src/slave.xml index 80fb28b548..f9e42ad47d 100644 --- a/lib/stdlib/doc/src/slave.xml +++ b/lib/stdlib/doc/src/slave.xml @@ -39,23 +39,30 @@ done through the master.</p> <p>Slave nodes on other hosts than the current one are started with - the <c>rsh</c> program. The user must be allowed to <c>rsh</c> to + the <c>ssh</c> program. The user must be allowed to <c>ssh</c> to the remote hosts without being prompted for a password. This can - be arranged in a number of ways (for details, see the <c>rsh</c> + be arranged in a number of ways (for details, see the <c>ssh</c> documentation). A slave node started on the same host as the master inherits certain environment values from the master, such as the current directory and the environment variables. For what can be assumed about the environment when a slave is started - on another host, see the documentation for the <c>rsh</c> + on another host, see the documentation for the <c>ssh</c> program.</p> - <p>An alternative to the <c>rsh</c> program can be specified on + <p>An alternative to the <c>ssh</c> program can be specified on the command line to - <seealso marker="erts:erl#erl"><c>erl(1)</c></seealso> as follows:</p> + <seealso marker="erts:erl"><c>erl(1)</c></seealso> as follows:</p> <pre> -rsh Program</pre> + <p>Note that the command specified with the <c>-rsh</c> flag is + treated as a file name which may contain spaces. It is thus not + possible to include any command line options. The remote node will + be launched as <c>"$RSH" "$REMOTE_HOSTNAME" erl -detached -noinput + ...</c>, so the + <c>erl</c> command must be found in the path on the remote host.</p> + <p>The slave node is to use the same file system at the master. At least, Erlang/OTP is to be installed in the same place on both computers and the same version of Erlang is to be used.</p> @@ -133,7 +140,7 @@ rpc:call(N, slave, pseudo, [node(), [pxw_server]]).</code> <p>Argument <c><anno>Args</anno></c> is used to set <c>erl</c> command-line arguments. If provided, it is passed to the new node and can be used for a variety of purposes; see - <seealso marker="erts:erl#erl"><c>erl(1)</c></seealso>.</p> + <seealso marker="erts:erl"><c>erl(1)</c></seealso>.</p> <p>As an example, suppose that you want to start a slave node at host <c>H</c> with node name <c>Name@H</c> and want the slave node to have the following properties:</p> @@ -166,7 +173,9 @@ slave:start(H, Name, Arg).</code> </item> <tag><c>no_rsh</c></tag> <item> - <p>There is no <c>rsh</c> program on the computer.</p> + <p>No remote shell program was found on the computer. Note + that <c>ssh</c> is used by default, but this can be overridden + with the <c>-rsh</c> flag.</p> </item> <tag><c>{already_running, <anno>Node</anno>}</c></tag> <item> diff --git a/lib/stdlib/doc/src/sys.xml b/lib/stdlib/doc/src/sys.xml index 6fc508b853..e22ca89ef5 100644 --- a/lib/stdlib/doc/src/sys.xml +++ b/lib/stdlib/doc/src/sys.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2018</year> + <year>1996</year><year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -114,6 +114,154 @@ </datatype> <datatype> <name name="system_event"/> + <desc> + <taglist> + <tag><c>{in,<anno>Msg</anno>}</c></tag> + <item> + <p> + Is produced by <c>gen_server</c> and <c>gen_event</c> + when the message <c>Msg</c> arrives. + </p> + </item> + <tag><c>{in,<anno>Msg</anno>,<anno>State</anno>}</c></tag> + <item> + <p> + Is produced by <c>gen_statem</c> + when the message <c>Msg</c> arrives in state <c>State</c>. + </p> + <p> + For <c>gen_statem</c> the <c><anno>Msg</anno></c> term is + an <c>{EventType,EventContent}</c> tuple. + </p> + </item> + <tag><c>{out,<anno>Msg</anno>,<anno>To</anno>}</c></tag> + <item> + <p> + Is produced by <c>gen_statem</c> when the reply <c>Msg</c> + is sent back to <c>To</c> by returning + a <c>{reply,To,Msg}</c> action from the callback module. + </p> + <p> + <c><anno>To</anno></c> is of the same type + as the first argument to <c>gen_statem:reply/2</c>. + </p> + </item> + <tag> + <c>{out,<anno>Msg</anno>,<anno>To</anno>,<anno>State</anno>}</c> + </tag> + <item> + <p> + Is produced by <c>gen_server</c> + when the reply <c><anno>Msg</anno></c> + is sent back to <c><anno>To</anno></c> + by returning a <c>{reply,...}</c> tuple + from the callback module. + </p> + <p> + <c><anno>To</anno></c> is of the same type + as the first argument to <c>gen_server:reply/2</c>. + </p> + <p> + <c><anno>State</anno></c> is the new server state. + </p> + </item> + <tag> + <c>{noreply,<anno>State</anno>}</c> + </tag> + <item> + <p> + Is produced by <c>gen_server</c> + when a <c>{noreply,...}</c> tuple is returned + from the callback module. + </p> + <p> + <c><anno>State</anno></c> is the new server state. + </p> + </item> + <tag> + <c>{continue,<anno>Continuation</anno>}</c> + </tag> + <item> + <p> + Is produced by <c>gen_server</c> + when a <c>{continue,<anno>Continuation</anno>}</c> + tuple is returned from the callback module. + </p> + </item> + <tag> + <c>{code_change,<anno>Event</anno>,<anno>State</anno>}</c> + </tag> + <item> + <p> + Is produced by <c>gen_statem</c> + when the message <c><anno>Event</anno></c> + arrives in state <c><anno>State</anno></c> + as the first event after a code change. + </p> + <p> + <c><anno>Event</anno></c> is + an <c>{EventType,EventContent}</c> tuple. + </p> + </item> + <tag> + <c> + {postpone,<anno>Event</anno>,<anno>State</anno>,<anno>NextState</anno>} + </c> + </tag> + <item> + <p> + Is produced by <c>gen_statem</c> + when the message <c><anno>Event</anno></c> + is postponed in state <c><anno>State</anno></c>. + <c><anno>NextState</anno></c> is the new state. + </p> + <p> + <c><anno>Event</anno></c> is + an <c>{EventType,EventContent}</c> tuple. + </p> + </item> + <tag> + <c> + {consume,<anno>Event</anno>,<anno>State</anno>,<anno>NextState</anno>} + </c> + </tag> + <item> + <p> + Is produced by <c>gen_statem</c> + when the message <c><anno>Event</anno></c> + is consumed in state <c><anno>State</anno></c>. + <c><anno>NextState</anno></c> is the new state. + </p> + <p> + <c><anno>Event</anno></c> is + an <c>{EventType,EventContent}</c> tuple. + </p> + </item> + <tag> + <c> + {enter,<anno>State</anno>} + </c> + </tag> + <item> + <p> + Is produced by <c>gen_statem</c> + when the first state <c><anno>State</anno></c> is entered. + </p> + </item> + <tag> + <c> + {terminate,<anno>Reason</anno>,<anno>State</anno>} + </c> + </tag> + <item> + <p> + Is produced by <c>gen_statem</c> + when it terminates with reason <c><anno>Reason</anno></c> + in state <c><anno>State</anno></c>. + </p> + </item> + </taglist> + </desc> </datatype> <datatype> <name name="dbg_opt"/> @@ -123,6 +271,9 @@ <name name="dbg_fun"/> </datatype> <datatype> + <name name="debug_option"/> + </datatype> + <datatype> <name name="format_fun"/> </datatype> </datatypes> @@ -532,6 +683,12 @@ <name name="get_debug" arity="3" since=""/> <fsummary>Get the data associated with a debug option.</fsummary> <desc> + <warning> + <p> + <c>get_debug/3</c> is deprecated since it returns + data of an internal type only useful for debugging. + </p> + </warning> <p>Gets the data associated with a debug option. <c><anno>Default</anno></c> is returned if <c><anno>Item</anno></c> is not found. Can be @@ -605,6 +762,18 @@ </func> <func> + <name name="get_log" arity="1" since="OTP-22.0"/> + <fsummary>Return the logged events in the debug structure.</fsummary> + <desc> + <p> + Returns the logged system events in the debug structure, + that is the last argument to + <seealso marker="#handle_debug/4"><c>handle_debug/4</c></seealso>. + </p> + </desc> + </func> + + <func> <name since="">Module:system_code_change(Misc, Module, OldVsn, Extra) -> {ok, NMisc}</name> <fsummary>Called when the process is to perform a code change.</fsummary> diff --git a/lib/stdlib/include/assert.hrl b/lib/stdlib/include/assert.hrl index 2ec89e7d8a..28d25c6589 100644 --- a/lib/stdlib/include/assert.hrl +++ b/lib/stdlib/include/assert.hrl @@ -140,7 +140,7 @@ -endif. %% This is mostly a convenience which gives more detailed reports. -%% Note: Guard is a guarded pattern, and can not be used for value. +%% Note: Guard is a guarded pattern, and cannot be used for value. -ifdef(NOASSERT). -define(assertMatch(Guard, Expr), ok). -define(assertMatch(Guard, Expr, Comment), ok). @@ -289,7 +289,7 @@ end). -endif. -%% Note: Class and Term are patterns, and can not be used for value. +%% Note: Class and Term are patterns, and cannot be used for value. %% Term can be a guarded pattern, but Class cannot. -ifdef(NOASSERT). -define(assertException(Class, Term, Expr), ok). @@ -364,7 +364,7 @@ ?assertException(throw, Term, Expr, Comment)). %% This is the inverse case of assertException, for convenience. -%% Note: Class and Term are patterns, and can not be used for value. +%% Note: Class and Term are patterns, and cannot be used for value. %% Both Class and Term can be guarded patterns. -ifdef(NOASSERT). -define(assertNotException(Class, Term, Expr), ok). diff --git a/lib/stdlib/src/Makefile b/lib/stdlib/src/Makefile index c95f7637f7..86003c953d 100644 --- a/lib/stdlib/src/Makefile +++ b/lib/stdlib/src/Makefile @@ -155,8 +155,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif -ERL_COMPILE_FLAGS += -I../include -I../../kernel/include -Werror +ERL_COMPILE_FLAGS += -I../include -I../../kernel/include # ---------------------------------------------------- # Targets diff --git a/lib/stdlib/src/array.erl b/lib/stdlib/src/array.erl index 939b1fb488..1504326c61 100644 --- a/lib/stdlib/src/array.erl +++ b/lib/stdlib/src/array.erl @@ -126,11 +126,12 @@ %% per write than base 10, but the speedup is only 21%.) -define(DEFAULT, undefined). --define(LEAFSIZE, 10). % the "base" --define(NODESIZE, ?LEAFSIZE). % (no reason to have a different size) +-define(LEAFSIZE, 10). % the "base" (assumed to be > 1) +-define(NODESIZE, ?LEAFSIZE). % must not be LEAFSIZE-1; keep same as leaf -define(NODEPATTERN(S), {_,_,_,_,_,_,_,_,_,_,S}). % NODESIZE+1 elements! --define(NEW_NODE(S), % beware of argument duplication! - setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(S)),(S))). +-define(NEW_NODE(E,S), % general case (currently unused) + setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(E)),(S))). +-define(NEW_NODE(S), erlang:make_tuple((?NODESIZE+1),(S))). % when E = S -define(NEW_LEAF(D), erlang:make_tuple(?LEAFSIZE,(D))). -define(NODELEAFS, ?NODESIZE*?LEAFSIZE). @@ -605,7 +606,7 @@ grow(I, E, M) -> grow_1(I, E, M). grow_1(I, E, M) when I >= M -> - grow(I, setelement(1, ?NEW_NODE(M), E), ?extend(M)); + grow_1(I, setelement(1, ?NEW_NODE(M), E), ?extend(M)); grow_1(_I, E, M) -> {E, M}. @@ -1631,12 +1632,11 @@ foldl_test_() -> ?_assert(foldl(Sum, 0, from_list(lists:seq(0,10))) =:= 55), ?_assert(foldl(Reverse, [], from_list(lists:seq(0,1000))) =:= lists:reverse(lists:seq(0,1000))), - ?_assert({999,[N0*100+1+2,N0*2+1+1,0]} =:= - foldl(Vals, {0,[]}, + ?_assertEqual({N0*100+1-2,[N0*100+1+2,N0*2+1+1,0]}, + foldl(Vals, {0,[]}, set(N0*100+1,2, set(N0*2+1,1, set(0,0,new()))))) - ]. -endif. @@ -1786,12 +1786,11 @@ foldr_test_() -> ?_assert(foldr(Sum, 0, from_list(lists:seq(0,10))) =:= 55), ?_assert(foldr(List, [], from_list(lists:seq(0,1000))) =:= lists:seq(0,1000)), - ?_assert({999,[0,N0*2+1+1,N0*100+1+2]} =:= - foldr(Vals, {0,[]}, + ?_assertEqual({N0*100+1-2,[0,N0*2+1+1,N0*100+1+2]}, + foldr(Vals, {0,[]}, set(N0*100+1,2, set(N0*2+1,1, set(0,0,new()))))) - ]. -endif. diff --git a/lib/stdlib/src/beam_lib.erl b/lib/stdlib/src/beam_lib.erl index 3386cfcbe6..aa992f17ab 100644 --- a/lib/stdlib/src/beam_lib.erl +++ b/lib/stdlib/src/beam_lib.erl @@ -32,8 +32,12 @@ all_chunks/1, diff_dirs/2, strip/1, + strip/2, strip_files/1, + strip_files/2, strip_release/1, + strip_release/2, + significant_chunks/0, build_module/1, version/1, md5/1, @@ -188,7 +192,16 @@ diff_dirs(Dir1, Dir2) -> Beam2 :: beam(). strip(FileName) -> - try strip_file(FileName) + strip(FileName, []). + +-spec strip(Beam1, AdditionalChunks) -> + {'ok', {module(), Beam2}} | {'error', 'beam_lib', info_rsn()} when + Beam1 :: beam(), + AdditionalChunks :: [chunkid()], + Beam2 :: beam(). + +strip(FileName, AdditionalChunks) -> + try strip_file(FileName, AdditionalChunks) catch Error -> Error end. -spec strip_files(Files) -> @@ -196,8 +209,17 @@ strip(FileName) -> Files :: [beam()], Beam :: beam(). -strip_files(Files) when is_list(Files) -> - try strip_fils(Files) +strip_files(Files) -> + strip_files(Files, []). + +-spec strip_files(Files, AdditionalChunks) -> + {'ok', [{module(), Beam}]} | {'error', 'beam_lib', info_rsn()} when + Files :: [beam()], + AdditionalChunks :: [chunkid()], + Beam :: beam(). + +strip_files(Files, AdditionalChunks) when is_list(Files) -> + try strip_fils(Files, AdditionalChunks) catch Error -> Error end. -spec strip_release(Dir) -> @@ -207,7 +229,17 @@ strip_files(Files) when is_list(Files) -> Reason :: {'not_a_directory', term()} | info_rsn(). strip_release(Root) -> - catch strip_rel(Root). + strip_release(Root, []). + +-spec strip_release(Dir, AdditionalChunks) -> + {'ok', [{module(), file:filename()}]} + | {'error', 'beam_lib', Reason} when + Dir :: atom() | file:filename(), + AdditionalChunks :: [chunkid()], + Reason :: {'not_a_directory', term()} | info_rsn(). + +strip_release(Root, AdditionalChunks) -> + catch strip_rel(Root, AdditionalChunks). -spec version(Beam) -> {'ok', {module(), [Version :: term()]}} | @@ -401,17 +433,17 @@ cmp_lists([{Id, C1} | R1], [{Id, C2} | R2]) -> cmp_lists(_, _) -> error(different_chunks). -strip_rel(Root) -> +strip_rel(Root, AdditionalChunks) -> ok = assert_directory(Root), - strip_fils(filelib:wildcard(filename:join(Root, "lib/*/ebin/*.beam"))). + strip_fils(filelib:wildcard(filename:join(Root, "lib/*/ebin/*.beam")), AdditionalChunks). %% -> {ok, [{Mod, BinaryOrFileName}]} | throw(Error) -strip_fils(Files) -> - {ok, [begin {ok, Reply} = strip_file(F), Reply end || F <- Files]}. +strip_fils(Files, AdditionalChunks) -> + {ok, [begin {ok, Reply} = strip_file(F, AdditionalChunks), Reply end || F <- Files]}. %% -> {ok, {Mod, FileName}} | {ok, {Mod, binary()}} | throw(Error) -strip_file(File) -> - {ok, {Mod, Chunks}} = read_significant_chunks(File, significant_chunks()), +strip_file(File, AdditionalChunks) -> + {ok, {Mod, Chunks}} = read_significant_chunks(File, AdditionalChunks ++ significant_chunks()), {ok, Stripped0} = build_module(Chunks), Stripped = compress(Stripped0), case File of diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index e0cd68617b..e0c37ca030 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -79,6 +79,8 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> -type fa() :: {atom(), arity()}. % function+arity -type ta() :: {atom(), arity()}. % type+arity +-type module_or_mfa() :: module() | mfa(). + -record(typeinfo, {attr, line}). %% Usage of records, functions, and imports. The variable table, which @@ -115,6 +117,8 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> :: erl_anno:anno(), clashes=[], %Exported functions named as BIFs not_deprecated=[], %Not considered deprecated + not_removed=gb_sets:empty() %Not considered removed + :: gb_sets:set(module_or_mfa()), func=[], %Current function warn_format=0, %Warn format calls enabled_warnings=[], %All enabled warnings (ordset). @@ -573,7 +577,10 @@ start(File, Opts) -> false, Opts)}, {missing_spec_all, bool_option(warn_missing_spec_all, nowarn_missing_spec_all, - false, Opts)} + false, Opts)}, + {removed, + bool_option(warn_removed, nowarn_removed, + true, Opts)} ], Enabled1 = [Category || {Category,true} <- Enabled0], Enabled = ordsets:from_list(Enabled1), @@ -670,8 +677,9 @@ forms(Forms0, St0) -> no_auto = AutoImportSuppressed}), St2 = bif_clashes(Forms, St1), St3 = not_deprecated(Forms, St2), - St4 = foldl(fun form/2, pre_scan(Forms, St3), Forms), - post_traversal_check(Forms, St4). + St4 = not_removed(Forms, St3), + St5 = foldl(fun form/2, pre_scan(Forms, St4), Forms), + post_traversal_check(Forms, St5). pre_scan([{attribute,L,compile,C} | Fs], St) -> case is_warn_enabled(export_all, St) andalso @@ -831,19 +839,39 @@ bif_clashes(Forms, #lint{nowarn_bif_clash=Nowarn} = St) -> %% not_deprecated(Forms, State0) -> State -not_deprecated(Forms, St0) -> +not_deprecated(Forms, #lint{compile=Opts}=St0) -> %% There are no line numbers in St0#lint.compile. MFAsL = [{MFA,L} || {attribute, L, compile, Args} <- Forms, {nowarn_deprecated_function, MFAs0} <- lists:flatten([Args]), MFA <- lists:flatten([MFAs0])], - Nowarn = [MFA || {MFA,_L} <- MFAsL], + Nowarn = [MFA || + {nowarn_deprecated_function, MFAs0} <- Opts, + MFA <- lists:flatten([MFAs0])], ML = [{M,L} || {{M,_F,_A},L} <- MFAsL, is_atom(M)], St1 = foldl(fun ({M,L}, St2) -> check_module_name(M, L, St2) end, St0, ML), St1#lint{not_deprecated = ordsets:from_list(Nowarn)}. +%% not_removed(Forms, State0) -> State + +not_removed(Forms, #lint{compile=Opts}=St0) -> + %% There are no line numbers in St0#lint.compile. + MFAsL = [{MFA,L} || + {attribute, L, compile, Args} <- Forms, + {nowarn_removed, MFAs0} <- lists:flatten([Args]), + MFA <- lists:flatten([MFAs0])], + Nowarn = [MFA || + {nowarn_removed, MFAs0} <- Opts, + MFA <- lists:flatten([MFAs0])], + St1 = foldl(fun ({{M, _F, _A}, L}, St2) -> + check_module_name(M, L, St2); + ({M,L}, St2) -> + check_module_name(M, L, St2) + end, St0, MFAsL), + St1#lint{not_removed = gb_sets:from_list(Nowarn)}. + %% The nowarn_bif_clash directive is not only deprecated, it's actually an error from R14A disallowed_compile_flags(Forms, St0) -> %% There are (still) no line numbers in St0#lint.compile. @@ -2248,6 +2276,9 @@ expr({'fun',Line,Body}, Vt, St) -> case Body of {clauses,Cs} -> fun_clauses(Cs, Vt, St); + {function,record_info,2} -> + %% It is illegal to call record_info/2 with unknown arguments. + {[],add_error(Line, illegal_record_info, St)}; {function,F,A} -> %% BifClash - Fun expression %% N.B. Only allows BIFs here as well, NO IMPORTS!! @@ -3767,13 +3798,23 @@ deprecated_function(Line, M, F, As, St) -> add_warning(Line, {deprecated, MFA, Replacement, Rel}, St) end; {removed, String} when is_list(String) -> - add_warning(Line, {removed, MFA, String}, St); + add_removed_warning(Line, MFA, {removed, MFA, String}, St); {removed, Replacement, Rel} -> - add_warning(Line, {removed, MFA, Replacement, Rel}, St); + add_removed_warning(Line, MFA, {removed, MFA, Replacement, Rel}, St); no -> St end. +add_removed_warning(Line, {M, _, _}=MFA, Warning, #lint{not_removed=NotRemoved}=St) -> + case is_warn_enabled(removed, St) andalso + not gb_sets:is_element(M, NotRemoved) andalso + not gb_sets:is_element(MFA, NotRemoved) of + true -> + add_warning(Line, Warning, St); + false -> + St + end. + -dialyzer({no_match, deprecated_type/5}). deprecated_type(L, M, N, As, St) -> diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index 5fa9c4f75c..4ad94f2507 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -841,7 +841,7 @@ Erlang code. -type af_record_field(T) :: {'record_field', anno(), af_field_name(), T}. -type af_map_pattern() :: - {'map', anno(), [af_assoc_exact(abstract_expr)]}. + {'map', anno(), [af_assoc_exact(abstract_expr())]}. -type abstract_type() :: af_annotated_type() | af_atom() diff --git a/lib/stdlib/src/erl_posix_msg.erl b/lib/stdlib/src/erl_posix_msg.erl index 8959fea498..b9ed4a3a9d 100644 --- a/lib/stdlib/src/erl_posix_msg.erl +++ b/lib/stdlib/src/erl_posix_msg.erl @@ -81,9 +81,9 @@ message_1(el2hlt) -> <<"level 2 halted">>; message_1(el2nsync) -> <<"level 2 not synchronized">>; message_1(el3hlt) -> <<"level 3 halted">>; message_1(el3rst) -> <<"level 3 reset">>; -message_1(elibacc) -> <<"can not access a needed shared library">>; +message_1(elibacc) -> <<"cannot access a needed shared library">>; message_1(elibbad) -> <<"accessing a corrupted shared library">>; -message_1(elibexec) -> <<"can not exec a shared library directly">>; +message_1(elibexec) -> <<"cannot exec a shared library directly">>; message_1(elibmax) -> <<"attempting to link in more shared libraries than system limit">>; message_1(elibscn) -> <<".lib section in a.out corrupted">>; diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl index ada3ff5de3..255c0ae81f 100644 --- a/lib/stdlib/src/erl_pp.erl +++ b/lib/stdlib/src/erl_pp.erl @@ -26,7 +26,7 @@ attribute/1,attribute/2,function/1,function/2, guard/1,guard/2,exprs/1,exprs/2,exprs/3,expr/1,expr/2,expr/3,expr/4]). --import(lists, [append/1,foldr/3,mapfoldl/3,reverse/1,reverse/2]). +-import(lists, [append/1,foldr/3,map/2,mapfoldl/3,reverse/1,reverse/2]). -import(io_lib, [write/1,format/2]). -import(erl_parse, [inop_prec/1,preop_prec/1,func_prec/0,max_prec/0, type_inop_prec/1, type_preop_prec/1]). @@ -41,10 +41,11 @@ io_lib:chars())). -type(option() :: {hook, hook_function()} - | {encoding, latin1 | unicode | utf8}). + | {encoding, latin1 | unicode | utf8} + | {quote_singleton_atom_types, boolean()}). -type(options() :: hook_function() | [option()]). --record(pp, {value_fun, string_fun, char_fun}). +-record(pp, {value_fun, singleton_atom_type_fun, string_fun, char_fun}). -record(options, {hook, encoding, opts}). @@ -206,22 +207,43 @@ options(Hook) -> #options{hook = Hook, encoding = encoding([]), opts = Hook}. state(Options) when is_list(Options) -> + Quote = proplists:get_bool(quote_singleton_atom_types, Options), case encoding(Options) of - latin1 -> state(); - unicode -> unicode_state() + latin1 -> latin1_state(Quote); + unicode -> unicode_state(Quote) end; state(_Hook) -> - state(). + latin1_state(false). -state() -> +latin1_state(Quote) -> Options = [{encoding,latin1}], - #pp{value_fun = fun(V) -> io_lib_pretty:print(V, Options) end, + ValueFun = fun(V) -> io_lib_pretty:print(V, Options) end, + SingletonFun = + case Quote of + true -> + fun(A) -> + io_lib:write_string_as_latin1(atom_to_list(A), $') + end; %' + false -> + ValueFun + end, + #pp{value_fun = ValueFun, + singleton_atom_type_fun = SingletonFun, string_fun = fun io_lib:write_string_as_latin1/1, char_fun = fun io_lib:write_char_as_latin1/1}. -unicode_state() -> +unicode_state(Quote) -> Options = [{encoding,unicode}], - #pp{value_fun = fun(V) -> io_lib_pretty:print(V, Options) end, + ValueFun = fun(V) -> io_lib_pretty:print(V, Options) end, + SingletonFun = + case Quote of + true -> + fun(A) -> io_lib:write_string(atom_to_list(A), $') end; %' + false -> + ValueFun + end, + #pp{value_fun = ValueFun, + singleton_atom_type_fun = SingletonFun, string_fun = fun io_lib:write_string/1, char_fun = fun io_lib:write_char/1}. @@ -350,7 +372,7 @@ ltype({user_type,Line,T,Ts}, _) -> ltype({remote_type,Line,[M,F,Ts]}, _) -> simple_type({remote,Line,M,F}, Ts); ltype({atom,_,T}, _) -> - {atom,T}; + {singleton_atom_type,T}; ltype(E, P) -> lexpr(E, P, options(none)). @@ -360,7 +382,12 @@ binary_type(I1, I2) -> P = max_prec(), E1 = [[leaf("_:"),lexpr(I1, P, options(none))] || B], E2 = [[leaf("_:_*"),lexpr(I2, P, options(none))] || U], - {seq,'<<','>>',[$,],E1++E2}. + case E1++E2 of + [] -> + leaf("<<>>"); + Es -> + {seq,'<<','>>',[$,],Es} + end. map_type(Fs) -> {first,[$#],map_pair_types(Fs)}. @@ -386,6 +413,8 @@ typed(B, Type) -> {_L,_P,R} = type_inop_prec('::'), {list,[{cstep,[B,' ::'],ltype(Type, R)}]}. +tuple_type([], _) -> + leaf("{}"); tuple_type(Ts, F) -> {seq,${,$},[$,],ltypes(Ts, F, 0)}. @@ -454,7 +483,7 @@ pname(A) when is_atom(A) -> write(A). falist([]) -> - [leaf("[]")]; + ['[]']; falist(Falist) -> L = [begin {Name,Arity} = Fa, @@ -562,22 +591,22 @@ lexpr({map, _, Map, Fs}, Prec, Opts) -> El = {first,[Rl,$#],map_fields(Fs, Opts)}, maybe_paren(P, Prec, El); lexpr({block,_,Es}, _, Opts) -> - {list,[{step,'begin',body(Es, Opts)},'end']}; + {list,[{step,'begin',body(Es, Opts)},{reserved,'end'}]}; lexpr({'if',_,Cs}, _, Opts) -> - {list,[{step,'if',if_clauses(Cs, Opts)},'end']}; + {list,[{step,'if',if_clauses(Cs, Opts)},{reserved,'end'}]}; lexpr({'case',_,Expr,Cs}, _, Opts) -> - {list,[{step,{list,[{step,'case',lexpr(Expr, Opts)},'of']}, + {list,[{step,{list,[{step,'case',lexpr(Expr, Opts)},{reserved,'of'}]}, cr_clauses(Cs, Opts)}, - 'end']}; + {reserved,'end'}]}; lexpr({'cond',_,Cs}, _, Opts) -> - {list,[{step,leaf("cond"),cond_clauses(Cs, Opts)},'end']}; + {list,[{step,leaf("cond"),cond_clauses(Cs, Opts)},{reserved,'end'}]}; lexpr({'receive',_,Cs}, _, Opts) -> - {list,[{step,'receive',cr_clauses(Cs, Opts)},'end']}; + {list,[{step,'receive',cr_clauses(Cs, Opts)},{reserved,'end'}]}; lexpr({'receive',_,Cs,To,ToOpt}, _, Opts) -> Al = {list,[{step,[lexpr(To, Opts),' ->'],body(ToOpt, Opts)}]}, {list,[{step,'receive',cr_clauses(Cs, Opts)}, {step,'after',Al}, - 'end']}; + {reserved,'end'}]}; lexpr({'fun',_,{function,F,A}}, _Prec, _Opts) -> [leaf("fun "),{atom,F},leaf(format("/~w", [A]))]; lexpr({'fun',L,{function,_,_}=Func,Extra}, Prec, Opts) -> @@ -596,15 +625,17 @@ lexpr({'fun',_,{function,M,F,A}}, _Prec, Opts) -> ArityItem = lexpr(A, Opts), ["fun ",NameItem,$:,CallItem,$/,ArityItem]; lexpr({'fun',_,{clauses,Cs}}, _Prec, Opts) -> - {list,[{first,'fun',fun_clauses(Cs, Opts, unnamed)},'end']}; + {list,[{first,'fun',fun_clauses(Cs, Opts, unnamed)},{reserved,'end'}]}; lexpr({named_fun,_,Name,Cs}, _Prec, Opts) -> - {list,[{first,['fun', " "],fun_clauses(Cs, Opts, {named, Name})},'end']}; + {list,[{first,['fun', " "],fun_clauses(Cs, Opts, {named, Name})}, + {reserved,'end'}]}; lexpr({'fun',_,{clauses,Cs},Extra}, _Prec, Opts) -> {force_nl,fun_info(Extra), - {list,[{first,'fun',fun_clauses(Cs, Opts, unnamed)},'end']}}; + {list,[{first,'fun',fun_clauses(Cs, Opts, unnamed)},{reserved,'end'}]}}; lexpr({named_fun,_,Name,Cs,Extra}, _Prec, Opts) -> {force_nl,fun_info(Extra), - {list,[{first,['fun', " "],fun_clauses(Cs, Opts, {named, Name})},'end']}}; + {list,[{first,['fun', " "],fun_clauses(Cs, Opts, {named, Name})}, + {reserved,'end'}]}}; lexpr({call,_,{remote,_,{atom,_,M},{atom,_,F}=N}=Name,Args}, Prec, Opts) -> case erl_internal:bif(M, F, length(Args)) of true -> @@ -619,7 +650,7 @@ lexpr({'try',_,Es,Scs,Ccs,As}, _, Opts) -> Scs =:= [] -> {step,'try',body(Es, Opts)}; true -> - {step,{list,[{step,'try',body(Es, Opts)},'of']}, + {step,{list,[{step,'try',body(Es, Opts)},{reserved,'of'}]}, cr_clauses(Scs, Opts)} end, if @@ -634,7 +665,7 @@ lexpr({'try',_,Es,Scs,Ccs,As}, _, Opts) -> true -> {step,'after',body(As, Opts)} end, - 'end']}; + {reserved,'end'}]}; lexpr({'catch',_,Expr}, Prec, Opts) -> {P,R} = preop_prec('catch'), El = {list,[{step,'catch',lexpr(Expr, R, Opts)}]}, @@ -647,7 +678,7 @@ lexpr({match,_,Lhs,Rhs}, Prec, Opts) -> maybe_paren(P, Prec, El); lexpr({op,_,Op,Arg}, Prec, Opts) -> {P,R} = preop_prec(Op), - Ol = leaf(format("~s ", [Op])), + Ol = {reserved, leaf(format("~s ", [Op]))}, El = [Ol,lexpr(Arg, R, Opts)], maybe_paren(P, Prec, El); lexpr({op,_,Op,Larg,Rarg}, Prec, Opts) when Op =:= 'orelse'; @@ -655,14 +686,14 @@ lexpr({op,_,Op,Larg,Rarg}, Prec, Opts) when Op =:= 'orelse'; %% Breaks lines since R12B. {L,P,R} = inop_prec(Op), Ll = lexpr(Larg, L, Opts), - Ol = leaf(format("~s", [Op])), + Ol = {reserved, leaf(format("~s", [Op]))}, Lr = lexpr(Rarg, R, Opts), El = {prefer_nl,[[]],[Ll,Ol,Lr]}, maybe_paren(P, Prec, El); lexpr({op,_,Op,Larg,Rarg}, Prec, Opts) -> {L,P,R} = inop_prec(Op), Ll = lexpr(Larg, L, Opts), - Ol = leaf(format("~s", [Op])), + Ol = {reserved, leaf(format("~s", [Op]))}, Lr = lexpr(Rarg, R, Opts), El = {list,[Ll,Ol,Lr]}, maybe_paren(P, Prec, El); @@ -882,16 +913,18 @@ lc_qual(Q, Opts) -> lexpr(Q, 0, Opts). proper_list(Es, Opts) -> - {seq,$[,$],$,,lexprs(Es, Opts)}. + {seq,$[,$],[$,],lexprs(Es, Opts)}. improper_list(Es, Opts) -> - {seq,$[,$],{$,,$|},lexprs(Es, Opts)}. + {seq,$[,$],[{$,,' |'}],lexprs(Es, Opts)}. tuple(L, Opts) -> tuple(L, fun lexpr/2, Opts). +tuple([], _F, _Opts) -> + leaf("{}"); tuple(Es, F, Opts) -> - {seq,${,$},$,,lexprs(Es, F, Opts)}. + {seq,${,$},[$,],lexprs(Es, F, Opts)}. args(As, Opts) -> {seq,$(,$),[$,],lexprs(As, Opts)}. @@ -939,6 +972,7 @@ frmt(Item, I, PP) -> %%% - {prefer_nl,Sep,IPs}: forces linebreak between Is unlesss negative %%% indentation. %%% - {atom,A}: an atom +%%% - {singleton_atom_type,A}: an singleton atom type %%% - {char,C}: a character %%% - {string,S}: a string. %%% - {value,T}: a term. @@ -983,8 +1017,10 @@ f({seq,Before,After,Sep,LItems}, I0, ST, WT, PP) -> end, {BCharsL++Chars,Size}; no -> - {BCharsL++insert_newlines(CharsSizeL, I, ST), - nsz(lists:last(Sizes), I0)} + CharsList = handle_step(CharsSizeL, I, ST), + {LChars, LSize} = + maybe_newlines(CharsList, LItems, I, NSepChars, ST), + {[BCharsL,LChars],nsz(LSize, I0)} end; f({force_nl,_ExtraInfoItem,Item}, I, ST, WT, PP) when I < 0 -> %% Extra info is a comment; cannot have that on the same line @@ -1000,23 +1036,28 @@ f({prefer_nl,Sep,LItems}, I0, ST, WT, PP) -> Sizes =:= [] -> {[], 0}; true -> - {insert_newlines(CharsSize2L, I0, ST),nsz(lists:last(Sizes), I0)} + {insert_newlines(CharsSize2L, I0, ST), + nsz(lists:last(Sizes), I0)} end; f({value,V}, I, ST, WT, PP) -> f(write_a_value(V, PP), I, ST, WT, PP); f({atom,A}, I, ST, WT, PP) -> f(write_an_atom(A, PP), I, ST, WT, PP); +f({singleton_atom_type,A}, I, ST, WT, PP) -> + f(write_a_singleton_atom_type(A, PP), I, ST, WT, PP); f({char,C}, I, ST, WT, PP) -> f(write_a_char(C, PP), I, ST, WT, PP); f({string,S}, I, ST, WT, PP) -> f(write_a_string(S, I, PP), I, ST, WT, PP); +f({reserved,R}, I, ST, WT, PP) -> + f(R, I, ST, WT, PP); f({hook,HookExpr,Precedence,Func,Options}, I, _ST, _WT, _PP) -> Chars = Func(HookExpr, I, Precedence, Options), {Chars,indentation(Chars, I)}; f({ehook,HookExpr,Precedence,{Mod,Func,Eas}=ModFuncEas}, I, _ST, _WT, _PP) -> Chars = apply(Mod, Func, [HookExpr,I,Precedence,ModFuncEas|Eas]), {Chars,indentation(Chars, I)}; -f(WordName, _I, _ST, WT, _PP) -> % when is_atom(WordName) +f(WordName, _I, _ST, WT, _PP) when is_atom(WordName) -> word(WordName, WT). -define(IND, 4). @@ -1038,12 +1079,18 @@ fl(CItems, Sep0, I0, After, ST, WT, PP) -> true -> [CharSize1,f([Item2,S], incr(I0, ?IND), ST, WT, PP)] end; + ({reserved,Word}, S) -> + [f([Word,S], I0, ST, WT, PP),{[],0}]; (Item, S) -> [f([Item,S], I0, ST, WT, PP),{[],0}] end, - {Sep,LastSep} = case Sep0 of {_,_} -> Sep0; _ -> {Sep0,Sep0} end, + {Sep,LastSep} = sep(Sep0), fl1(CItems, F, Sep, LastSep, After). +sep([{S,LS}]) -> {[S],[LS]}; +sep({_,_}=Sep) -> Sep; +sep(S) -> {S, S}. + fl1([CItem], F, _Sep, _LastSep, After) -> [F(CItem,After)]; fl1([CItem1,CItem2], F, _Sep, LastSep, After) -> @@ -1069,20 +1116,64 @@ unz1(CharSizes) -> nonzero(CharSizes) -> lists:filter(fun({_,Sz}) -> Sz =/= 0 end, CharSizes). -insert_newlines(CharsSizesL, I, ST) when I >= 0 -> - insert_nl(foldr(fun([{_C1,0},{_C2,0}], A) -> - A; - ([{C1,_Sz1},{_C2,0}], A) -> - [C1|A]; - ([{C1,_Sz1},{C2,Sz2}], A) when Sz2 > 0 -> - [insert_nl([C1,C2], I+?IND, ST)|A] - end, [], CharsSizesL), I, ST). +maybe_newlines([{Chars,Size}], [], _I, _NSepChars, _ST) -> + {Chars,Size}; +maybe_newlines(CharsSizeList, Items, I, NSepChars, ST) when I >= 0 -> + maybe_sep(CharsSizeList, Items, I, NSepChars, nl_indent(I, ST)). + +maybe_sep([{Chars1,Size1}|CharsSizeL], [Item|Items], I0, NSepChars, Sep) -> + I1 = case classify_item(Item) of + atomic -> + I0 + Size1; + _ -> + ?MAXLINE+1 + end, + maybe_sep1(CharsSizeL, Items, I0, I1, Sep, NSepChars, Size1, [Chars1]). + +maybe_sep1([{Chars,Size}|CharsSizeL], [Item|Items], + I0, I, Sep, NSepChars, Sz0, A) -> + case classify_item(Item) of + atomic when is_integer(Size) -> + Size1 = Size + 1, + I1 = I + Size1, + if + I1 =< ?MAXLINE -> + A1 = if + NSepChars > 0 -> [Chars,$\s|A]; + true -> [Chars|A] + end, + maybe_sep1(CharsSizeL, Items, I0, I1, Sep, NSepChars, + Sz0 + Size1, A1); + true -> + A1 = [Chars,Sep|A], + maybe_sep1(CharsSizeL, Items, I0, I0 + Size, Sep, + NSepChars, Size1, A1) + end; + _ -> + A1 = [Chars,Sep|A], + maybe_sep1(CharsSizeL, Items, I0, ?MAXLINE+1, Sep, NSepChars, + 0, A1) + end; +maybe_sep1(_CharsSizeL, _Items, _Io, _I, _Sep, _NSepChars, Sz, A) -> + {lists:reverse(A), Sz}. +insert_newlines(CharsSizesL, I, ST) when I >= 0 -> + {CharsL, _} = unz1(handle_step(CharsSizesL, I, ST)), + insert_nl(CharsL, I, ST). + +handle_step(CharsSizesL, I, ST) -> + map(fun([{_C1,0},{_C2,0}]) -> + {[], 0}; + ([{C1,Sz1},{_C2,0}]) -> + {C1, Sz1}; + ([{C1,Sz1},{C2,Sz2}]) when Sz2 > 0 -> + {insert_nl([C1,C2], I+?IND, ST),line_size([Sz1,Sz2])} + end, CharsSizesL). insert_nl(CharsL, I, ST) -> insert_sep(CharsL, nl_indent(I, ST)). -insert_sep([Chars1 | CharsL], Sep) -> +insert_sep([Chars1|CharsL], Sep) -> [Chars1 | [[Sep,Chars] || Chars <- CharsL]]. nl_indent(0, _T) -> @@ -1090,6 +1181,12 @@ nl_indent(0, _T) -> nl_indent(I, T) when I > 0 -> [$\n|spaces(I, T)]. +classify_item({atom, _}) -> atomic; +classify_item({singleton_atom_type, _}) -> atomic; +classify_item(Atom) when is_atom(Atom) -> atomic; +classify_item({leaf, _, _}) -> atomic; +classify_item(_) -> complex. + same_line(I0, SizeL, NSepChars) -> try Size = lists:sum(SizeL) + NSepChars, @@ -1150,6 +1247,9 @@ write_a_value(V, PP) -> write_an_atom(A, PP) -> flat_leaf(write_atom(A, PP)). +write_a_singleton_atom_type(A, PP) -> + flat_leaf(write_singleton_atom_type(A, PP)). + write_a_char(C, PP) -> flat_leaf(write_char(C, PP)). @@ -1184,6 +1284,9 @@ write_value(V, PP) -> write_atom(A, PP) -> (PP#pp.value_fun)(A). +write_singleton_atom_type(A, PP) -> + (PP#pp.singleton_atom_type_fun)(A). + write_string(S, PP) -> (PP#pp.string_fun)(S). diff --git a/lib/stdlib/src/gen_fsm.erl b/lib/stdlib/src/gen_fsm.erl index caaaf8fa2e..1e18710738 100644 --- a/lib/stdlib/src/gen_fsm.erl +++ b/lib/stdlib/src/gen_fsm.erl @@ -411,12 +411,13 @@ decode_msg(Msg,Parent, Name, StateName, StateData, Mod, Time, HibernateAfterTime sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, [Name, StateName, StateData, Mod, Time, HibernateAfterTimeout], Hib); {'EXIT', Parent, Reason} -> - terminate(Reason, Name, Msg, Mod, StateName, StateData, Debug); + terminate( + Reason, Name, undefined, Msg, Mod, StateName, StateData, Debug); _Msg when Debug =:= [] -> handle_msg(Msg, Parent, Name, StateName, StateData, Mod, Time, HibernateAfterTimeout); _Msg -> Debug1 = sys:handle_debug(Debug, fun print_event/3, - {Name, StateName}, {in, Msg}), + Name, {in, Msg, StateName}), handle_msg(Msg, Parent, Name, StateName, StateData, Mod, Time, HibernateAfterTimeout, Debug1) end. @@ -431,7 +432,7 @@ system_continue(Parent, Debug, [Name, StateName, StateData, Mod, Time, Hibernate system_terminate(Reason, _Parent, Debug, [Name, StateName, StateData, Mod, _Time, _HibernateAfterTimeout]) -> - terminate(Reason, Name, [], Mod, StateName, StateData, Debug). + terminate(Reason, Name, undefined, [], Mod, StateName, StateData, Debug). system_code_change([Name, StateName, StateData, Mod, Time, HibernateAfterTimeout], _Module, OldVsn, Extra) -> @@ -452,7 +453,7 @@ system_replace_state(StateFun, [Name, StateName, StateData, Mod, Time, Hibernate %% Format debug messages. Print them as the call-back module sees %% them, not as the real erlang messages. Use trace for that. %%----------------------------------------------------------------- -print_event(Dev, {in, Msg}, {Name, StateName}) -> +print_event(Dev, {in, Msg, StateName}, Name) -> case Msg of {'$gen_event', Event} -> io:format(Dev, "*DBG* ~tp got event ~tp in state ~tw~n", @@ -461,6 +462,16 @@ print_event(Dev, {in, Msg}, {Name, StateName}) -> io:format(Dev, "*DBG* ~tp got all_state_event ~tp in state ~tw~n", [Name, Event, StateName]); + {'$gen_sync_event', {From,_Tag}, Event} -> + io:format(Dev, + "*DBG* ~tp got sync_event ~tp " + "from ~tw in state ~tw~n", + [Name, Event, From, StateName]); + {'$gen_sync_all_state_event', {From,_Tag}, Event} -> + io:format(Dev, + "*DBG* ~tp got sync_all_state_event ~tp " + "from ~tw in state ~tw~n", + [Name, Event, From, StateName]); {timeout, Ref, {'$gen_timer', Message}} -> io:format(Dev, "*DBG* ~tp got timer ~tp in state ~tw~n", @@ -473,11 +484,11 @@ print_event(Dev, {in, Msg}, {Name, StateName}) -> io:format(Dev, "*DBG* ~tp got ~tp in state ~tw~n", [Name, Msg, StateName]) end; -print_event(Dev, {out, Msg, To, StateName}, Name) -> +print_event(Dev, {out, Msg, {To,_Tag}, StateName}, Name) -> io:format(Dev, "*DBG* ~tp sent ~tp to ~tw~n" " and switched to state ~tw~n", [Name, Msg, To, StateName]); -print_event(Dev, return, {Name, StateName}) -> +print_event(Dev, {noreply, StateName}, Name) -> io:format(Dev, "*DBG* ~tp switched to state ~tw~n", [Name, StateName]). @@ -495,9 +506,9 @@ handle_msg(Msg, Parent, Name, StateName, StateData, Mod, _Time, HibernateAfterTi reply(From, Reply), loop(Parent, Name, NStateName, NStateData, Mod, Time1, HibernateAfterTimeout, []); {stop, Reason, NStateData} -> - terminate(Reason, Name, Msg, Mod, StateName, NStateData, []); + terminate(Reason, Name, From, Msg, Mod, StateName, NStateData, []); {stop, Reason, Reply, NStateData} when From =/= undefined -> - {'EXIT', R} = (catch terminate(Reason, Name, Msg, Mod, + {'EXIT', R} = (catch terminate(Reason, Name, From, Msg, Mod, StateName, NStateData, [])), reply(From, Reply), exit(R); @@ -510,10 +521,10 @@ handle_msg(Msg, Parent, Name, StateName, StateData, Mod, _Time, HibernateAfterTi error_logger=>#{tag=>warning_msg}}), loop(Parent, Name, StateName, StateData, Mod, infinity, HibernateAfterTimeout, []); {'EXIT', What} -> - terminate(What, Name, Msg, Mod, StateName, StateData, []); + terminate(What, Name, From, Msg, Mod, StateName, StateData, []); Reply -> terminate({bad_return_value, Reply}, - Name, Msg, Mod, StateName, StateData, []) + Name, From, Msg, Mod, StateName, StateData, []) end. handle_msg(Msg, Parent, Name, StateName, StateData, Mod, _Time, HibernateAfterTimeout, Debug) -> @@ -521,11 +532,11 @@ handle_msg(Msg, Parent, Name, StateName, StateData, Mod, _Time, HibernateAfterTi case catch dispatch(Msg, Mod, StateName, StateData) of {next_state, NStateName, NStateData} -> Debug1 = sys:handle_debug(Debug, fun print_event/3, - {Name, NStateName}, return), + Name, {noreply, NStateName}), loop(Parent, Name, NStateName, NStateData, Mod, infinity, HibernateAfterTimeout, Debug1); {next_state, NStateName, NStateData, Time1} -> Debug1 = sys:handle_debug(Debug, fun print_event/3, - {Name, NStateName}, return), + Name, {noreply, NStateName}), loop(Parent, Name, NStateName, NStateData, Mod, Time1, HibernateAfterTimeout, Debug1); {reply, Reply, NStateName, NStateData} when From =/= undefined -> Debug1 = reply(Name, From, Reply, Debug, NStateName), @@ -534,17 +545,18 @@ handle_msg(Msg, Parent, Name, StateName, StateData, Mod, _Time, HibernateAfterTi Debug1 = reply(Name, From, Reply, Debug, NStateName), loop(Parent, Name, NStateName, NStateData, Mod, Time1, HibernateAfterTimeout, Debug1); {stop, Reason, NStateData} -> - terminate(Reason, Name, Msg, Mod, StateName, NStateData, Debug); + terminate( + Reason, Name, From, Msg, Mod, StateName, NStateData, Debug); {stop, Reason, Reply, NStateData} when From =/= undefined -> - {'EXIT', R} = (catch terminate(Reason, Name, Msg, Mod, + {'EXIT', R} = (catch terminate(Reason, Name, From, Msg, Mod, StateName, NStateData, Debug)), _ = reply(Name, From, Reply, Debug, StateName), exit(R); {'EXIT', What} -> - terminate(What, Name, Msg, Mod, StateName, StateData, Debug); + terminate(What, Name, From, Msg, Mod, StateName, StateData, Debug); Reply -> terminate({bad_return_value, Reply}, - Name, Msg, Mod, StateName, StateData, Debug) + Name, From, Msg, Mod, StateName, StateData, Debug) end. dispatch({'$gen_event', Event}, Mod, StateName, StateData) -> @@ -571,24 +583,25 @@ from(_) -> undefined. reply({To, Tag}, Reply) -> catch To ! {Tag, Reply}. -reply(Name, {To, Tag}, Reply, Debug, StateName) -> - reply({To, Tag}, Reply), +reply(Name, From, Reply, Debug, StateName) -> + reply(From, Reply), sys:handle_debug(Debug, fun print_event/3, Name, - {out, Reply, To, StateName}). + {out, Reply, From, StateName}). %%% --------------------------------------------------- %%% Terminate the server. %%% --------------------------------------------------- --spec terminate(term(), _, _, atom(), _, _, _) -> no_return(). +-spec terminate(term(), _, _, _, atom(), _, _, _) -> no_return(). -terminate(Reason, Name, Msg, Mod, StateName, StateData, Debug) -> +terminate(Reason, Name, From, Msg, Mod, StateName, StateData, Debug) -> case erlang:function_exported(Mod, terminate, 3) of true -> case catch Mod:terminate(Reason, StateName, StateData) of {'EXIT', R} -> FmtStateData = format_status(terminate, Mod, get(), StateData), - error_info(R, Name, Msg, StateName, FmtStateData, Debug), + error_info( + R, Name, From, Msg, StateName, FmtStateData, Debug), exit(R); _ -> ok @@ -605,29 +618,51 @@ terminate(Reason, Name, Msg, Mod, StateName, StateData, Debug) -> exit(Shutdown); _ -> FmtStateData1 = format_status(terminate, Mod, get(), StateData), - error_info(Reason,Name,Msg,StateName,FmtStateData1,Debug), + error_info( + Reason, Name, From, Msg, StateName, FmtStateData1, Debug), exit(Reason) end. -error_info(Reason, Name, Msg, StateName, StateData, Debug) -> +error_info(Reason, Name, From, Msg, StateName, StateData, Debug) -> + Log = sys:get_log(Debug), ?LOG_ERROR(#{label=>{gen_fsm,terminate}, name=>Name, last_message=>Msg, state_name=>StateName, state_data=>StateData, - reason=>Reason}, + log=>Log, + reason=>Reason, + client_info=>client_stacktrace(From)}, #{domain=>[otp], report_cb=>fun gen_fsm:format_log/1, error_logger=>#{tag=>error}}), - sys:print_log(Debug), ok. +client_stacktrace(undefined) -> + undefined; +client_stacktrace({Pid,_Tag}) -> + client_stacktrace(Pid); +client_stacktrace(Pid) when is_pid(Pid), node(Pid) =:= node() -> + case process_info(Pid, [current_stacktrace, registered_name]) of + undefined -> + {Pid,dead}; + [{current_stacktrace, Stacktrace}, {registered_name, []}] -> + {Pid,{Pid,Stacktrace}}; + [{current_stacktrace, Stacktrace}, {registered_name, Name}] -> + {Pid,{Name,Stacktrace}} + end; +client_stacktrace(Pid) when is_pid(Pid) -> + {Pid,remote}. + + format_log(#{label:={gen_fsm,terminate}, name:=Name, last_message:=Msg, state_name:=StateName, state_data:=StateData, - reason:=Reason}) -> + log:=Log, + reason:=Reason, + client_info:=ClientInfo}) -> Reason1 = case Reason of {undef,[{M,F,A,L}|MFAs]} -> @@ -645,27 +680,39 @@ format_log(#{label:={gen_fsm,terminate}, _ -> Reason end, + {ClientFmt,ClientArgs} = format_client_log(ClientInfo), {"** State machine ~tp terminating \n" ++ get_msg_str(Msg) ++ "** When State == ~tp~n" "** Data == ~tp~n" - "** Reason for termination = ~n** ~tp~n", - [Name, get_msg(Msg), StateName, StateData, Reason1]}; + "** Reason for termination ==~n** ~tp~n" ++ + case Log of + [] -> []; + _ -> "** Log ==~n** ~tp~n" + end ++ ClientFmt, + [Name|error_logger:limit_term(get_msg(Msg))] ++ + [StateName, + error_logger:limit_term(StateData), + error_logger:limit_term(Reason1) | + case Log of + [] -> []; + _ -> [[error_logger:limit_term(D) || D <- Log]] + end] ++ ClientArgs}; format_log(#{label:={gen_fsm,no_handle_info}, module:=Mod, message:=Msg}) -> {"** Undefined handle_info in ~p~n" "** Unhandled message: ~tp~n", - [Mod, Msg]}. + [Mod, error_logger:limit_term(Msg)]}. get_msg_str({'$gen_event', _Event}) -> "** Last event in was ~tp~n"; -get_msg_str({'$gen_sync_event', _Event}) -> - "** Last sync event in was ~tp~n"; +get_msg_str({'$gen_sync_event', _From, _Event}) -> + "** Last sync event in was ~tp from ~tw~n"; get_msg_str({'$gen_all_state_event', _Event}) -> "** Last event in was ~tp (for all states)~n"; -get_msg_str({'$gen_sync_all_state_event', _Event}) -> - "** Last sync event in was ~tp (for all states)~n"; +get_msg_str({'$gen_sync_all_state_event', _From, _Event}) -> + "** Last sync event in was ~tp (for all states) from ~tw~n"; get_msg_str({timeout, _Ref, {'$gen_timer', _Msg}}) -> "** Last timer event in was ~tp~n"; get_msg_str({timeout, _Ref, {'$gen_event', _Msg}}) -> @@ -673,13 +720,24 @@ get_msg_str({timeout, _Ref, {'$gen_event', _Msg}}) -> get_msg_str(_Msg) -> "** Last message in was ~tp~n". -get_msg({'$gen_event', Event}) -> Event; -get_msg({'$gen_sync_event', Event}) -> Event; -get_msg({'$gen_all_state_event', Event}) -> Event; -get_msg({'$gen_sync_all_state_event', Event}) -> Event; -get_msg({timeout, Ref, {'$gen_timer', Msg}}) -> {timeout, Ref, Msg}; -get_msg({timeout, _Ref, {'$gen_event', Event}}) -> Event; -get_msg(Msg) -> Msg. +get_msg({'$gen_event', Event}) -> [Event]; +get_msg({'$gen_sync_event', {From,_Tag}, Event}) -> [Event,From]; +get_msg({'$gen_all_state_event', Event}) -> [Event]; +get_msg({'$gen_sync_all_state_event', {From,_Tag}, Event}) -> [Event,From]; +get_msg({timeout, Ref, {'$gen_timer', Msg}}) -> [{timeout, Ref, Msg}]; +get_msg({timeout, _Ref, {'$gen_event', Event}}) -> [Event]; +get_msg(Msg) -> [Msg]. + +format_client_log(undefined) -> + {"", []}; +format_client_log({From,dead}) -> + {"** Client ~p is dead~n", [From]}; +format_client_log({From,remote}) -> + {"** Client ~p is remote on node ~p~n", [From, node(From)]}; +format_client_log({_From,{Name,Stacktrace}}) -> + {"** Client ~tp stacktrace~n" + "** ~tp~n", + [Name, error_logger:limit_term(Stacktrace)]}. %%----------------------------------------------------------------- %% Status information @@ -689,18 +747,18 @@ format_status(Opt, StatusData) -> StatusData, Header = gen:format_status_header("Status for state machine", Name), - Log = sys:get_debug(log, Debug, []), - Specfic = format_status(Opt, Mod, PDict, StateData), - Specfic = case format_status(Opt, Mod, PDict, StateData) of - S when is_list(S) -> S; - S -> [S] - end, + Log = sys:get_log(Debug), + Specific = + case format_status(Opt, Mod, PDict, StateData) of + S when is_list(S) -> S; + S -> [S] + end, [{header, Header}, {data, [{"Status", SysState}, {"Parent", Parent}, {"Logged events", Log}, {"StateName", StateName}]} | - Specfic]. + Specific]. format_status(Opt, Mod, PDict, State) -> DefStatus = case Opt of diff --git a/lib/stdlib/src/gen_server.erl b/lib/stdlib/src/gen_server.erl index 44e9231ebe..c7b6406f54 100644 --- a/lib/stdlib/src/gen_server.erl +++ b/lib/stdlib/src/gen_server.erl @@ -773,10 +773,10 @@ handle_common_reply(Reply, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, terminate({bad_return_value, BadReply}, ?STACKTRACE(), Name, From, Msg, Mod, State, Debug) end. -reply(Name, {To, Tag}, Reply, State, Debug) -> - reply({To, Tag}, Reply), +reply(Name, From, Reply, State, Debug) -> + reply(From, Reply), sys:handle_debug(Debug, fun print_event/3, Name, - {out, Reply, To, State} ). + {out, Reply, From, State} ). %%----------------------------------------------------------------- @@ -810,7 +810,7 @@ system_replace_state(StateFun, [Name, State, Mod, Time, HibernateAfterTimeout]) print_event(Dev, {in, Msg}, Name) -> case Msg of {'$gen_call', {From, _Tag}, Call} -> - io:format(Dev, "*DBG* ~tp got call ~tp from ~w~n", + io:format(Dev, "*DBG* ~tp got call ~tp from ~tw~n", [Name, Call, From]); {'$gen_cast', Cast} -> io:format(Dev, "*DBG* ~tp got cast ~tp~n", @@ -818,8 +818,8 @@ print_event(Dev, {in, Msg}, Name) -> _ -> io:format(Dev, "*DBG* ~tp got ~tp~n", [Name, Msg]) end; -print_event(Dev, {out, Msg, To, State}, Name) -> - io:format(Dev, "*DBG* ~tp sent ~tp to ~w, new state ~tp~n", +print_event(Dev, {out, Msg, {To,_Tag}, State}, Name) -> + io:format(Dev, "*DBG* ~tp sent ~tp to ~tw, new state ~tp~n", [Name, Msg, To, State]); print_event(Dev, {noreply, State}, Name) -> io:format(Dev, "*DBG* ~tp new state ~tp~n", [Name, State]); @@ -885,16 +885,17 @@ error_info(_Reason, application_controller, _From, _Msg, _Mod, _State, _Debug) - %% of it instead ok; error_info(Reason, Name, From, Msg, Mod, State, Debug) -> + Log = sys:get_log(Debug), ?LOG_ERROR(#{label=>{gen_server,terminate}, name=>Name, last_message=>Msg, state=>format_status(terminate, Mod, get(), State), + log=>format_log_state(Mod, Log), reason=>Reason, client_info=>client_stacktrace(From)}, #{domain=>[otp], report_cb=>fun gen_server:format_log/1, error_logger=>#{tag=>error}}), - sys:print_log(Debug), ok. client_stacktrace(undefined) -> @@ -917,6 +918,7 @@ format_log(#{label:={gen_server,terminate}, name:=Name, last_message:=Msg, state:=State, + log:=Log, reason:=Reason, client_info:=Client}) -> Reason1 = @@ -934,20 +936,30 @@ format_log(#{label:={gen_server,terminate}, end end; _ -> - error_logger:limit_term(Reason) + Reason end, {ClientFmt,ClientArgs} = format_client_log(Client), + [LimitedMsg,LimitedState,LimitedReason|LimitedLog] = + [error_logger:limit_term(D) || D <- [Msg,State,Reason1|Log]], {"** Generic server ~tp terminating \n" "** Last message in was ~tp~n" "** When Server state == ~tp~n" - "** Reason for termination == ~n** ~tp~n" ++ ClientFmt, - [Name, Msg, error_logger:limit_term(State), Reason1] ++ ClientArgs}; + "** Reason for termination ==~n** ~tp~n" ++ + case LimitedLog of + [] -> []; + _ -> "** Log ==~n** ~tp~n" + end ++ ClientFmt, + [Name, LimitedMsg, LimitedState, LimitedReason] ++ + case LimitedLog of + [] -> []; + _ -> [LimitedLog] + end ++ ClientArgs}; format_log(#{label:={gen_server,no_handle_info}, module:=Mod, message:=Msg}) -> {"** Undefined handle_info in ~p~n" "** Unhandled message: ~tp~n", - [Mod, Msg]}. + [Mod, error_logger:limit_term(Msg)]}. format_client_log(undefined) -> {"", []}; @@ -958,7 +970,7 @@ format_client_log({From,remote}) -> format_client_log({_From,{Name,Stacktrace}}) -> {"** Client ~tp stacktrace~n" "** ~tp~n", - [Name, Stacktrace]}. + [Name, error_logger:limit_term(Stacktrace)]}. %%----------------------------------------------------------------- %% Status information @@ -966,16 +978,25 @@ format_client_log({_From,{Name,Stacktrace}}) -> format_status(Opt, StatusData) -> [PDict, SysState, Parent, Debug, [Name, State, Mod, _Time, _HibernateAfterTimeout]] = StatusData, Header = gen:format_status_header("Status for generic server", Name), - Log = sys:get_debug(log, Debug, []), - Specfic = case format_status(Opt, Mod, PDict, State) of + Log = sys:get_log(Debug), + Specific = case format_status(Opt, Mod, PDict, State) of S when is_list(S) -> S; S -> [S] end, [{header, Header}, {data, [{"Status", SysState}, {"Parent", Parent}, - {"Logged events", Log}]} | - Specfic]. + {"Logged events", format_log_state(Mod, Log)}]} | + Specific]. + +format_log_state(Mod, Log) -> + [case Event of + {out,Msg,From,State} -> + {out,Msg,From,format_status(terminate, Mod, get(), State)}; + {noreply,State} -> + {noreply,format_status(terminate, Mod, get(), State)}; + _ -> Event + end || Event <- Log]. format_status(Opt, Mod, PDict, State) -> DefStatus = case Opt of diff --git a/lib/stdlib/src/gen_statem.erl b/lib/stdlib/src/gen_statem.erl index 8965af253b..49911eac2c 100644 --- a/lib/stdlib/src/gen_statem.erl +++ b/lib/stdlib/src/gen_statem.erl @@ -67,6 +67,14 @@ %% Type that is exported just to be documented -export_type([transition_option/0]). +%% Type exports for start_link & friends +-export_type( + [server_name/0, + server_ref/0, + start_opt/0, + start_ret/0, + enter_loop_opt/0]). + %%%========================================================================== %%% Interface functions. %%%========================================================================== @@ -330,6 +338,7 @@ %% Type validation functions +%% - return true if the value is of the type, false otherwise -compile( {inline, [callback_mode/1, state_enter/1, @@ -373,71 +382,76 @@ timeout_event_type(Type) -> -define( + relative_timeout(T), + ((is_integer(T) andalso 0 =< (T)) orelse (T) =:= infinity)). + +-define( + absolute_timeout(T), + (is_integer(T) orelse (T) =:= infinity)). + +-define( STACKTRACE(), element(2, erlang:process_info(self(), current_stacktrace))). -define(not_sys_debug, []). %% %% This is a macro to only evaluate arguments if Debug =/= []. -%% Debug is evaluated multiple times. +%% Debug is evaluated 2 times. -define( - sys_debug(Debug, NameState, Entry), + sys_debug(Debug, Extra, SystemEvent), case begin Debug end of ?not_sys_debug -> begin Debug end; _ -> - sys_debug(begin Debug end, begin NameState end, begin Entry end) + sys_debug( + begin Debug end, begin Extra end, begin SystemEvent end) end). --record(state, +-record(params, {callback_mode = undefined :: callback_mode() | undefined, state_enter = false :: boolean(), + parent :: pid(), module :: atom(), - name :: atom(), - state :: term(), - data :: term(), + name :: atom() | pid(), + hibernate_after = infinity :: timeout() + }). + +-record(state, + {state_data = {undefined,undefined} :: + {State :: term(),Data :: term()}, postponed = [] :: [{event_type(),term()}], - %% timers = {#{},#{}} :: - {%% timer ref => the timer's event type + {%% TimerRef => TimeoutType TimerRefs :: #{reference() => timeout_event_type()}, - %% timer's event type => timer ref - TimerTypes :: #{timeout_event_type() => reference()}}, - hibernate = false :: boolean(), - hibernate_after = infinity :: timeout()}). - --record(trans_opts, - {hibernate = false, - postpone = false, - timeouts_r = [], - next_events_r = []}). + %% TimeoutType => TimerRef + TimeoutTypes :: #{timeout_event_type() => reference()}}, + hibernate = false :: boolean() + }). %%%========================================================================== %%% API -type server_name() :: - {'global', GlobalName :: term()} + {'global', GlobalName :: term()} | {'via', RegMod :: module(), Name :: term()} | {'local', atom()}. -type server_ref() :: - pid() + pid() | (LocalName :: atom()) | {Name :: atom(), Node :: atom()} | {'global', GlobalName :: term()} | {'via', RegMod :: module(), ViaName :: term()}. --type debug_opt() :: - {'debug', - Dbgs :: - ['trace' | 'log' | 'statistics' | 'debug' - | {'logfile', string()}]}. --type hibernate_after_opt() :: - {'hibernate_after', HibernateAfterTimeout :: timeout()}. -type start_opt() :: - debug_opt() - | {'timeout', Time :: timeout()} - | hibernate_after_opt() - | {'spawn_opt', [proc_lib:spawn_option()]}. --type start_ret() :: {'ok', pid()} | 'ignore' | {'error', term()}. + {'timeout', Time :: timeout()} + | {'spawn_opt', [proc_lib:spawn_option()]} + | enter_loop_opt(). +-type start_ret() :: + {'ok', pid()} + | 'ignore' + | {'error', term()}. +-type enter_loop_opt() :: + {'hibernate_after', HibernateAfterTimeout :: timeout()} + | {'debug', Dbgs :: [sys:debug_option()]}. @@ -550,14 +564,14 @@ reply({To,Tag}, Reply) when is_pid(To) -> %% started by proc_lib into a state machine using %% the same arguments as you would have returned from init/1 -spec enter_loop( - Module :: module(), Opts :: [debug_opt() | hibernate_after_opt()], + Module :: module(), Opts :: [enter_loop_opt()], State :: state(), Data :: data()) -> no_return(). enter_loop(Module, Opts, State, Data) -> enter_loop(Module, Opts, State, Data, self()). %% -spec enter_loop( - Module :: module(), Opts :: [debug_opt() | hibernate_after_opt()], + Module :: module(), Opts :: [enter_loop_opt()], State :: state(), Data :: data(), Server_or_Actions :: server_name() | pid() | [action()]) -> @@ -571,7 +585,7 @@ enter_loop(Module, Opts, State, Data, Server_or_Actions) -> end. %% -spec enter_loop( - Module :: module(), Opts :: [debug_opt() | hibernate_after_opt()], + Module :: module(), Opts :: [enter_loop_opt()], State :: state(), Data :: data(), Server :: server_name() | pid(), Actions :: [action()] | action()) -> @@ -579,7 +593,12 @@ enter_loop(Module, Opts, State, Data, Server_or_Actions) -> enter_loop(Module, Opts, State, Data, Server, Actions) -> is_atom(Module) orelse error({atom,Module}), Parent = gen:get_parent(), - enter(Module, Opts, State, Data, Server, Actions, Parent). + Name = gen:get_proc_name(Server), + Debug = gen:debug_options(Name, Opts), + HibernateAfterTimeout = gen:hibernate_after(Opts), + enter( + Parent, Debug, Module, Name, HibernateAfterTimeout, + State, Data, Actions). %%--------------------------------------------------------------------------- %% API helpers @@ -651,36 +670,29 @@ send(Proc, Msg) -> ok. %% Here the init_it/6 and enter_loop/5,6,7 functions converge -enter(Module, Opts, State, Data, Server, Actions, Parent) -> +enter( + Parent, Debug, Module, Name, HibernateAfterTimeout, + State, Data, Actions) -> %% The values should already have been type checked - Name = gen:get_proc_name(Server), - Debug = gen:debug_options(Name, Opts), - HibernateAfterTimeout = gen:hibernate_after(Opts), - Events = [], - Event = {internal,init_state}, + Q = [{internal,init_state}], %% We enforce {postpone,false} to ensure that %% our fake Event gets discarded, thought it might get logged - NewActions = listify(Actions) ++ [{postpone,false}], - S = - #state{ + Actions_1 = listify(Actions) ++ [{postpone,false}], + P = + #params{ + parent = Parent, module = Module, name = Name, - state = State, - data = Data, hibernate_after = HibernateAfterTimeout}, - CallEnter = true, - NewDebug = ?sys_debug(Debug, {Name,State}, {enter,Event,State}), - case call_callback_mode(S) of - #state{} = NewS -> - loop_event_actions_list( - Parent, NewDebug, NewS, - Events, Event, State, Data, false, - NewActions, CallEnter); - [Class,Reason,Stacktrace] -> - terminate( - Class, Reason, Stacktrace, NewDebug, - S, [Event|Events]) - end. + S = #state{state_data = {State,Data}}, + Debug_1 = ?sys_debug(Debug, Name, {enter,State}), + loop_callback_mode( + P, Debug_1, S, Q, {State,Data}, + %% Tunneling Actions through CallbackEvent here... + %% Special path to go to action handling, after first + %% finding out the callback mode. CallbackEvent is + %% a 2-tuple and Actions a list, which achieves this distinction. + Actions_1). %%%========================================================================== %%% gen callbacks @@ -688,34 +700,46 @@ enter(Module, Opts, State, Data, Server, Actions, Parent) -> init_it(Starter, self, ServerRef, Module, Args, Opts) -> init_it(Starter, self(), ServerRef, Module, Args, Opts); init_it(Starter, Parent, ServerRef, Module, Args, Opts) -> + Name = gen:get_proc_name(ServerRef), + Debug = gen:debug_options(Name, Opts), + HibernateAfterTimeout = gen:hibernate_after(Opts), try Module:init(Args) of Result -> - init_result(Starter, Parent, ServerRef, Module, Result, Opts) + init_result( + Starter, Parent, ServerRef, Module, Result, + Name, Debug, HibernateAfterTimeout) catch Result -> - init_result(Starter, Parent, ServerRef, Module, Result, Opts); + init_result( + Starter, Parent, ServerRef, Module, Result, + Name, Debug, HibernateAfterTimeout); Class:Reason:Stacktrace -> - Name = gen:get_proc_name(ServerRef), gen:unregister_name(ServerRef), proc_lib:init_ack(Starter, {error,Reason}), error_info( - Class, Reason, Stacktrace, - #state{name = Name}, - []), + Class, Reason, Stacktrace, Debug, + #params{parent = Parent, name = Name, module = Module}, + #state{}, []), erlang:raise(Class, Reason, Stacktrace) end. %%--------------------------------------------------------------------------- %% gen callbacks helpers -init_result(Starter, Parent, ServerRef, Module, Result, Opts) -> +init_result( + Starter, Parent, ServerRef, Module, Result, + Name, Debug, HibernateAfterTimeout) -> case Result of {ok,State,Data} -> proc_lib:init_ack(Starter, {ok,self()}), - enter(Module, Opts, State, Data, ServerRef, [], Parent); + enter( + Parent, Debug, Module, Name, HibernateAfterTimeout, + State, Data, []); {ok,State,Data,Actions} -> proc_lib:init_ack(Starter, {ok,self()}), - enter(Module, Opts, State, Data, ServerRef, Actions, Parent); + enter( + Parent, Debug, Module, Name, HibernateAfterTimeout, + State, Data, Actions); {stop,Reason} -> gen:unregister_name(ServerRef), proc_lib:init_ack(Starter, {error,Reason}), @@ -725,31 +749,34 @@ init_result(Starter, Parent, ServerRef, Module, Result, Opts) -> proc_lib:init_ack(Starter, ignore), exit(normal); _ -> - Name = gen:get_proc_name(ServerRef), gen:unregister_name(ServerRef), Error = {bad_return_from_init,Result}, proc_lib:init_ack(Starter, {error,Error}), error_info( - error, Error, ?STACKTRACE(), - #state{name = Name}, - []), + error, Error, ?STACKTRACE(), Debug, + #params{parent = Parent, name = Name, module = Module}, + #state{}, []), exit(Error) end. %%%========================================================================== %%% sys callbacks +%%% +%%% We use {P,S} as state (Misc) for the sys module, +%%% wrap/unwrap it for the server loop* and update +%%% P#params{parent = Parent}. -system_continue(Parent, Debug, S) -> - loop(Parent, Debug, S). +system_continue(Parent, Debug, {P,S}) -> + loop(update_parent(P, Parent), Debug, S). -system_terminate(Reason, _Parent, Debug, S) -> - terminate(exit, Reason, ?STACKTRACE(), Debug, S, []). +system_terminate(Reason, Parent, Debug, {P,S}) -> + terminate( + exit, Reason, ?STACKTRACE(), + update_parent(P, Parent), Debug, S, []). system_code_change( - #state{ - module = Module, - state = State, - data = Data} = S, + {#params{module = Module} = P, + #state{state_data = {State,Data}} = S}, _Mod, OldVsn, Extra) -> case try Module:code_change(OldVsn, State, Data, Extra) @@ -759,44 +786,54 @@ system_code_change( of {ok,NewState,NewData} -> {ok, - S#state{ - callback_mode = undefined, - state = NewState, - data = NewData}}; + {P#params{callback_mode = undefined}, + S#state{state_data = {NewState,NewData}}}}; {ok,_} = Error -> error({case_clause,Error}); Error -> Error end. -system_get_state(#state{state = State, data = Data}) -> - {ok,{State,Data}}. +system_get_state({_P,#state{state_data = State_Data}}) -> + {ok,State_Data}. system_replace_state( - StateFun, - #state{ - state = State, - data = Data} = S) -> - {NewState,NewData} = Result = StateFun({State,Data}), - {ok,Result,S#state{state = NewState, data = NewData}}. + StateFun, {P,#state{state_data = State_Data} = S}) -> + %% + NewState_NewData = StateFun(State_Data), + {ok,NewState_NewData,{P,S#state{state_data = NewState_NewData}}}. format_status( Opt, [PDict,SysState,Parent,Debug, - #state{name = Name, postponed = P} = S]) -> + {#params{name = Name} = P, + #state{postponed = Postponed} = S}]) -> Header = gen:format_status_header("Status for state machine", Name), - Log = sys:get_debug(log, Debug, []), + Log = sys:get_log(Debug), [{header,Header}, {data, [{"Status",SysState}, {"Parent",Parent}, {"Logged Events",Log}, - {"Postponed",P}]} | - case format_status(Opt, PDict, S) of + {"Postponed",Postponed}]} | + case format_status(Opt, PDict, update_parent(P, Parent), S) of L when is_list(L) -> L; T -> [T] end]. +%% Update #params.parent only if it differs. This should not +%% be possible today (OTP-22.0), but could happen for example +%% if someone implements changing a server's parent +%% in a new sys call. +-compile({inline, update_parent/2}). +update_parent(P, Parent) -> + case P of + #params{parent = Parent} -> + P; + #params{} -> + P#params{parent = Parent} + end. + %%--------------------------------------------------------------------------- %% Format debug messages. Print them as the call-back module sees %% them, not as the real erlang messages. Use trace for that. @@ -805,34 +842,46 @@ format_status( sys_debug(Debug, NameState, Entry) -> sys:handle_debug(Debug, fun print_event/3, NameState, Entry). -print_event(Dev, {in,Event}, {Name,State}) -> - io:format( - Dev, "*DBG* ~tp receive ~ts in state ~tp~n", - [Name,event_string(Event),State]); -print_event(Dev, {out,Reply,{To,_Tag}}, {Name,State}) -> - io:format( - Dev, "*DBG* ~tp send ~tp to ~p from state ~tp~n", - [Name,Reply,To,State]); -print_event(Dev, {terminate,Reason}, {Name,State}) -> - io:format( - Dev, "*DBG* ~tp terminate ~tp in state ~tp~n", - [Name,Reason,State]); -print_event(Dev, {Tag,Event,NextState}, {Name,State}) -> - StateString = - case NextState of - State -> - io_lib:format("~tp", [State]); - _ -> - io_lib:format("~tp => ~tp", [State,NextState]) - end, - io:format( - Dev, "*DBG* ~tp ~tw ~ts in state ~ts~n", - [Name,Tag,event_string(Event),StateString]). +print_event(Dev, SystemEvent, Name) -> + case SystemEvent of + {in,Event,State} -> + io:format( + Dev, "*DBG* ~tp receive ~ts in state ~tp~n", + [Name,event_string(Event),State]); + {code_change,Event,State} -> + io:format( + Dev, "*DBG* ~tp receive ~ts after code change in state ~tp~n", + [Name,event_string(Event),State]); + {out,Reply,{To,_Tag}} -> + io:format( + Dev, "*DBG* ~tp send ~tp to ~tw~n", + [Name,Reply,To]); + {enter,State} -> + io:format( + Dev, "*DBG* ~tp enter in state ~tp~n", + [Name,State]); + {terminate,Reason,State} -> + io:format( + Dev, "*DBG* ~tp terminate ~tp in state ~tp~n", + [Name,Reason,State]); + {Tag,Event,State,NextState} + when Tag =:= postpone; Tag =:= consume -> + StateString = + case NextState of + State -> + io_lib:format("~tp", [State]); + _ -> + io_lib:format("~tp => ~tp", [State,NextState]) + end, + io:format( + Dev, "*DBG* ~tp ~tw ~ts in state ~ts~n", + [Name,Tag,event_string(Event),StateString]) + end. event_string(Event) -> case Event of {{call,{Pid,_Tag}},Request} -> - io_lib:format("call ~tp from ~w", [Request,Pid]); + io_lib:format("call ~tp from ~tw", [Request,Pid]); {EventType,EventContent} -> io_lib:format("~tw ~tp", [EventType,EventContent]) end. @@ -840,874 +889,1169 @@ event_string(Event) -> %%%========================================================================== %%% Internal callbacks -wakeup_from_hibernate(Parent, Debug, S) -> +wakeup_from_hibernate(P, Debug, S) -> %% It is a new message that woke us up so we have to receive it now - loop_receive(Parent, Debug, S). + loop_receive(P, Debug, S). %%%========================================================================== -%%% State Machine engine implementation of proc_lib/gen server +%%% State Machine engine implementation on proc_lib/gen %% Server loop, consists of all loop* functions %% and detours through sys:handle_system_message/7 and proc_lib:hibernate/3 +%% +%% The loop tries to keep all temporary values in arguments +%% and takes shortcuts for ?not_sys_debug, empty lists, etc. +%% The engine state #state{} is picked apart during the loop, +%% new values are kept in arguments, and a new #state{} is +%% composed at the end of the loop. #params{} collect engine +%% state fields that rarely changes. +%% +%% The loop is optimized a bit for staying in the loop, assuming that +%% system events are rare. So a detour to sys requires re-packing +%% of the engine state. %% Entry point for system_continue/3 -loop(Parent, Debug, #state{hibernate = true} = S) -> - loop_hibernate(Parent, Debug, S); -loop(Parent, Debug, S) -> - loop_receive(Parent, Debug, S). +%% +loop(P, Debug, #state{hibernate = true} = S) -> + loop_hibernate(P, Debug, S); +loop(P, Debug, S) -> + loop_receive(P, Debug, S). -loop_hibernate(Parent, Debug, S) -> +%% Go to hibernation +%% +loop_hibernate(P, Debug, S) -> %% %% Does not return but restarts process at %% wakeup_from_hibernate/3 that jumps to loop_receive/3 %% - proc_lib:hibernate( - ?MODULE, wakeup_from_hibernate, [Parent,Debug,S]), + proc_lib:hibernate(?MODULE, wakeup_from_hibernate, [P, Debug, S]), error( {should_not_have_arrived_here_but_instead_in, - {wakeup_from_hibernate,3}}). + {?MODULE,wakeup_from_hibernate,3}}). + %% Entry point for wakeup_from_hibernate/3 +%% +%% Receive a new process message +%% loop_receive( - Parent, Debug, #state{hibernate_after = HibernateAfterTimeout} = S) -> + #params{hibernate_after = HibernateAfterTimeout} = P, Debug, S) -> %% receive Msg -> case Msg of + {'$gen_call',From,Request} -> + loop_receive_result(P, Debug, S, {{call,From},Request}); + {'$gen_cast',Cast} -> + loop_receive_result(P, Debug, S, {cast,Cast}); + %% + {timeout,TimerRef,TimeoutMsg} -> + {TimerRefs,TimeoutTypes} = S#state.timers, + case TimerRefs of + #{TimerRef := TimeoutType} -> + %% Our timer + Timers = + {maps:remove(TimerRef, TimerRefs), + maps:remove(TimeoutType, TimeoutTypes)}, + S_1 = S#state{timers = Timers}, + loop_receive_result( + P, Debug, S_1, {TimeoutType,TimeoutMsg}); + #{} -> + loop_receive_result(P, Debug, S, {info,Msg}) + end; + %% {system,Pid,Req} -> %% Does not return but tail recursively calls %% system_continue/3 that jumps to loop/3 sys:handle_system_msg( - Req, Pid, Parent, ?MODULE, Debug, S, + Req, Pid, P#params.parent, ?MODULE, Debug, + {P,S}, S#state.hibernate); - {'EXIT',Parent,Reason} = EXIT -> - %% EXIT is not a 2-tuple therefore - %% not an event but this will stand out - %% in the crash report... - Q = [EXIT], - terminate(exit, Reason, ?STACKTRACE(), Debug, S, Q); - {timeout,TimerRef,TimerMsg} -> - case S#state.timers of - {#{TimerRef := TimerType} = TimerRefs,TimerTypes} -> - %% Our timer - NewTimers = - {maps:remove(TimerRef, TimerRefs), - maps:remove(TimerType, TimerTypes)}, - loop_receive_result( - Parent, Debug, - S#state{timers = NewTimers}, - TimerType, TimerMsg); - {#{},_} -> - %% Not our timer; present it as an event - loop_receive_result(Parent, Debug, S, info, Msg) - end; - _ -> - %% External msg - case Msg of - {'$gen_call',From,Request} -> - loop_receive_result( - Parent, Debug, S, {call,From}, Request); - {'$gen_cast',Cast} -> - loop_receive_result(Parent, Debug, S, cast, Cast); + {'EXIT',Pid,Reason} -> + case P#params.parent of + Pid -> + terminate( + exit, Reason, ?STACKTRACE(), P, Debug, S, []); _ -> - loop_receive_result(Parent, Debug, S, info, Msg) - end + loop_receive_result(P, Debug, S, {info,Msg}) + end; + %% + _ -> + loop_receive_result(P, Debug, S, {info,Msg}) end after - HibernateAfterTimeout -> - loop_hibernate(Parent, Debug, S) + HibernateAfterTimeout -> + loop_hibernate(P, Debug, S) end. -loop_receive_result(Parent, ?not_sys_debug, S, Type, Content) -> +%% We have received an event +%% +loop_receive_result(P, ?not_sys_debug = Debug, S, Event) -> %% Here is the queue of not yet handled events created Events = [], - loop_event(Parent, ?not_sys_debug, S, Events, Type, Content); + loop_event(P, Debug, S, Event, Events); loop_receive_result( - Parent, Debug, #state{name = Name, state = State} = S, Type, Content) -> - NewDebug = sys_debug(Debug, {Name,State}, {in,{Type,Content}}), + #params{name = Name, callback_mode = CallbackMode} = P, Debug, + #state{state_data = {State,_Data}} = S, Event) -> + Debug_1 = + case CallbackMode of + undefined -> + sys_debug(Debug, Name, {code_change,Event,State}); + _ -> + sys_debug(Debug, Name, {in,Event,State}) + end, %% Here is the queue of not yet handled events created Events = [], - loop_event(Parent, NewDebug, S, Events, Type, Content). + loop_event(P, Debug_1, S, Event, Events). -%% Entry point for handling an event, received or enqueued +%% Handle one event; received or enqueued +%% loop_event( - Parent, Debug, #state{hibernate = Hibernate} = S, - Events, Type, Content) -> + P, Debug, #state{hibernate = true} = S, Event, Events) -> %% - case Hibernate of - true -> - %% - %% If (this old) Hibernate is true here it can only be - %% because it was set from an event action - %% and we did not go into hibernation since there were - %% events in queue, so we do what the user - %% might rely on i.e collect garbage which - %% would have happened if we actually hibernated - %% and immediately was awakened. - %% - _ = garbage_collect(), - loop_event_state_function( - Parent, Debug, S, Events, Type, Content); - false -> - loop_event_state_function( - Parent, Debug, S, Events, Type, Content) - end. + %% If (this old) Hibernate is true here it can only be + %% because it was set from an event action + %% and we did not go into hibernation since there were + %% events in queue, so we do what the user + %% might rely on i.e collect garbage which + %% would have happened if we actually hibernated + %% and immediately was awakened. + %% + _ = garbage_collect(), + loop_event_handler(P, Debug, S, Event, Events); +loop_event(P, Debug, S, Event, Events) -> + loop_event_handler(P, Debug, S, Event, Events). -%% Call the state function -loop_event_state_function( - Parent, Debug, - #state{state = State, data = Data} = S, - Events, Type, Content) -> +%% Call the state function, eventually +%% +-compile({inline, [loop_event_handler/5]}). +loop_event_handler( + P, Debug, #state{state_data = State_Data} = S, Event, Events) -> %% %% The field 'hibernate' in S is now invalid and will be - %% restored when looping back to loop/3 or loop_event/6. + %% restored when looping back to loop/3 or loop_event/5. %% - Event = {Type,Content}, - TransOpts = false, - case call_state_function(S, Type, Content, State, Data) of - {Result, NewS} -> - loop_event_result( - Parent, Debug, NewS, - Events, Event, State, Data, TransOpts, Result); - [Class,Reason,Stacktrace] -> - terminate( - Class, Reason, Stacktrace, Debug, S, [Event|Events]) - end. + Q = [Event|Events], + loop_callback_mode(P, Debug, S, Q, State_Data, Event). -%% Make a state enter call to the state function -loop_event_state_enter( - Parent, Debug, #state{state = PrevState} = S, - Events, Event, NextState, NewData, TransOpts) -> +%% Figure out the callback mode +%% +loop_callback_mode( + #params{callback_mode = undefined} = P, Debug, S, + Q, State_Data, CallbackEvent) -> %% - case call_state_function(S, enter, PrevState, NextState, NewData) of - {Result, NewS} -> - loop_event_result( - Parent, Debug, NewS, - Events, Event, NextState, NewData, TransOpts, Result); - [Class,Reason,Stacktrace] -> + Module = P#params.module, + try Module:callback_mode() of + CallbackMode -> + loop_callback_mode_result( + P, Debug, S, + Q, State_Data, CallbackEvent, + CallbackMode, listify(CallbackMode), undefined, false) + catch + CallbackMode -> + loop_callback_mode_result( + P, Debug, S, + Q, State_Data, CallbackEvent, + CallbackMode, listify(CallbackMode), undefined, false); + Class:Reason:Stacktrace -> terminate( - Class, Reason, Stacktrace, Debug, S, [Event|Events]) + Class, Reason, Stacktrace, P, Debug, S, Q) + end; +loop_callback_mode(P, Debug, S, Q, State_Data, CallbackEvent) -> + loop_state_callback(P, Debug, S, Q, State_Data, CallbackEvent). + +%% Check the result of Module:callback_mode() +%% +loop_callback_mode_result( + P, Debug, S, Q, State_Data, CallbackEvent, + CallbackMode, [H|T], NewCallbackMode, NewStateEnter) -> + %% + case callback_mode(H) of + true -> + loop_callback_mode_result( + P, Debug, S, Q, State_Data, CallbackEvent, + CallbackMode, T, H, NewStateEnter); + false -> + case state_enter(H) of + true -> + loop_callback_mode_result( + P, Debug, S, Q, State_Data, CallbackEvent, + CallbackMode, T, NewCallbackMode, true); + false -> + terminate( + error, + {bad_return_from_callback_mode,CallbackMode}, + ?STACKTRACE(), + P, Debug, S, Q) + end + end; +loop_callback_mode_result( + P, Debug, S, Q, State_Data, CallbackEvent, + CallbackMode, [], NewCallbackMode, NewStateEnter) -> + %% + case NewCallbackMode of + undefined -> + terminate( + error, + {bad_return_from_callback_mode,CallbackMode}, + ?STACKTRACE(), + P, Debug, S, Q); + _ -> + P_1 = + P#params{ + callback_mode = NewCallbackMode, + state_enter = NewStateEnter}, + loop_state_callback( + P_1, Debug, S, Q, State_Data, CallbackEvent) end. -%% Process the result from the state function. -%% When TransOpts =:= false it was a state function call, -%% otherwise it is an option tuple and it was a state enter call. + +%% Make a state enter call to the state function, we loop back here +%% from further down if state enter calls are enabled +%% +loop_state_enter( + P, Debug, #state{state_data = {PrevState,_PrevData}} = S, + Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone) -> + %% + StateCall = false, + CallbackEvent = {enter,PrevState}, + loop_state_callback( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + StateCall, CallbackEvent). + +%% Make a state call (not state enter call) to the state function +%% +loop_state_callback(P, Debug, S, Q, State_Data, CallbackEvent) -> + NextEventsR = [], + Hibernate = false, + TimeoutsR = [], + Postpone = false, + StateCall = true, + loop_state_callback( + P, Debug, S, Q, State_Data, + NextEventsR, Hibernate, TimeoutsR, Postpone, + StateCall, CallbackEvent). +%% +loop_state_callback( + #params{callback_mode = CallbackMode, module = Module} = P, + Debug, S, Q, {State,Data} = State_Data, + NextEventsR, Hibernate, TimeoutsR, Postpone, + StateCall, {Type,Content}) -> + try + case CallbackMode of + state_functions -> + Module:State(Type, Content, Data); + handle_event_function -> + Module:handle_event(Type, Content, State, Data) + end + of + Result -> + loop_state_callback_result( + P, Debug, S, Q, State_Data, + NextEventsR, Hibernate, TimeoutsR, Postpone, + StateCall, Result) + catch + Result -> + loop_state_callback_result( + P, Debug, S, Q, State_Data, + NextEventsR, Hibernate, TimeoutsR, Postpone, + StateCall, Result); + Class:Reason:Stacktrace -> + terminate(Class, Reason, Stacktrace, P, Debug, S, Q) + end; +loop_state_callback( + P, Debug, S, Q, State_Data, + NextEventsR, Hibernate, TimeoutsR, Postpone, + StateCall, Actions) when is_list(Actions) -> + %% Tunneled actions from enter/8 + CallEnter = true, + loop_actions_list( + P, Debug, S, Q, State_Data, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions). + +%% Process the result from the state function %% -loop_event_result( - Parent, Debug, S, - Events, Event, State, Data, TransOpts, Result) -> +loop_state_callback_result( + P, Debug, S, Q, {State,_Data} = State_Data, + NextEventsR, Hibernate, TimeoutsR, Postpone, + StateCall, Result) -> %% case Result of {next_state,State,NewData} -> - loop_event_actions( - Parent, Debug, S, - Events, Event, State, NewData, TransOpts, - [], false); + loop_actions( + P, Debug, S, Q, {State,NewData}, + NextEventsR, Hibernate, TimeoutsR, Postpone, + false); {next_state,NextState,NewData} - when TransOpts =:= false -> - loop_event_actions( - Parent, Debug, S, - Events, Event, NextState, NewData, TransOpts, - [], true); + when StateCall -> + loop_actions( + P, Debug, S, Q, {NextState,NewData}, + NextEventsR, Hibernate, TimeoutsR, Postpone, + true); {next_state,_NextState,_NewData} -> terminate( error, {bad_state_enter_return_from_state_function,Result}, - ?STACKTRACE(), Debug, + ?STACKTRACE(), P, Debug, S#state{ - state = State, data = Data, - hibernate = hibernate_in_trans_opts(TransOpts)}, - [Event|Events]); + state_data = State_Data, + hibernate = Hibernate}, + Q); {next_state,State,NewData,Actions} -> - loop_event_actions( - Parent, Debug, S, - Events, Event, State, NewData, TransOpts, - Actions, false); + loop_actions( + P, Debug, S, Q, {State,NewData}, + NextEventsR, Hibernate, TimeoutsR, Postpone, + false, StateCall, Actions); {next_state,NextState,NewData,Actions} - when TransOpts =:= false -> - loop_event_actions( - Parent, Debug, S, - Events, Event, NextState, NewData, TransOpts, - Actions, true); + when StateCall -> + loop_actions( + P, Debug, S, Q, {NextState,NewData}, + NextEventsR, Hibernate, TimeoutsR, Postpone, + true, StateCall, Actions); {next_state,_NextState,_NewData,_Actions} -> terminate( error, {bad_state_enter_return_from_state_function,Result}, - ?STACKTRACE(), Debug, + ?STACKTRACE(), P, Debug, S#state{ - state = State, data = Data, - hibernate = hibernate_in_trans_opts(TransOpts)}, - [Event|Events]); + state_data = State_Data, + hibernate = Hibernate}, + Q); %% {keep_state,NewData} -> - loop_event_actions( - Parent, Debug, S, - Events, Event, State, NewData, TransOpts, - [], false); + loop_actions( + P, Debug, S, Q, {State,NewData}, + NextEventsR, Hibernate, TimeoutsR, Postpone, + false); {keep_state,NewData,Actions} -> - loop_event_actions( - Parent, Debug, S, - Events, Event, State, NewData, TransOpts, - Actions, false); + loop_actions( + P, Debug, S, Q, {State,NewData}, + NextEventsR, Hibernate, TimeoutsR, Postpone, + false, StateCall, Actions); %% keep_state_and_data -> - loop_event_actions( - Parent, Debug, S, - Events, Event, State, Data, TransOpts, - [], false); + loop_actions( + P, Debug, S, Q, State_Data, + NextEventsR, Hibernate, TimeoutsR, Postpone, + false); {keep_state_and_data,Actions} -> - loop_event_actions( - Parent, Debug, S, - Events, Event, State, Data, TransOpts, - Actions, false); + loop_actions( + P, Debug, S, Q, State_Data, + NextEventsR, Hibernate, TimeoutsR, Postpone, + false, StateCall, Actions); %% {repeat_state,NewData} -> - loop_event_actions( - Parent, Debug, S, - Events, Event, State, NewData, TransOpts, - [], true); + loop_actions( + P, Debug, S, Q, {State,NewData}, + NextEventsR, Hibernate, TimeoutsR, Postpone, + true); {repeat_state,NewData,Actions} -> - loop_event_actions( - Parent, Debug, S, - Events, Event, State, NewData, TransOpts, - Actions, true); + loop_actions( + P, Debug, S, Q, {State,NewData}, + NextEventsR, Hibernate, TimeoutsR, Postpone, + true, StateCall, Actions); %% repeat_state_and_data -> - loop_event_actions( - Parent, Debug, S, - Events, Event, State, Data, TransOpts, - [], true); + loop_actions( + P, Debug, S, Q, State_Data, + NextEventsR, Hibernate, TimeoutsR, Postpone, + true); {repeat_state_and_data,Actions} -> - loop_event_actions( - Parent, Debug, S, - Events, Event, State, Data, TransOpts, - Actions, true); + loop_actions( + P, Debug, S, Q, State_Data, + NextEventsR, Hibernate, TimeoutsR, Postpone, + true, StateCall, Actions); %% stop -> terminate( - exit, normal, ?STACKTRACE(), Debug, + exit, normal, ?STACKTRACE(), P, Debug, S#state{ - state = State, data = Data, - hibernate = hibernate_in_trans_opts(TransOpts)}, - [Event|Events]); + state_data = State_Data, + hibernate = Hibernate}, + Q); {stop,Reason} -> terminate( - exit, Reason, ?STACKTRACE(), Debug, + exit, Reason, ?STACKTRACE(), P, Debug, S#state{ - state = State, data = Data, - hibernate = hibernate_in_trans_opts(TransOpts)}, - [Event|Events]); + state_data = State_Data, + hibernate = Hibernate}, + Q); {stop,Reason,NewData} -> terminate( - exit, Reason, ?STACKTRACE(), Debug, + exit, Reason, ?STACKTRACE(), P, Debug, S#state{ - state = State, data = NewData, - hibernate = hibernate_in_trans_opts(TransOpts)}, - [Event|Events]); + state_data = {State,NewData}, + hibernate = Hibernate}, + Q); %% {stop_and_reply,Reason,Replies} -> reply_then_terminate( - exit, Reason, ?STACKTRACE(), Debug, + exit, Reason, ?STACKTRACE(), P, Debug, S#state{ - state = State, data = Data, - hibernate = hibernate_in_trans_opts(TransOpts)}, - [Event|Events], Replies); + state_data = State_Data, + hibernate = Hibernate}, + Q, Replies); {stop_and_reply,Reason,Replies,NewData} -> reply_then_terminate( - exit, Reason, ?STACKTRACE(), Debug, + exit, Reason, ?STACKTRACE(), P, Debug, S#state{ - state = State, data = NewData, - hibernate = hibernate_in_trans_opts(TransOpts)}, - [Event|Events], Replies); + state_data = {State,NewData}, + hibernate = Hibernate}, + Q, Replies); %% _ -> terminate( error, {bad_return_from_state_function,Result}, - ?STACKTRACE(), Debug, + ?STACKTRACE(), P, Debug, S#state{ - state = State, data = Data, - hibernate = hibernate_in_trans_opts(TransOpts)}, - [Event|Events]) + state_data = State_Data, + hibernate = Hibernate}, + Q) end. %% Ensure that Actions are a list -loop_event_actions( - Parent, Debug, S, - Events, Event, NextState, NewerData, TransOpts, - Actions, CallEnter) -> - loop_event_actions_list( - Parent, Debug, S, - Events, Event, NextState, NewerData, TransOpts, - listify(Actions), CallEnter). - -%% Process actions from the state function -loop_event_actions_list( - Parent, Debug, #state{state_enter = StateEnter} = S, - Events, Event, NextState, NewerData, TransOpts, - Actions, CallEnter) -> +%% +loop_actions( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, _StateCall, []) -> + loop_actions( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter); +loop_actions( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions) -> %% - case parse_actions(TransOpts, Debug, S, Actions) of - {NewDebug,NewTransOpts} - when StateEnter, CallEnter -> - loop_event_state_enter( - Parent, NewDebug, S, - Events, Event, NextState, NewerData, NewTransOpts); - {NewDebug,NewTransOpts} -> - loop_event_done( - Parent, NewDebug, S, - Events, Event, NextState, NewerData, NewTransOpts); - [Class,Reason,Stacktrace,NewDebug] -> - terminate( - Class, Reason, Stacktrace, NewDebug, - S#state{ - state = NextState, - data = NewerData, - hibernate = hibernate_in_trans_opts(TransOpts)}, - [Event|Events]) + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, listify(Actions)). +%% +%% Shortcut for no actions +-compile({inline, [loop_actions/10]}). +loop_actions( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter) -> + %% + %% Shortcut for no actions + case CallEnter andalso P#params.state_enter of + true -> + loop_state_enter( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone); + false -> + loop_state_transition( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone) end. --compile({inline, [hibernate_in_trans_opts/1]}). -hibernate_in_trans_opts(false) -> - (#trans_opts{})#trans_opts.hibernate; -hibernate_in_trans_opts(#trans_opts{hibernate = Hibernate}) -> - Hibernate. - -parse_actions(false, Debug, S, Actions) -> - parse_actions(true, Debug, S, Actions, #trans_opts{}); -parse_actions(TransOpts, Debug, S, Actions) -> - parse_actions(false, Debug, S, Actions, TransOpts). +%% Process the returned actions %% -parse_actions(_StateCall, Debug, _S, [], TransOpts) -> - {Debug,TransOpts}; -parse_actions(StateCall, Debug, S, [Action|Actions], TransOpts) -> +loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, _StateCall, []) -> + %% + case P#params.state_enter of + true when CallEnter -> + loop_state_enter( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone); + _ -> + loop_state_transition( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone) + end; +loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, [Action|Actions]) -> + %% case Action of %% Actual actions {reply,From,Reply} -> - parse_actions_reply( - StateCall, Debug, S, Actions, TransOpts, From, Reply); + loop_actions_reply( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions, + From, Reply); %% %% Actions that set options - {hibernate,NewHibernate} when is_boolean(NewHibernate) -> - parse_actions( - StateCall, Debug, S, Actions, - TransOpts#trans_opts{hibernate = NewHibernate}); + {hibernate,Hibernate_1} when is_boolean(Hibernate_1) -> + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate_1, TimeoutsR, Postpone, + CallEnter, StateCall, Actions); hibernate -> - parse_actions( - StateCall, Debug, S, Actions, - TransOpts#trans_opts{hibernate = true}); + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, true, TimeoutsR, Postpone, + CallEnter, StateCall, Actions); %% - {postpone,NewPostpone} when not NewPostpone orelse StateCall -> - parse_actions( - StateCall, Debug, S, Actions, - TransOpts#trans_opts{postpone = NewPostpone}); + {postpone,Postpone_1} when not Postpone_1 orelse StateCall -> + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone_1, + CallEnter, StateCall, Actions); postpone when StateCall -> - parse_actions( - StateCall, Debug, S, Actions, - TransOpts#trans_opts{postpone = true}); + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, true, + CallEnter, StateCall, Actions); postpone -> - [error, - {bad_state_enter_action_from_state_function,Action}, - ?STACKTRACE(), - Debug]; + terminate( + error, + {bad_state_enter_action_from_state_function,Action}, + ?STACKTRACE(), P, Debug, + S#state{ + state_data = NextState_NewData, + hibernate = Hibernate}, + Q); %% {next_event,Type,Content} -> - parse_actions_next_event( - StateCall, Debug, S, Actions, TransOpts, Type, Content); + loop_actions_next_event( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions, Type, Content); %% - _ -> - parse_actions_timeout( - StateCall, Debug, S, Actions, TransOpts, Action) + Timeout -> + loop_actions_timeout( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions, Timeout) end. -parse_actions_reply( - StateCall, ?not_sys_debug, S, Actions, TransOpts, +%% Process a reply action +%% +loop_actions_reply( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions, From, Reply) -> %% case from(From) of true -> + %% No need for a separate ?not_sys_debug clause here + %% since the external call to erlang:'!'/2 in reply/2 + %% will cause swap out of all live registers anyway reply(From, Reply), - parse_actions(StateCall, ?not_sys_debug, S, Actions, TransOpts); - false -> - [error, - {bad_action_from_state_function,{reply,From,Reply}}, - ?STACKTRACE(), - ?not_sys_debug] - end; -parse_actions_reply( - StateCall, Debug, #state{name = Name, state = State} = S, - Actions, TransOpts, From, Reply) -> - %% - case from(From) of - true -> - reply(From, Reply), - NewDebug = sys_debug(Debug, {Name,State}, {out,Reply,From}), - parse_actions(StateCall, NewDebug, S, Actions, TransOpts); + Debug_1 = ?sys_debug(Debug, P#params.name, {out,Reply,From}), + loop_actions_list( + P, Debug_1, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions); false -> - [error, - {bad_action_from_state_function,{reply,From,Reply}}, - ?STACKTRACE(), - Debug] + terminate( + error, + {bad_action_from_state_function,{reply,From,Reply}}, + ?STACKTRACE(), P, Debug, + S#state{ + state_data = NextState_NewData, + hibernate = Hibernate}, + Q) end. -parse_actions_next_event( - StateCall, ?not_sys_debug, S, - Actions, TransOpts, Type, Content) -> - case event_type(Type) of - true when StateCall -> - NextEventsR = TransOpts#trans_opts.next_events_r, - parse_actions( - StateCall, ?not_sys_debug, S, Actions, - TransOpts#trans_opts{ - next_events_r = [{Type,Content}|NextEventsR]}); - _ -> - [error, - {bad_state_enter_action_from_state_function, - {next_event,Type,Content}}, - ?STACKTRACE(), - ?not_sys_debug] - end; -parse_actions_next_event( - StateCall, Debug, #state{name = Name, state = State} = S, - Actions, TransOpts, Type, Content) -> +%% Process a next_event action +%% +loop_actions_next_event( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions, Type, Content) -> case event_type(Type) of true when StateCall -> - NewDebug = sys_debug(Debug, {Name,State}, {in,{Type,Content}}), - NextEventsR = TransOpts#trans_opts.next_events_r, - parse_actions( - StateCall, NewDebug, S, Actions, - TransOpts#trans_opts{ - next_events_r = [{Type,Content}|NextEventsR]}); + NextEvent = {Type,Content}, + case Debug of + ?not_sys_debug -> + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + [NextEvent|NextEventsR], + Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions); + _ -> + Name = P#params.name, + {State,_Data} = S#state.state_data, + Debug_1 = + sys_debug(Debug, Name, {in,{Type,Content},State}), + loop_actions_list( + P, Debug_1, S, Q, NextState_NewData, + [NextEvent|NextEventsR], + Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions) + end; _ -> - [error, - {bad_state_enter_action_from_state_function, - {next_event,Type,Content}}, - ?STACKTRACE(), - Debug] + terminate( + error, + {if + StateCall -> + bad_action_from_state_function; + true -> + bad_state_enter_action_from_state_function + end, + {next_event,Type,Content}}, + ?STACKTRACE(), P, Debug, + S#state{ + state_data = NextState_NewData, + hibernate = Hibernate}, + Q) end. -parse_actions_timeout( - StateCall, Debug, S, Actions, TransOpts, - {TimeoutType,Time,TimerMsg,TimerOpts} = AbsoluteTimeout) -> +%% Process a timeout action, or also any unrecognized action +%% +loop_actions_timeout( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions, + {TimeoutType,Time,TimeoutMsg,TimeoutOpts} = Timeout) -> %% - case classify_timeout(TimeoutType, Time, listify(TimerOpts)) of - absolute -> - parse_actions_timeout_add( - StateCall, Debug, S, Actions, - TransOpts, AbsoluteTimeout); - relative -> - RelativeTimeout = {TimeoutType,Time,TimerMsg}, - parse_actions_timeout_add( - StateCall, Debug, S, Actions, - TransOpts, RelativeTimeout); - badarg -> - [error, - {bad_action_from_state_function,AbsoluteTimeout}, - ?STACKTRACE(), - Debug] + case timeout_event_type(TimeoutType) of + true -> + case listify(TimeoutOpts) of + %% Optimization cases + [] when ?relative_timeout(Time) -> + RelativeTimeout = {TimeoutType,Time,TimeoutMsg}, + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, + [RelativeTimeout|TimeoutsR], Postpone, + CallEnter, StateCall, Actions); + [{abs,true}] when ?absolute_timeout(Time) -> + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, + [Timeout|TimeoutsR], Postpone, + CallEnter, StateCall, Actions); + [{abs,false}] when ?relative_timeout(Time) -> + RelativeTimeout = {TimeoutType,Time,TimeoutMsg}, + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, + [RelativeTimeout|TimeoutsR], Postpone, + CallEnter, StateCall, Actions); + %% Generic case + TimeoutOptsList -> + case parse_timeout_opts_abs(TimeoutOptsList) of + true when ?absolute_timeout(Time) -> + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, + [Timeout|TimeoutsR], Postpone, + CallEnter, StateCall, Actions); + false when ?relative_timeout(Time) -> + RelativeTimeout = + {TimeoutType,Time,TimeoutMsg}, + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, + [RelativeTimeout|TimeoutsR], Postpone, + CallEnter, StateCall, Actions); + badarg -> + terminate( + error, + {bad_action_from_state_function,Timeout}, + ?STACKTRACE(), P, Debug, + S#state{ + state_data = NextState_NewData, + hibernate = Hibernate}, + Q) + end + end; + false -> + terminate( + error, + {bad_action_from_state_function,Timeout}, + ?STACKTRACE(), P, Debug, + S#state{ + state_data = NextState_NewData, + hibernate = Hibernate}, + Q) end; -parse_actions_timeout( - StateCall, Debug, S, Actions, TransOpts, - {TimeoutType,Time,_} = RelativeTimeout) -> - case classify_timeout(TimeoutType, Time, []) of - relative -> - parse_actions_timeout_add( - StateCall, Debug, S, Actions, - TransOpts, RelativeTimeout); - badarg -> - [error, - {bad_action_from_state_function,RelativeTimeout}, - ?STACKTRACE(), - Debug] +loop_actions_timeout( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions, + {TimeoutType,Time,_} = Timeout) -> + %% + case timeout_event_type(TimeoutType) of + true when ?relative_timeout(Time) -> + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, + [Timeout|TimeoutsR], Postpone, + CallEnter, StateCall, Actions); + _ -> + terminate( + error, + {bad_action_from_state_function,Timeout}, + ?STACKTRACE(), P, Debug, + S#state{ + state_data = NextState_NewData, + hibernate = Hibernate}, + Q) end; -parse_actions_timeout( - StateCall, Debug, S, Actions, TransOpts, - Time) -> - case classify_timeout(timeout, Time, []) of - relative -> +loop_actions_timeout( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone, + CallEnter, StateCall, Actions, Time) -> + %% + if + ?relative_timeout(Time) -> RelativeTimeout = {timeout,Time,Time}, - parse_actions_timeout_add( - StateCall, Debug, S, Actions, - TransOpts, RelativeTimeout); - badarg -> - [error, - {bad_action_from_state_function,Time}, - ?STACKTRACE(), - Debug] + loop_actions_list( + P, Debug, S, Q, NextState_NewData, + NextEventsR, Hibernate, + [RelativeTimeout|TimeoutsR], Postpone, + CallEnter, StateCall, Actions); + true -> + terminate( + error, + {bad_action_from_state_function,Time}, + ?STACKTRACE(), P, Debug, + S#state{ + state_data = NextState_NewData, + hibernate = Hibernate}, + Q) end. -parse_actions_timeout_add( - StateCall, Debug, S, Actions, - #trans_opts{timeouts_r = TimeoutsR} = TransOpts, Timeout) -> - parse_actions( - StateCall, Debug, S, Actions, - TransOpts#trans_opts{timeouts_r = [Timeout|TimeoutsR]}). - %% Do the state transition -loop_event_done( - Parent, ?not_sys_debug, - #state{postponed = P} = S, - Events, Event, NextState, NewData, - #trans_opts{ - postpone = Postpone, hibernate = Hibernate, - timeouts_r = [], next_events_r = []}) -> - %% - %% Optimize the simple cases - %% i.e no timer changes, no inserted events and no debug, - %% by duplicate stripped down code - %% - %% Fast path - %% - case Postpone of - true -> - loop_event_done_fast( - Parent, Hibernate, - S, - Events, [Event|P], NextState, NewData); - false -> - loop_event_done_fast( - Parent, Hibernate, - S, - Events, P, NextState, NewData) - end; -loop_event_done( - Parent, Debug_0, - #state{ - state = State, postponed = P_0, timers = Timers_0} = S, - Events_0, Event_0, NextState, NewData, - #trans_opts{ - hibernate = Hibernate, timeouts_r = TimeoutsR, - postpone = Postpone, next_events_r = NextEventsR}) -> +%% +loop_state_transition( + P, Debug, #state{state_data = {State,_Data}, postponed = Postponed} = S, + [Event|Events], {NextState,_NewData} = NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postpone) -> %% %% All options have been collected and next_events are buffered. %% Do the actual state transition. %% - %% Full feature path - %% - [Debug_1|P_1] = % Move current event to postponed if Postpone + Postponed_1 = % Move current event to postponed if Postpone case Postpone of true -> - [?sys_debug( - Debug_0, - {S#state.name,State}, - {postpone,Event_0,NextState}), - Event_0|P_0]; + [Event|Postponed]; false -> - [?sys_debug( - Debug_0, - {S#state.name,State}, - {consume,Event_0,NextState})|P_0] + Postponed end, - {Events_2,P_2,Timers_2} = - %% Move all postponed events to queue, - %% cancel the event timer, - %% and cancel the state timeout if the state changes - if - NextState =:= State -> - {Events_0,P_1, - cancel_timer_by_type(timeout, Timers_0)}; - true -> - {lists:reverse(P_1, Events_0), - [], - cancel_timer_by_type( - state_timeout, - cancel_timer_by_type(timeout, Timers_0))} - end, - {Timers_3,TimeoutEvents} = - %% Stop and start timers - parse_timers(Timers_2, TimeoutsR), - %% Place next events last in reversed queue - Events_3R = lists:reverse(Events_2, NextEventsR), - %% Enqueue immediate timeout events - Events_4R = prepend_timeout_events(TimeoutEvents, Events_3R), - loop_event_done( - Parent, Debug_1, - S#state{ - state = NextState, - data = NewData, - postponed = P_2, - timers = Timers_3, - hibernate = Hibernate}, - lists:reverse(Events_4R)). - -%% Fast path -%% -loop_event_done_fast( - Parent, Hibernate, - #state{ - state = NextState, - timers = {_,#{timeout := _}} = Timers} = S, - Events, P, NextState, NewData) -> + case Debug of + ?not_sys_debug -> + %% Optimization for no sys_debug + %% - avoid calling sys_debug/3 + if + NextState =:= State -> + loop_keep_state( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed_1); + true -> + loop_state_change( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed_1) + end; + _ -> + %% With sys_debug + Name = P#params.name, + Debug_1 = + case Postpone of + true -> + sys_debug( + Debug, Name, + {postpone,Event,State,NextState}); + false -> + sys_debug( + Debug, Name, + {consume,Event,State,NextState}) + end, + if + NextState =:= State -> + loop_keep_state( + P, Debug_1, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed_1); + true -> + loop_state_change( + P, Debug_1, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed_1) + end + end. + +%% State transition to the same state +%% +loop_keep_state( + P, Debug, #state{timers = {TimerRefs,TimeoutTypes} = Timers} = S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed) -> %% - %% Same state, event timeout active + %% Cancel event timeout %% - loop_event_done_fast( - Parent, Hibernate, S, - Events, P, NextState, NewData, - cancel_timer_by_type(timeout, Timers)); -loop_event_done_fast( - Parent, Hibernate, - #state{state = NextState} = S, - Events, P, NextState, NewData) -> + case TimeoutTypes of + %% Optimization + %% - only cancel timer when there is a timer to cancel + #{timeout := TimerRef} -> + %% Event timeout active + loop_next_events( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + cancel_timer_by_ref_and_type( + TimerRef, timeout, TimerRefs, TimeoutTypes)); + _ -> + %% No event timeout active + loop_next_events( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers) + end. + +%% State transition to a different state +%% +loop_state_change( + P, Debug, S, Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed) -> %% - %% Same state + %% Retry postponed events %% - loop_event_done( - Parent, ?not_sys_debug, - S#state{ - data = NewData, - postponed = P, - hibernate = Hibernate}, - Events); -loop_event_done_fast( - Parent, Hibernate, - #state{ - timers = {_,#{timeout := _}} = Timers} = S, - Events, P, NextState, NewData) -> + case Postponed of + [] -> + loop_state_change( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR); + [E1] -> + loop_state_change( + P, Debug, S, + [E1|Events], NextState_NewData, + NextEventsR, Hibernate, TimeoutsR); + [E2,E1] -> + loop_state_change( + P, Debug, S, + [E1,E2|Events], NextState_NewData, + NextEventsR, Hibernate, TimeoutsR); + _ -> + loop_state_change( + P, Debug, S, + lists:reverse(Postponed, Events), NextState_NewData, + NextEventsR, Hibernate, TimeoutsR) + end. +%% +loop_state_change( + P, Debug, #state{timers = {TimerRefs,TimeoutTypes} = Timers} = S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR) -> %% - %% State change, event timeout active + %% Cancel state and event timeout %% - loop_event_done_fast( - Parent, Hibernate, S, - lists:reverse(P, Events), [], NextState, NewData, - cancel_timer_by_type( - state_timeout, - cancel_timer_by_type(timeout, Timers))); -loop_event_done_fast( - Parent, Hibernate, - #state{ - timers = {_,#{state_timeout := _}} = Timers} = S, - Events, P, NextState, NewData) -> + case TimeoutTypes of + %% Optimization + %% - only cancel timeout when there is an active timeout + %% + #{state_timeout := TimerRef} -> + %% State timeout active + %% - cancel event timeout too since it is faster than inspecting + loop_next_events( + P, Debug, S, Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, [], + cancel_timer_by_type( + timeout, + cancel_timer_by_ref_and_type( + TimerRef, state_timeout, TimerRefs, TimeoutTypes))); + #{timeout := TimerRef} -> + %% Event timeout active but not state timeout + %% - cancel event timeout only + loop_next_events( + P, Debug, S, Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, [], + cancel_timer_by_ref_and_type( + TimerRef, timeout, TimerRefs, TimeoutTypes)); + _ -> + %% No state nor event timeout active. + loop_next_events( + P, Debug, S, Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, [], + Timers) + end. + +%% Continue state transition with processing of +%% inserted events and timeout events +%% +loop_next_events( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, [], Postponed, + Timers) -> %% - %% State change, state timeout active + %% Optimization when there are no timeout actions + %% hence no timeout zero events to append to Events + %% - avoid loop_timeouts + loop_done( + P, Debug, + S#state{ + state_data = NextState_NewData, + postponed = Postponed, + timers = Timers, + hibernate = Hibernate}, + NextEventsR, Events); +loop_next_events( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers) -> %% - loop_event_done_fast( - Parent, Hibernate, S, - lists:reverse(P, Events), [], NextState, NewData, - cancel_timer_by_type( - state_timeout, - cancel_timer_by_type(timeout, Timers))); -loop_event_done_fast( - Parent, Hibernate, - #state{} = S, - Events, P, NextState, NewData) -> + Seen = #{}, + TimeoutEvents = [], + loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers, Seen, TimeoutEvents). + +%% Continue state transition with processing of timeout events +%% +loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, [], Postponed, + Timers, _Seen, TimeoutEvents) -> %% - %% State change, no timeout to automatically cancel + S_1 = + S#state{ + state_data = NextState_NewData, + postponed = Postponed, + timers = Timers, + hibernate = Hibernate}, + case TimeoutEvents of + [] -> + loop_done(P, Debug, S_1, NextEventsR, Events); + _ -> + case Events of + [] -> + loop_prepend_timeout_events( + P, Debug, S_1, TimeoutEvents, + NextEventsR); + [E1] -> + loop_prepend_timeout_events( + P, Debug, S_1, TimeoutEvents, + [E1|NextEventsR]); + [E2,E1] -> + loop_prepend_timeout_events( + P, Debug, S_1, TimeoutEvents, + [E1,E2|NextEventsR]); + _ -> + loop_prepend_timeout_events( + P, Debug, S_1, TimeoutEvents, + lists:reverse(Events, NextEventsR)) + end + end; +loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, [Timeout|TimeoutsR], Postponed, + Timers, Seen, TimeoutEvents) -> %% - loop_event_done( - Parent, ?not_sys_debug, - S#state{ - state = NextState, - data = NewData, - postponed = [], - hibernate = Hibernate}, - lists:reverse(P, Events)). + case Timeout of + {TimeoutType,Time,TimeoutMsg} -> + %% Relative timeout + case Seen of + #{TimeoutType := _} -> + %% Type seen before - ignore + loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers, Seen, TimeoutEvents); + #{} -> + loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers, Seen, TimeoutEvents, + TimeoutType, Time, TimeoutMsg, []) + end; + {TimeoutType,Time,TimeoutMsg,TimeoutOpts} -> + %% Absolute timeout + case Seen of + #{TimeoutType := _} -> + %% Type seen before - ignore + loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers, Seen, TimeoutEvents); + #{} -> + loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers, Seen, TimeoutEvents, + TimeoutType, Time, TimeoutMsg, listify(TimeoutOpts)) + end + end. %% -%% Fast path +loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers, Seen, TimeoutEvents, + TimeoutType, Time, TimeoutMsg, TimeoutOpts) -> + %% + case Time of + infinity -> + %% Cancel any running timer + loop_timeouts_cancel( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers, Seen, TimeoutEvents, + TimeoutType); + 0 when TimeoutOpts =:= [] -> + %% Relative timeout zero + %% - cancel any running timer + %% handle timeout zero events later + %% + loop_timeouts_cancel( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers, Seen, [{TimeoutType,TimeoutMsg}|TimeoutEvents], + TimeoutType); + _ -> + %% (Re)start the timer + TimerRef = + erlang:start_timer(Time, self(), TimeoutMsg, TimeoutOpts), + {TimerRefs,TimeoutTypes} = Timers, + case TimeoutTypes of + #{TimeoutType := OldTimerRef} -> + %% Cancel the running timer, + %% update the timeout type, + %% insert the new timer ref, + %% and remove the old timer ref + Timers_1 = + {maps:remove( + OldTimerRef, + TimerRefs#{TimerRef => TimeoutType}), + TimeoutTypes#{TimeoutType := TimerRef}}, + cancel_timer(OldTimerRef), + loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers_1, Seen#{TimeoutType => true}, TimeoutEvents); + #{} -> + %% Insert the new timer type and ref + Timers_1 = + {TimerRefs#{TimerRef => TimeoutType}, + TimeoutTypes#{TimeoutType => TimerRef}}, + loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers_1, Seen#{TimeoutType => true}, TimeoutEvents) + end + end. + +%% Loop helper to cancel a timeout %% -loop_event_done_fast( - Parent, Hibernate, S, Events, P, NextState, NewData, Timers) -> +loop_timeouts_cancel( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + {TimerRefs,TimeoutTypes} = Timers, Seen, TimeoutEvents, + TimeoutType) -> + %% This function body should have been: + %% Timers_1 = cancel_timer_by_type(TimeoutType, Timers), + %% loop_timeouts( + %% P, Debug, S, + %% Events, NextState_NewData, + %% NextEventsR, Hibernate, TimeoutsR, Postponed, + %% Timers_1, Seen#{TimeoutType => true}, TimeoutEvents). %% - loop_event_done( - Parent, ?not_sys_debug, - S#state{ - state = NextState, - data = NewData, - postponed = P, - timers = Timers, - hibernate = Hibernate}, - Events). + %% Explicitly separate cases to get separate code paths for when + %% the map key exists vs. not, since otherwise the external call + %% to erlang:cancel_timer/1 and to map:remove/2 within + %% cancel_timer_by_type/2 would cause all live registers + %% to be saved to and restored from the stack also for + %% the case when the map key TimeoutType does not exist + case TimeoutTypes of + #{TimeoutType := TimerRef} -> + Timers_1 = + cancel_timer_by_ref_and_type( + TimerRef, TimeoutType, TimerRefs, TimeoutTypes), + loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers_1, Seen#{TimeoutType => true}, TimeoutEvents); + #{} -> + loop_timeouts( + P, Debug, S, + Events, NextState_NewData, + NextEventsR, Hibernate, TimeoutsR, Postponed, + Timers, Seen#{TimeoutType => true}, TimeoutEvents) + end. -loop_event_done(Parent, Debug, S, Q) -> +%% Continue state transition with prepending timeout zero events +%% before event queue reversal i.e appending timeout zero events +%% +loop_prepend_timeout_events(P, Debug, S, TimeoutEvents, EventsR) -> + {Debug_1,Events_1R} = + prepend_timeout_events(P, Debug, S, TimeoutEvents, EventsR), + loop_done(P, Debug_1, S, Events_1R, []). + +%% Place inserted events first in the event queue +%% +loop_done(P, Debug, S, NextEventsR, Events) -> + case NextEventsR of + [] -> + loop_done(P, Debug, S, Events); + [E1] -> + loop_done(P, Debug, S, [E1|Events]); + [E2,E1] -> + loop_done(P, Debug, S, [E1,E2|Events]); + _ -> + loop_done(P, Debug, S, lists:reverse(NextEventsR, Events)) + end. +%% +%% State transition is done, keep looping if there are +%% enqueued events, otherwise get a new event +%% +loop_done(P, Debug, S, Q) -> +%%% io:format( +%%% "loop_done: state_data = ~p,~n" +%%% " postponed = ~p, Q = ~p,~n", +%%% " timers = ~p.~n" +%%% [S#state.state_data,,S#state.postponed,Q,S#state.timers]), case Q of [] -> %% Get a new event - loop(Parent, Debug, S); - [{Type,Content}|Events] -> + loop(P, Debug, S); + [Event|Events] -> %% Loop until out of enqueued events - loop_event(Parent, Debug, S, Events, Type, Content) + loop_event(P, Debug, S, Event, Events) end. - %%--------------------------------------------------------------------------- %% Server loop helpers -call_callback_mode(#state{module = Module} = S) -> - try Module:callback_mode() of - CallbackMode -> - callback_mode_result(S, CallbackMode) - catch - CallbackMode -> - callback_mode_result(S, CallbackMode); - Class:Reason:Stacktrace -> - [Class,Reason,Stacktrace] - end. - -callback_mode_result(S, CallbackMode) -> - callback_mode_result( - S, CallbackMode, listify(CallbackMode), undefined, false). -%% -callback_mode_result(_S, CallbackMode, [], undefined, _StateEnter) -> - [error, - {bad_return_from_callback_mode,CallbackMode}, - ?STACKTRACE()]; -callback_mode_result(S, _CallbackMode, [], CBMode, StateEnter) -> - S#state{callback_mode = CBMode, state_enter = StateEnter}; -callback_mode_result(S, CallbackMode, [H|T], CBMode, StateEnter) -> - case callback_mode(H) of - true -> - callback_mode_result(S, CallbackMode, T, H, StateEnter); - false -> - case state_enter(H) of - true -> - callback_mode_result(S, CallbackMode, T, CBMode, true); - false -> - [error, - {bad_return_from_callback_mode,CallbackMode}, - ?STACKTRACE()] - end - end. - - -call_state_function( - #state{callback_mode = undefined} = S, Type, Content, State, Data) -> - case call_callback_mode(S) of - #state{} = NewS -> - call_state_function(NewS, Type, Content, State, Data); - Error -> - Error - end; -call_state_function( - #state{callback_mode = CallbackMode, module = Module} = S, - Type, Content, State, Data) -> - try - case CallbackMode of - state_functions -> - Module:State(Type, Content, Data); - handle_event_function -> - Module:handle_event(Type, Content, State, Data) - end - of - Result -> - {Result,S} - catch - Result -> - {Result,S}; - Class:Reason:Stacktrace -> - [Class,Reason,Stacktrace] - end. - - -%% -> absolute | relative | badarg -classify_timeout(TimeoutType, Time, Opts) -> - case timeout_event_type(TimeoutType) of - true -> - classify_time(false, Time, Opts); - false -> - badarg - end. - -classify_time(Abs, Time, []) -> - case Abs of - true when - is_integer(Time); - Time =:= infinity -> - absolute; - false when - is_integer(Time), 0 =< Time; - Time =:= infinity -> - relative; - _ -> - badarg - end; -classify_time(_, Time, [{abs,Abs}|Opts]) when is_boolean(Abs) -> - classify_time(Abs, Time, Opts); -classify_time(_, _, Opts) when is_list(Opts) -> - badarg. - -%% Stop and start timers as well as create timeout zero events -%% and pending event timer +%% Parse an option list for erlang:start_timer/4 to figure out +%% if the timeout will be absolute or relative %% -%% Stop and start timers non-event timers -parse_timers(Timers, TimeoutsR) -> - parse_timers(Timers, TimeoutsR, #{}, []). +parse_timeout_opts_abs(Opts) -> + parse_timeout_opts_abs(Opts, false). %% -parse_timers(Timers, [], _Seen, TimeoutEvents) -> - %% - {Timers,TimeoutEvents}; -parse_timers( - Timers, [Timeout|TimeoutsR], Seen, TimeoutEvents) -> - %% - case Timeout of - {TimerType,Time,TimerMsg,TimerOpts} -> - %% Absolute timer - parse_timers( - Timers, TimeoutsR, Seen, TimeoutEvents, - TimerType, Time, TimerMsg, listify(TimerOpts)); - %% Relative timers below - {TimerType,0,TimerMsg} -> - parse_timers( - Timers, TimeoutsR, Seen, TimeoutEvents, - TimerType, zero, TimerMsg, []); - {TimerType,Time,TimerMsg} -> - parse_timers( - Timers, TimeoutsR, Seen, TimeoutEvents, - TimerType, Time, TimerMsg, []) - end. - -parse_timers( - Timers, TimeoutsR, Seen, TimeoutEvents, - TimerType, Time, TimerMsg, TimerOpts) -> - case Seen of - #{TimerType := _} -> - %% Type seen before - ignore - parse_timers( - Timers, TimeoutsR, Seen, TimeoutEvents); - #{} -> - %% Unseen type - handle - NewSeen = Seen#{TimerType => true}, - case Time of - infinity -> - %% Cancel any running timer - parse_timers( - cancel_timer_by_type(TimerType, Timers), - TimeoutsR, NewSeen, TimeoutEvents); - zero -> - %% Cancel any running timer - %% Handle zero time timeouts later - parse_timers( - cancel_timer_by_type(TimerType, Timers), - TimeoutsR, NewSeen, - [{TimerType,TimerMsg}|TimeoutEvents]); - _ -> - %% (Re)start the timer - TimerRef = - erlang:start_timer( - Time, self(), TimerMsg, TimerOpts), - {TimerRefs,TimerTypes} = Timers, - case TimerTypes of - #{TimerType := OldTimerRef} -> - %% Cancel the running timer, - %% update the timeout type, - %% insert the new timer ref, - %% and remove the old timer ref - cancel_timer(OldTimerRef), - %% Insert the new timer into - %% both TimerRefs and TimerTypes - parse_timers( - {maps:remove( - OldTimerRef, - TimerRefs#{TimerRef => TimerType}), - TimerTypes#{TimerType := TimerRef}}, - TimeoutsR, NewSeen, TimeoutEvents); - #{} -> - %% Insert the new timer type and ref - parse_timers( - {TimerRefs#{TimerRef => TimerType}, - TimerTypes#{TimerType => TimerRef}}, - TimeoutsR, NewSeen, TimeoutEvents) - end - end +parse_timeout_opts_abs(Opts, Abs) -> + case Opts of + [] -> + Abs; + [{abs,Abs_1}|Opts] when is_boolean(Abs_1) -> + parse_timeout_opts_abs(Opts, Abs_1); + _ -> + badarg end. %% Enqueue immediate timeout events (timeout 0 events) @@ -1717,62 +2061,94 @@ parse_timers( %% so if there are enqueued events before the event timer %% timeout 0 event - the event timer is cancelled hence no event. %% -%% Other (state_timeout) timeout 0 events that are after -%% the event timer timeout 0 events are considered to +%% Other (state_timeout and {timeout,Name}) timeout 0 events +%% that are after an event timer timeout 0 event are considered to %% belong to timers that were started after the event timer %% timeout 0 event fired, so they do not cancel the event timer. %% -prepend_timeout_events([], EventsR) -> - EventsR; -prepend_timeout_events([{timeout,_} = TimeoutEvent|TimeoutEvents], []) -> - prepend_timeout_events(TimeoutEvents, [TimeoutEvent]); -prepend_timeout_events([{timeout,_}|TimeoutEvents], EventsR) -> +prepend_timeout_events(_P, Debug, _S, [], EventsR) -> + {Debug,EventsR}; +prepend_timeout_events( + P, Debug, S, [{timeout,_} = TimeoutEvent|TimeoutEvents], []) -> + %% Prepend this since there are no other events in queue + case Debug of + ?not_sys_debug -> + prepend_timeout_events( + P, Debug, S, TimeoutEvents, [TimeoutEvent]); + _ -> + {State,_Data} = S#state.state_data, + Debug_1 = + sys_debug( + Debug, P#params.name, {in,TimeoutEvent,State}), + prepend_timeout_events( + P, Debug_1, S, TimeoutEvents, [TimeoutEvent]) + end; +prepend_timeout_events( + P, Debug, S, [{timeout,_}|TimeoutEvents], EventsR) -> %% Ignore since there are other events in queue %% so they have cancelled the event timeout 0. - prepend_timeout_events(TimeoutEvents, EventsR); -prepend_timeout_events([TimeoutEvent|TimeoutEvents], EventsR) -> + prepend_timeout_events(P, Debug, S, TimeoutEvents, EventsR); +prepend_timeout_events( + P, Debug, S, [TimeoutEvent|TimeoutEvents], EventsR) -> %% Just prepend all others - prepend_timeout_events(TimeoutEvents, [TimeoutEvent|EventsR]). + case Debug of + ?not_sys_debug -> + prepend_timeout_events( + P, Debug, S, TimeoutEvents, [TimeoutEvent|EventsR]); + _ -> + {State,_Data} = S#state.state_data, + Debug_1 = + sys_debug( + Debug, P#params.name, {in,TimeoutEvent,State}), + prepend_timeout_events( + P, Debug_1, S, TimeoutEvents, [TimeoutEvent|EventsR]) + end. %%--------------------------------------------------------------------------- %% Server helpers -reply_then_terminate(Class, Reason, Stacktrace, Debug, S, Q, Replies) -> +reply_then_terminate(Class, Reason, Stacktrace, P, Debug, S, Q, Replies) -> do_reply_then_terminate( - Class, Reason, Stacktrace, Debug, S, Q, listify(Replies)). + Class, Reason, Stacktrace, P, Debug, S, Q, listify(Replies)). %% do_reply_then_terminate( - Class, Reason, Stacktrace, Debug, S, Q, []) -> - terminate(Class, Reason, Stacktrace, Debug, S, Q); + Class, Reason, Stacktrace, P, Debug, S, Q, []) -> + terminate(Class, Reason, Stacktrace, P, Debug, S, Q); do_reply_then_terminate( - Class, Reason, Stacktrace, Debug, S, Q, [R|Rs]) -> + Class, Reason, Stacktrace, P, Debug, S, Q, [R|Rs]) -> case R of - {reply,{_To,_Tag}=From,Reply} -> - reply(From, Reply), - NewDebug = - ?sys_debug( - Debug, - begin - #state{name = Name, state = State} = S, - {Name,State} - end, - {out,Reply,From}), - do_reply_then_terminate( - Class, Reason, Stacktrace, NewDebug, S, Q, Rs); + {reply,From,Reply} -> + case from(From) of + true -> + reply(From, Reply), + Debug_1 = + ?sys_debug( + Debug, + P#params.name, + {out,Reply,From}), + do_reply_then_terminate( + Class, Reason, Stacktrace, P, Debug_1, S, Q, Rs); + false -> + terminate( + error, + {bad_reply_action_from_state_function,R}, + ?STACKTRACE(), + P, Debug, S, Q) + end; _ -> terminate( error, {bad_reply_action_from_state_function,R}, ?STACKTRACE(), - Debug, S, Q) + P, Debug, S, Q) end. terminate( - Class, Reason, Stacktrace, Debug, - #state{module = Module, state = State, data = Data} = S, - Q) -> + Class, Reason, Stacktrace, + #params{module = Module} = P, Debug, + #state{state_data = {State,Data}} = S, Q) -> case erlang:function_exported(Module, terminate, 3) of true -> try Module:terminate(Reason, State, Data) of @@ -1780,8 +2156,7 @@ terminate( catch _ -> ok; C:R:ST -> - error_info(C, R, ST, S, Q), - sys:print_log(Debug), + error_info(C, R, ST, Debug, P, S, Q), erlang:raise(C, R, ST) end; false -> @@ -1790,14 +2165,13 @@ terminate( _ = case Reason of normal -> - terminate_sys_debug(Debug, S, State, Reason); + terminate_sys_debug(Debug, P, State, Reason); shutdown -> - terminate_sys_debug(Debug, S, State, Reason); + terminate_sys_debug(Debug, P, State, Reason); {shutdown,_} -> - terminate_sys_debug(Debug, S, State, Reason); + terminate_sys_debug(Debug, P, State, Reason); _ -> - error_info(Class, Reason, Stacktrace, S, Q), - sys:print_log(Debug) + error_info(Class, Reason, Stacktrace, Debug, P, S, Q) end, case Stacktrace of [] -> @@ -1806,38 +2180,67 @@ terminate( erlang:raise(Class, Reason, Stacktrace) end. -terminate_sys_debug(Debug, S, State, Reason) -> - ?sys_debug(Debug, {S#state.name,State}, {terminate,Reason}). +terminate_sys_debug(Debug, P, State, Reason) -> + ?sys_debug(Debug, P#params.name, {terminate,Reason,State}). error_info( - Class, Reason, Stacktrace, - #state{ + Class, Reason, Stacktrace, Debug, + #params{ name = Name, callback_mode = CallbackMode, - state_enter = StateEnter, - postponed = P} = S, + state_enter = StateEnter} = P, + #state{postponed = Postponed} = S, Q) -> + Log = sys:get_log(Debug), ?LOG_ERROR(#{label=>{gen_statem,terminate}, name=>Name, queue=>Q, - postponed=>P, + postponed=>Postponed, callback_mode=>CallbackMode, state_enter=>StateEnter, - state=>format_status(terminate, get(), S), - reason=>{Class,Reason,Stacktrace}}, + state=>format_status(terminate, get(), P, S), + log=>Log, + reason=>{Class,Reason,Stacktrace}, + client_info=>client_stacktrace(Q)}, #{domain=>[otp], report_cb=>fun gen_statem:format_log/1, error_logger=>#{tag=>error}}). +client_stacktrace([]) -> + undefined; +client_stacktrace([{{call,{Pid,_Tag}},_Req}|_]) when is_pid(Pid) -> + if + node(Pid) =:= node() -> + case + process_info(Pid, [current_stacktrace, registered_name]) + of + undefined -> + {Pid,dead}; + [{current_stacktrace, Stacktrace}, + {registered_name, []}] -> + {Pid,{Pid,Stacktrace}}; + [{current_stacktrace, Stacktrace}, + {registered_name, Name}] -> + {Pid,{Name,Stacktrace}} + end; + true -> + {Pid,remote} + end; +client_stacktrace([_|_]) -> + undefined. + + format_log(#{label:={gen_statem,terminate}, name:=Name, queue:=Q, - postponed:=P, + postponed:=Postponed, callback_mode:=CallbackMode, state_enter:=StateEnter, state:=FmtData, - reason:={Class,Reason,Stacktrace}}) -> + log:=Log, + reason:={Class,Reason,Stacktrace}, + client_info:=ClientInfo}) -> {FixedReason,FixedStacktrace} = case Stacktrace of [{M,F,Args,_}|ST] @@ -1857,14 +2260,12 @@ format_log(#{label:={gen_statem,terminate}, true -> {Reason,Stacktrace}; false -> - {{'function not exported',{M,F,Arity}}, - ST} + {{'function not exported',{M,F,Arity}},ST} end end; _ -> {Reason,Stacktrace} end, - [LimitedP, LimitedFmtData, LimitedFixedReason] = - [error_logger:limit_term(D) || D <- [P, FmtData, FixedReason]], + {ClientFmt,ClientArgs} = format_client_log(ClientInfo), CBMode = case StateEnter of true -> @@ -1884,39 +2285,60 @@ format_log(#{label:={gen_statem,terminate}, [_,_|_] -> "** Queued = ~tp~n"; _ -> "" end ++ - case P of + case Postponed of [] -> ""; _ -> "** Postponed = ~tp~n" end ++ case FixedStacktrace of [] -> ""; _ -> "** Stacktrace =~n** ~tp~n" - end, + end ++ + case Log of + [] -> ""; + _ -> "** Log =~n** ~tp~n" + end ++ ClientFmt, [Name | case Q of [] -> []; - [Event|_] -> [Event] + [Event|_] -> [error_logger:limit_term(Event)] end] ++ - [LimitedFmtData, - Class,LimitedFixedReason, + [error_logger:limit_term(FmtData), + Class,error_logger:limit_term(FixedReason), CBMode] ++ case Q of - [_|[_|_] = Events] -> [Events]; + [_|[_|_] = Events] -> [error_logger:limit_term(Events)]; _ -> [] end ++ - case P of + case Postponed of [] -> []; - _ -> [LimitedP] + _ -> [error_logger:limit_term(Postponed)] end ++ case FixedStacktrace of [] -> []; - _ -> [FixedStacktrace] - end}. + _ -> [error_logger:limit_term(FixedStacktrace)] + end ++ + case Log of + [] -> []; + _ -> [[error_logger:limit_term(T) || T <- Log]] + end ++ ClientArgs}. + +format_client_log(undefined) -> + {"", []}; +format_client_log({Pid,dead}) -> + {"** Client ~p is dead~n", [Pid]}; +format_client_log({Pid,remote}) -> + {"** Client ~p is remote on node ~p~n", [Pid, node(Pid)]}; +format_client_log({_Pid,{Name,Stacktrace}}) -> + {"** Client ~tp stacktrace~n" + "** ~tp~n", + [Name, error_logger:limit_term(Stacktrace)]}. + %% Call Module:format_status/2 or return a default value format_status( Opt, PDict, - #state{module = Module, state = State, data = Data}) -> + #params{module = Module}, + #state{state_data = {State,Data} = State_Data}) -> case erlang:function_exported(Module, format_status, 2) of true -> try Module:format_status(Opt, [PDict,State,Data]) @@ -1924,21 +2346,21 @@ format_status( Result -> Result; _:_ -> format_status_default( - Opt, State, - atom_to_list(Module) ++ ":format_status/2 crashed") + Opt, + {State, + atom_to_list(Module) ++ ":format_status/2 crashed"}) end; false -> - format_status_default(Opt, State, Data) + format_status_default(Opt, State_Data) end. -%% The default Module:format_status/2 -format_status_default(Opt, State, Data) -> - StateData = {State,Data}, +%% The default Module:format_status/3 +format_status_default(Opt, State_Data) -> case Opt of terminate -> - StateData; + State_Data; _ -> - [{data,[{"State",StateData}]}] + [{data,[{"State",State_Data}]}] end. -compile({inline, [listify/1]}). @@ -1967,14 +2389,21 @@ cancel_timer(TimerRef) -> %% Cancel timer if running, otherwise no op %% -%% Remove the timer from Timers. +%% Remove the timer from Timers -compile({inline, [cancel_timer_by_type/2]}). -cancel_timer_by_type(TimerType, {TimerRefs,TimerTypes} = Timers) -> - case TimerTypes of - #{TimerType := TimerRef} -> +cancel_timer_by_type(TimeoutType, {TimerRefs,TimeoutTypes} = Timers) -> + case TimeoutTypes of + #{TimeoutType := TimerRef} -> ?cancel_timer(TimerRef), - {maps:remove(TimerRef, TimerRefs), - maps:remove(TimerType, TimerTypes)}; - #{} -> - Timers + {maps:remove(TimerRef, TimerRefs), + maps:remove(TimeoutType, TimeoutTypes)}; + #{} -> + Timers end. + +-compile({inline, [cancel_timer_by_ref_and_type/4]}). +cancel_timer_by_ref_and_type( + TimerRef, TimeoutType, TimerRefs, TimeoutTypes) -> + ?cancel_timer(TimerRef), + {maps:remove(TimerRef, TimerRefs), + maps:remove(TimeoutType, TimeoutTypes)}. diff --git a/lib/stdlib/src/maps.erl b/lib/stdlib/src/maps.erl index 60463feec2..51965ddb57 100644 --- a/lib/stdlib/src/maps.erl +++ b/lib/stdlib/src/maps.erl @@ -21,7 +21,7 @@ -module(maps). -export([get/3, filter/2,fold/3, - map/2, size/1, + map/2, size/1, new/0, update_with/3, update_with/4, without/2, with/2, iterator/1, next/1]). @@ -29,13 +29,15 @@ %% BIFs -export([get/2, find/2, from_list/1, is_key/2, keys/1, merge/2, - new/0, put/3, remove/2, take/2, + put/3, remove/2, take/2, to_list/1, update/3, values/1]). --opaque iterator() :: {term(), term(), iterator()} - | none | nonempty_improper_list(integer(),map()). +-opaque iterator(Key, Value) :: {Key, Value, iterator(Key, Value)} | none + | nonempty_improper_list(integer(), #{Key => Value}). --export_type([iterator/0]). +-type iterator() :: iterator(term(), term()). + +-export_type([iterator/2, iterator/0]). -dialyzer({no_improper_lists, iterator/1}). @@ -50,9 +52,7 @@ get(_,_) -> erlang:nif_error(undef). -spec find(Key,Map) -> {ok, Value} | error when - Key :: term(), - Map :: map(), - Value :: term(). + Map :: #{Key => Value, _ => _}. find(_,_) -> erlang:nif_error(undef). @@ -75,9 +75,8 @@ is_key(_,_) -> erlang:nif_error(undef). -spec keys(Map) -> Keys when - Map :: map(), - Keys :: [Key], - Key :: term(). + Map :: #{Key => _}, + Keys :: [Key]. keys(_) -> erlang:nif_error(undef). @@ -91,13 +90,6 @@ keys(_) -> erlang:nif_error(undef). merge(_,_) -> erlang:nif_error(undef). - --spec new() -> Map when - Map :: map(). - -new() -> erlang:nif_error(undef). - - %% Shadowed by erl_bif_types: maps:put/3 -spec put(Key,Value,Map1) -> Map2 when Key :: term(), @@ -116,17 +108,13 @@ put(_,_,_) -> erlang:nif_error(undef). remove(_,_) -> erlang:nif_error(undef). -spec take(Key,Map1) -> {Value,Map2} | error when - Key :: term(), - Map1 :: map(), - Value :: term(), - Map2 :: map(). + Map1 :: #{Key => Value, _ => _}, + Map2 :: #{_ => _}. take(_,_) -> erlang:nif_error(undef). -spec to_list(Map) -> [{Key,Value}] when - Map :: map(), - Key :: term(), - Value :: term(). + Map :: #{Key => Value}. to_list(Map) when is_map(Map) -> to_list_internal(erts_internal:map_next(0, Map, [])); @@ -140,79 +128,69 @@ to_list_internal(Acc) -> %% Shadowed by erl_bif_types: maps:update/3 -spec update(Key,Value,Map1) -> Map2 when - Key :: term(), - Value :: term(), - Map1 :: map(), - Map2 :: map(). + Map1 :: #{Key := _, _ => _}, + Map2 :: #{Key := Value, _ => _}. update(_,_,_) -> erlang:nif_error(undef). -spec values(Map) -> Values when - Map :: map(), - Values :: [Value], - Value :: term(). + Map :: #{_ => Value}, + Values :: [Value]. values(_) -> erlang:nif_error(undef). %% End of BIFs +-spec new() -> Map when + Map :: #{}. + +new() -> #{}. + -spec update_with(Key,Fun,Map1) -> Map2 when - Key :: term(), - Map1 :: map(), - Map2 :: map(), - Fun :: fun((Value1 :: term()) -> Value2 :: term()). + Map1 :: #{Key := Value1, _ => _}, + Map2 :: #{Key := Value2, _ => _}, + Fun :: fun((Value1) -> Value2). update_with(Key,Fun,Map) when is_function(Fun,1), is_map(Map) -> - try maps:get(Key,Map) of - Val -> maps:update(Key,Fun(Val),Map) - catch - error:{badkey,_} -> - erlang:error({badkey,Key},[Key,Fun,Map]) + case Map of + #{Key := Value} -> Map#{Key := Fun(Value)}; + #{} -> erlang:error({badkey,Key},[Key,Fun,Map]) end; update_with(Key,Fun,Map) -> erlang:error(error_type(Map),[Key,Fun,Map]). -spec update_with(Key,Fun,Init,Map1) -> Map2 when - Key :: term(), - Map1 :: Map1, - Map2 :: Map2, - Fun :: fun((Value1 :: term()) -> Value2 :: term()), - Init :: term(). + Map1 :: #{Key => Value1, _ => _}, + Map2 :: #{Key := Value2 | Init, _ => _}, + Fun :: fun((Value1) -> Value2). update_with(Key,Fun,Init,Map) when is_function(Fun,1), is_map(Map) -> - case maps:find(Key,Map) of - {ok,Val} -> maps:update(Key,Fun(Val),Map); - error -> maps:put(Key,Init,Map) + case Map of + #{Key := Value} -> Map#{Key := Fun(Value)}; + #{} -> Map#{Key => Init} end; update_with(Key,Fun,Init,Map) -> erlang:error(error_type(Map),[Key,Fun,Init,Map]). -spec get(Key, Map, Default) -> Value | Default when - Key :: term(), - Map :: map(), - Value :: term(), - Default :: term(). + Map :: #{Key => Value, _ => _}. get(Key,Map,Default) when is_map(Map) -> - case maps:find(Key, Map) of - {ok, Value} -> - Value; - error -> - Default + case Map of + #{Key := Value} -> Value; + #{} -> Default end; get(Key,Map,Default) -> erlang:error({badmap,Map},[Key,Map,Default]). --spec filter(Pred,MapOrIter) -> Map when +-spec filter(Pred, MapOrIter) -> Map when Pred :: fun((Key, Value) -> boolean()), - Key :: term(), - Value :: term(), - MapOrIter :: map() | iterator(), - Map :: map(). + MapOrIter :: #{Key => Value} | iterator(Key, Value), + Map :: #{Key => Value}. filter(Pred,Map) when is_function(Pred,2), is_map(Map) -> maps:from_list(filter_1(Pred, iterator(Map))); @@ -235,14 +213,11 @@ filter_1(Pred, Iter) -> end. -spec fold(Fun,Init,MapOrIter) -> Acc when - Fun :: fun((K, V, AccIn) -> AccOut), + Fun :: fun((Key, Value, AccIn) -> AccOut), Init :: term(), - Acc :: term(), - AccIn :: term(), - AccOut :: term(), - MapOrIter :: map() | iterator(), - K :: term(), - V :: term(). + Acc :: AccOut, + AccIn :: Init | AccOut, + MapOrIter :: #{Key => Value} | iterator(Key, Value). fold(Fun,Init,Map) when is_function(Fun,3), is_map(Map) -> fold_1(Fun,Init,iterator(Map)); @@ -260,12 +235,9 @@ fold_1(Fun, Acc, Iter) -> end. -spec map(Fun,MapOrIter) -> Map when - Fun :: fun((K, V1) -> V2), - MapOrIter :: map() | iterator(), - Map :: map(), - K :: term(), - V1 :: term(), - V2 :: term(). + Fun :: fun((Key, Value1) -> Value2), + MapOrIter :: #{Key => Value1} | iterator(Key, Value1), + Map :: #{Key => Value2}. map(Fun,Map) when is_function(Fun, 2), is_map(Map) -> maps:from_list(map_1(Fun, iterator(Map))); @@ -291,17 +263,15 @@ size(Val) -> erlang:error({badmap,Val},[Val]). -spec iterator(Map) -> Iterator when - Map :: map(), - Iterator :: iterator(). + Map :: #{Key => Value}, + Iterator :: iterator(Key, Value). iterator(M) when is_map(M) -> [0 | M]; iterator(M) -> erlang:error({badmap, M}, [M]). -spec next(Iterator) -> {Key, Value, NextIterator} | 'none' when - Iterator :: iterator(), - Key :: term(), - Value :: term(), - NextIterator :: iterator(). + Iterator :: iterator(Key, Value), + NextIterator :: iterator(Key, Value). next({K, V, I}) -> {K, V, I}; next([Path | Map]) when is_integer(Path), is_map(Map) -> @@ -318,29 +288,26 @@ next(Iter) -> K :: term(). without(Ks,M) when is_list(Ks), is_map(M) -> - lists:foldl(fun(K, M1) -> maps:remove(K, M1) end, M, Ks); + lists:foldl(fun maps:remove/2, M, Ks); without(Ks,M) -> erlang:error(error_type(M),[Ks,M]). -spec with(Ks, Map1) -> Map2 when Ks :: [K], - Map1 :: map(), - Map2 :: map(), - K :: term(). + Map1 :: #{K => V, _ => _}, + Map2 :: #{K => V}. with(Ks,Map1) when is_list(Ks), is_map(Map1) -> - Fun = fun(K, List) -> - case maps:find(K, Map1) of - {ok, V} -> - [{K, V} | List]; - error -> - List - end - end, - maps:from_list(lists:foldl(Fun, [], Ks)); + maps:from_list(with_1(Ks, Map1)); with(Ks,M) -> erlang:error(error_type(M),[Ks,M]). +with_1([K|Ks], Map) -> + case Map of + #{K := V} -> [{K,V}|with_1(Ks, Map)]; + #{} -> with_1(Ks, Map) + end; +with_1([], _Map) -> []. error_type(M) when is_map(M) -> badarg; error_type(V) -> {badmap, V}. diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index aaed13ba3a..fa34f19637 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -44,8 +44,19 @@ obsolete(Module, Name, Arity) -> no end. -obsolete_1(net, _, _) -> - {deprecated, "module 'net' obsolete; use 'net_adm'"}; +obsolete_1(net, call, 4) -> + {deprecated, {rpc, call, 4}}; +obsolete_1(net, cast, 4) -> + {deprecated, {rpc, cast, 4}}; +obsolete_1(net, broadcast, 3) -> + {deprecated, {rpc, eval_everywhere, 3}}; +obsolete_1(net, ping, 1) -> + {deprecated, {net_adm, ping, 1}}; +obsolete_1(net, sleep, 1) -> + {deprecated, "Use 'receive after T -> ok end' instead"}; +obsolete_1(net, relay, 1) -> + {deprecated, {slave, relay, 1}}; + obsolete_1(erlang, now, 0) -> {deprecated, @@ -55,6 +66,14 @@ obsolete_1(erlang, now, 0) -> obsolete_1(calendar, local_time_to_universal_time, 1) -> {deprecated, {calendar, local_time_to_universal_time_dst, 1}}; +%% *** STDLIB added in OTP 22 *** + +obsolete_1(sys, get_debug, 3) -> + {deprecated, + "Deprecated function. " + "Incorrectly documented and in fact only for internal use. " + "Can often be replaced with sys:get_log/1."}; + %% *** STDLIB added in OTP 20 *** obsolete_1(gen_fsm, start, 3) -> @@ -304,90 +323,6 @@ obsolete_1(snmp, N, A) -> obsolete_1(snmpa, old_info_format, 1) -> {deprecated, "Deprecated; (will be removed in OTP 18); use \"new\" format instead"}; -obsolete_1(snmpm, agent_info, 3) -> - {removed, {snmpm, agent_info, 2}, "R16B"}; -obsolete_1(snmpm, update_agent_info, 5) -> - {removed, {snmpm, update_agent_info, 4}, "R16B"}; -obsolete_1(snmpm, g, 3) -> - {removed, {snmpm, sync_get, 3}, "R16B"}; -obsolete_1(snmpm, g, 4) -> - {removed, {snmpm, sync_get, [3,4]}, "R16B"}; -obsolete_1(snmpm, g, 5) -> - {removed, {snmpm, sync_get, [4,5]}, "R16B"}; -obsolete_1(snmpm, g, 6) -> - {removed, {snmpm, sync_get, [5,6]}, "R16B"}; -obsolete_1(snmpm, g, 7) -> - {removed, {snmpm, sync_get, 6}, "R16B"}; -obsolete_1(snmpm, ag, 3) -> - {removed, {snmpm, async_get, 3}, "R16B"}; -obsolete_1(snmpm, ag, 4) -> - {removed, {snmpm, async_get, [3,4]}, "R16B"}; -obsolete_1(snmpm, ag, 5) -> - {removed, {snmpm, async_get, [4,5]}, "R16B"}; -obsolete_1(snmpm, ag, 6) -> - {removed, {snmpm, async_get, [5,6]}, "R16B"}; -obsolete_1(snmpm, ag, 7) -> - {removed, {snmpm, async_get, 6}, "R16B"}; -obsolete_1(snmpm, gn, 3) -> - {removed, {snmpm, sync_get_next, 3}, "R16B"}; -obsolete_1(snmpm, gn, 4) -> - {removed, {snmpm, sync_get_next, [3,4]}, "R16B"}; -obsolete_1(snmpm, gn, 5) -> - {removed, {snmpm, sync_get_next, [4,5]}, "R16B"}; -obsolete_1(snmpm, gn, 6) -> - {removed, {snmpm, sync_get_next, [5,6]}, "R16B"}; -obsolete_1(snmpm, gn, 7) -> - {removed, {snmpm, sync_get_next, 6}, "R16B"}; -obsolete_1(snmpm, agn, 3) -> - {removed, {snmpm, async_get_next, 3}, "R16B"}; -obsolete_1(snmpm, agn, 4) -> - {removed, {snmpm, async_get_next, [3,4]}, "R16B"}; -obsolete_1(snmpm, agn, 5) -> - {removed, {snmpm, async_get_next, [4,5]}, "R16B"}; -obsolete_1(snmpm, agn, 6) -> - {removed, {snmpm, async_get_next, [5,6]}, "R16B"}; -obsolete_1(snmpm, agn, 7) -> - {removed, {snmpm, async_get_next, 6}, "R16B"}; -obsolete_1(snmpm, s, 3) -> - {removed, {snmpm, sync_set, 3}, "R16B"}; -obsolete_1(snmpm, s, 4) -> - {removed, {snmpm, sync_set, [3,4]}, "R16B"}; -obsolete_1(snmpm, s, 5) -> - {removed, {snmpm, sync_set, [4,5]}, "R16B"}; -obsolete_1(snmpm, s, 6) -> - {removed, {snmpm, sync_set, [5,6]}, "R16B"}; -obsolete_1(snmpm, s, 7) -> - {removed, {snmpm, sync_set, 6}, "R16B"}; -obsolete_1(snmpm, as, 3) -> - {removed, {snmpm, async_set, 3}, "R16B"}; -obsolete_1(snmpm, as, 4) -> - {removed, {snmpm, async_set, [3,4]}, "R16B"}; -obsolete_1(snmpm, as, 5) -> - {removed, {snmpm, async_set, [4,5]}, "R16B"}; -obsolete_1(snmpm, as, 6) -> - {removed, {snmpm, async_set, [5,6]}, "R16B"}; -obsolete_1(snmpm, as, 7) -> - {removed, {snmpm, async_set, 6}, "R16B"}; -obsolete_1(snmpm, gb, 5) -> - {removed, {snmpm, sync_get_bulk, 5}, "R16B"}; -obsolete_1(snmpm, gb, 6) -> - {removed, {snmpm, sync_get_bulk, [5,6]}, "R16B"}; -obsolete_1(snmpm, gb, 7) -> - {removed, {snmpm, sync_get_bulk, [6,7]}, "R16B"}; -obsolete_1(snmpm, gb, 8) -> - {removed, {snmpm, sync_get_bulk, [7,8]}, "R16B"}; -obsolete_1(snmpm, gb, 9) -> - {removed, {snmpm, sync_get_bulk, 8}, "R16B"}; -obsolete_1(snmpm, agb, 5) -> - {removed, {snmpm, async_get_bulk, 5}, "R16B"}; -obsolete_1(snmpm, agb, 6) -> - {removed, {snmpm, async_get_bulk, [5,6]}, "R16B"}; -obsolete_1(snmpm, agb, 7) -> - {removed, {snmpm, async_get_bulk, [6,7]}, "R16B"}; -obsolete_1(snmpm, agb, 8) -> - {removed, {snmpm, async_get_bulk, [7,8]}, "R16B"}; -obsolete_1(snmpm, agb, 9) -> - {removed, {snmpm, async_get_bulk, 8}, "R16B"}; %% *** MEGACO *** @@ -398,10 +333,9 @@ obsolete_1(megaco, format_versions, 1) -> %% *** OS-MON-MIB *** -obsolete_1(os_mon_mib, init, 1) -> - {deprecated, {os_mon_mib, load, 1}}; -obsolete_1(os_mon_mib, stop, 1) -> - {deprecated, {os_mon_mib, unload, 1}}; +%% FIXME: Remove this warning in OTP 24. +obsolete_1(os_mon_mib, _, _) -> + {removed, "was removed in 22.0"}; obsolete_1(auth, is_auth, 1) -> {deprecated, {net_adm, ping, 1}}; @@ -414,64 +348,6 @@ obsolete_1(auth, node_cookie, 1) -> obsolete_1(auth, node_cookie, 2) -> {deprecated, "Deprecated; use erlang:set_cookie/2 and net_adm:ping/1 instead"}; -obsolete_1(http, request, 1) -> {removed,{httpc,request,1},"R15B"}; -obsolete_1(http, request, 2) -> {removed,{httpc,request,2},"R15B"}; -obsolete_1(http, request, 4) -> {removed,{httpc,request,4},"R15B"}; -obsolete_1(http, request, 5) -> {removed,{httpc,request,5},"R15B"}; -obsolete_1(http, cancel_request, 1) -> {removed,{httpc,cancel_request,1},"R15B"}; -obsolete_1(http, cancel_request, 2) -> {removed,{httpc,cancel_request,2},"R15B"}; -obsolete_1(http, set_option, 2) -> {removed,{httpc,set_option,2},"R15B"}; -obsolete_1(http, set_option, 3) -> {removed,{httpc,set_option,3},"R15B"}; -obsolete_1(http, set_options, 1) -> {removed,{httpc,set_options,1},"R15B"}; -obsolete_1(http, set_options, 2) -> {removed,{httpc,set_options,2},"R15B"}; -obsolete_1(http, verify_cookies, 2) -> {removed,{httpc,store_cookies,2},"R15B"}; -obsolete_1(http, verify_cookies, 3) -> {removed,{httpc,store_cookies,3},"R15B"}; -obsolete_1(http, cookie_header, 1) -> {removed,{httpc,cookie_header,1},"R15B"}; -obsolete_1(http, cookie_header, 2) -> {removed,{httpc,cookie_header,2},"R15B"}; -obsolete_1(http, stream_next, 1) -> {removed,{httpc,stream_next,1},"R15B"}; -obsolete_1(http, default_profile, 0) -> {removed,{httpc,default_profile,0},"R15B"}; - -%% Added in R13A. -obsolete_1(regexp, _, _) -> - {removed, "removed in R15; use the re module instead"}; - -%% Added in R13B04. -obsolete_1(erlang, concat_binary, 1) -> - {removed,{erlang,list_to_binary,1},"R15B"}; - -%% Added in R14A. -obsolete_1(ssl, peercert, 2) -> - {removed ,"removed in R15A; use ssl:peercert/1 and public_key:pkix_decode_cert/2 instead"}; - -%% Added in R14B. -obsolete_1(public_key, pem_to_der, 1) -> - {removed,"removed in R15A; use file:read_file/1 and public_key:pem_decode/1"}; -obsolete_1(public_key, decode_private_key, A) when A =:= 1; A =:= 2 -> - {removed, "removed in R15A; use public_key:pem_entry_decode/1"}; - -%% Added in R14B03. -obsolete_1(docb_gen, _, _) -> - {removed,"the DocBuilder application was removed in R15B"}; -obsolete_1(docb_transform, _, _) -> - {removed,"the DocBuilder application was removed in R15B"}; -obsolete_1(docb_xml_check, _, _) -> - {removed,"the DocBuilder application was removed in R15B"}; - -%% Added in R15B -obsolete_1(asn1rt, F, _) when F == load_driver; F == unload_driver -> - {removed,"removed (will be removed in OTP 18); has no effect as drivers are no longer used"}; -obsolete_1(ssl, pid, 1) -> - {removed,"was removed in R16; is no longer needed"}; -obsolete_1(inviso, _, _) -> - {removed,"the inviso application was removed in R16"}; - -%% Added in R15B01. -obsolete_1(ssh, sign_data, 2) -> - {removed,"removed in R16A; use public_key:pem_decode/1, public_key:pem_entry_decode/1 " - "and public_key:sign/3 instead"}; -obsolete_1(ssh, verify_data, 3) -> - {removed,"removed in R16A; use public_key:ssh_decode/1, and public_key:verify/4 instead"}; - %% Added in R16 obsolete_1(wxCalendarCtrl, enableYearChange, _) -> %% wx bug documented? {deprecated,"deprecated function not available in wxWidgets-2.9 and later"}; @@ -592,10 +468,8 @@ obsolete_1(queue, lait, 1) -> %% Removed in OTP 19. -obsolete_1(overload, _, _) -> - {removed, "removed in OTP 19"}; obsolete_1(rpc, safe_multi_server_call, A) when A =:= 2; A =:= 3 -> - {removed, {rpc, multi_server_call, A}, "removed in OTP 19"}; + {removed, {rpc, multi_server_call, A}, "19.0"}; %% Added in OTP 20. diff --git a/lib/stdlib/src/rand.erl b/lib/stdlib/src/rand.erl index 362e98006e..3a9a1e007b 100644 --- a/lib/stdlib/src/rand.erl +++ b/lib/stdlib/src/rand.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2015-2017. All Rights Reserved. +%% Copyright Ericsson AB 2015-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,18 +32,24 @@ uniform/0, uniform/1, uniform_s/1, uniform_s/2, uniform_real/0, uniform_real_s/1, jump/0, jump/1, - normal/0, normal/2, normal_s/1, normal_s/3 + normal/0, normal/2, normal_s/1, normal_s/3 ]). +%% Test, dev and internal +-export([exro928_jump_2pow512/1, exro928_jump_2pow20/1, + exro928_seed/1, exro928_next/1, exro928_next_state/1, + format_jumpconst58/1, seed58/2]). + %% Debug -export([make_float/3, float2str/1, bc64/1]). --compile({inline, [exs64_next/1, exsplus_next/1, +-compile({inline, [exs64_next/1, exsplus_next/1, exsss_next/1, exs1024_next/1, exs1024_calc/2, + exro928_next_state/4, exrop_next/1, exrop_next_s/2, get_52/1, normal_kiwi/1]}). --define(DEFAULT_ALG_HANDLER, exrop). +-define(DEFAULT_ALG_HANDLER, exsss). -define(SEED_DICT, rand_seed). %% ===================================================================== @@ -80,8 +86,8 @@ %% This depends on the algorithm handler function -type alg_state() :: - exs64_state() | exsplus_state() | exs1024_state() | - exrop_state() | term(). + exsplus_state() | exro928_state() | exrop_state() | exs1024_state() | + exs64_state() | term(). %% This is the algorithm handling definition within this module, %% and the type to use for plugins. @@ -124,14 +130,17 @@ %% Algorithm state -type state() :: {alg_handler(), alg_state()}. --type builtin_alg() :: exs64 | exsplus | exsp | exs1024 | exs1024s | exrop. +-type builtin_alg() :: + exsss | exro928ss | exrop | exs1024s | exsp | exs64 | exsplus | exs1024. -type alg() :: builtin_alg() | atom(). -type export_state() :: {alg(), alg_state()}. +-type seed() :: [integer()] | integer() | {integer(), integer(), integer()}. -export_type( [builtin_alg/0, alg/0, alg_handler/0, alg_state/0, - state/0, export_state/0]). + state/0, export_state/0, seed/0]). -export_type( - [exs64_state/0, exsplus_state/0, exs1024_state/0, exrop_state/0]). + [exsplus_state/0, exro928_state/0, exrop_state/0, exs1024_state/0, + exs64_state/0]). %% ===================================================================== %% Range macro and helper @@ -229,12 +238,12 @@ export_seed() -> end. -spec export_seed_s(State :: state()) -> export_state(). -export_seed_s({#{type:=Alg}, Seed}) -> {Alg, Seed}. +export_seed_s({#{type:=Alg}, AlgState}) -> {Alg, AlgState}. %% seed(Alg) seeds RNG with runtime dependent values %% and return the NEW state -%% seed({Alg,Seed}) setup RNG with a previously exported seed +%% seed({Alg,AlgState}) setup RNG with a previously exported seed %% and return the NEW state -spec seed( @@ -246,11 +255,11 @@ seed(Alg) -> -spec seed_s( AlgOrStateOrExpState :: builtin_alg() | state() | export_state()) -> state(). -seed_s({AlgHandler, _Seed} = State) when is_map(AlgHandler) -> +seed_s({AlgHandler, _AlgState} = State) when is_map(AlgHandler) -> State; -seed_s({Alg0, Seed}) -> - {Alg,_SeedFun} = mk_alg(Alg0), - {Alg, Seed}; +seed_s({Alg, AlgState}) when is_atom(Alg) -> + {AlgHandler,_SeedFun} = mk_alg(Alg), + {AlgHandler,AlgState}; seed_s(Alg) -> seed_s(Alg, {erlang:phash2([{node(),self()}]), erlang:system_time(), @@ -259,19 +268,15 @@ seed_s(Alg) -> %% seed/2: seeds RNG with the algorithm and given values %% and returns the NEW state. --spec seed( - Alg :: builtin_alg(), Seed :: {integer(), integer(), integer()}) -> - state(). -seed(Alg0, S0) -> - seed_put(seed_s(Alg0, S0)). +-spec seed(Alg :: builtin_alg(), Seed :: seed()) -> state(). +seed(Alg, Seed) -> + seed_put(seed_s(Alg, Seed)). --spec seed_s( - Alg :: builtin_alg(), Seed :: {integer(), integer(), integer()}) -> - state(). -seed_s(Alg0, S0 = {_, _, _}) -> - {Alg, Seed} = mk_alg(Alg0), - AS = Seed(S0), - {Alg, AS}. +-spec seed_s(Alg :: builtin_alg(), Seed :: seed()) -> state(). +seed_s(Alg, Seed) -> + {AlgHandler,SeedFun} = mk_alg(Alg), + AlgState = SeedFun(Seed), + {AlgHandler,AlgState}. %%% uniform/0, uniform/1, uniform_s/1, uniform_s/2 are all %%% uniformly distributed random numbers. @@ -281,8 +286,8 @@ seed_s(Alg0, S0 = {_, _, _}) -> -spec uniform() -> X :: float(). uniform() -> - {X, Seed} = uniform_s(seed_get()), - _ = seed_put(Seed), + {X, State} = uniform_s(seed_get()), + _ = seed_put(State), X. %% uniform/1: given an integer N >= 1, @@ -291,8 +296,8 @@ uniform() -> -spec uniform(N :: pos_integer()) -> X :: pos_integer(). uniform(N) -> - {X, Seed} = uniform_s(N, seed_get()), - _ = seed_put(Seed), + {X, State} = uniform_s(N, seed_get()), + _ = seed_put(State), X. %% uniform_s/1: given a state, uniform_s/1 @@ -486,7 +491,7 @@ uniform_real_s(Alg, Next, M0, BitNo, R1, V1, Bits) -> {M1 * math:pow(2.0, BitNo - 56), {Alg, R1}}; BitNo =:= -1008 -> %% Endgame - %% For the last round we can not have 14 zeros or more + %% For the last round we cannot have 14 zeros or more %% at the top of M1 because then we will underflow, %% so we need at least 43 bits if @@ -613,6 +618,11 @@ mk_alg(exsp) -> uniform=>fun exsp_uniform/1, uniform_n=>fun exsp_uniform/2, jump=>fun exsplus_jump/1}, fun exsplus_seed/1}; +mk_alg(exsss) -> + {#{type=>exsss, bits=>58, next=>fun exsss_next/1, + uniform=>fun exsss_uniform/1, uniform_n=>fun exsss_uniform/2, + jump=>fun exsplus_jump/1}, + fun exsss_seed/1}; mk_alg(exs1024) -> {#{type=>exs1024, max=>?MASK(64), next=>fun exs1024_next/1, jump=>fun exs1024_jump/1}, @@ -625,7 +635,13 @@ mk_alg(exrop) -> {#{type=>exrop, bits=>58, weak_low_bits=>1, next=>fun exrop_next/1, uniform=>fun exrop_uniform/1, uniform_n=>fun exrop_uniform/2, jump=>fun exrop_jump/1}, - fun exrop_seed/1}. + fun exrop_seed/1}; +mk_alg(exro928ss) -> + {#{type=>exro928ss, bits=>58, next=>fun exro928ss_next/1, + uniform=>fun exro928ss_uniform/1, + uniform_n=>fun exro928ss_uniform/2, + jump=>fun exro928_jump/1}, + fun exro928_seed/1}. %% ===================================================================== %% exs64 PRNG: Xorshift64* @@ -635,6 +651,14 @@ mk_alg(exrop) -> -opaque exs64_state() :: uint64(). +exs64_seed(L) when is_list(L) -> + [R] = seed64_nz(1, L), + R; +exs64_seed(A) when is_integer(A) -> + [R] = seed64(1, ?MASK(64, A)), + R; +%% +%% Traditional integer triplet seed exs64_seed({A1, A2, A3}) -> {V1, _} = exs64_next((?MASK(32, A1) * 4294967197 + 1)), {V2, _} = exs64_next((?MASK(32, A2) * 4294967231 + 1)), @@ -656,11 +680,49 @@ exs64_next(R) -> %% 58 bits fits into an immediate on 64bits erlang and is thus much faster. %% Modification of the original Xorshift128+ algorithm to 116 %% by Sebastiano Vigna, a lot of thanks for his help and work. +%% +%% Reference C code for Xorshift116+ and Xorshift116** +%% +%% #include <stdint.h> +%% +%% #define MASK(b, v) (((UINT64_C(1) << (b)) - 1) & (v)) +%% #define BSL(b, v, n) (MASK((b)-(n), (v)) << (n)) +%% #define ROTL(b, v, n) (BSL((b), (v), (n)) | ((v) >> ((b)-(n)))) +%% +%% uint64_t s[2]; +%% +%% uint64_t next(void) { +%% uint64_t s1 = s[0]; +%% const uint64_t s0 = s[1]; +%% +%% s1 ^= BSL(58, s1, 24); // a +%% s1 ^= s0 ^ (s1 >> 11) ^ (s0 >> 41); // b, c +%% s[0] = s0; +%% s[1] = s1; +%% +%% const uint64_t result_plus = MASK(58, s0 + s1); +%% uint64_t result_starstar = s0; +%% result_starstar = MASK(58, result_starstar * 5); +%% result_starstar = ROTL(58, result_starstar, 7); +%% result_starstar = MASK(58, result_starstar * 9); +%% +%% return result_plus; +%% return result_starstar; +%% } +%% %% ===================================================================== -opaque exsplus_state() :: nonempty_improper_list(uint58(), uint58()). -dialyzer({no_improper_lists, exsplus_seed/1}). +exsplus_seed(L) when is_list(L) -> + [S0,S1] = seed58_nz(2, L), + [S0|S1]; +exsplus_seed(X) when is_integer(X) -> + [S0,S1] = seed58(2, ?MASK(64, X)), + [S0|S1]; +%% +%% Traditional integer triplet seed exsplus_seed({A1, A2, A3}) -> {_, R1} = exsplus_next( [?MASK(58, (A1 * 4294967197) + 1)| @@ -670,16 +732,62 @@ exsplus_seed({A1, A2, A3}) -> tl(R1)]), R2. +-dialyzer({no_improper_lists, exsss_seed/1}). + +exsss_seed(L) when is_list(L) -> + [S0,S1] = seed58_nz(2, L), + [S0|S1]; +exsss_seed(X) when is_integer(X) -> + [S0,S1] = seed58(2, ?MASK(64, X)), + [S0|S1]; +%% +%% Seed from traditional integer triple - mix into splitmix +exsss_seed({A1, A2, A3}) -> + {_, X0} = seed58(?MASK(64, A1)), + {S0, X1} = seed58(?MASK(64, A2) bxor X0), + {S1, _} = seed58(?MASK(64, A3) bxor X1), + [S0|S1]. + +%% Advance Xorshift116 state one step +-define( + exs_next(S0, S1, S1_b), + begin + S1_b = S1 bxor ?BSL(58, S1, 24), + S1_b bxor S0 bxor (S1_b bsr 11) bxor (S0 bsr 41) + end). + +-define( + scramble_starstar(S, V_a, V_b), + begin + %% The multiply by add shifted trick avoids creating bignums + %% which improves performance significantly + %% + V_a = ?MASK(58, S + ?BSL(58, S, 2)), % V_a = S * 5 + V_b = ?ROTL(58, V_a, 7), + ?MASK(58, V_b + ?BSL(58, V_b, 3)) % V_b * 9 + end). + -dialyzer({no_improper_lists, exsplus_next/1}). -%% Advance xorshift116+ state for one step and generate 58bit unsigned integer +%% Advance state and generate 58bit unsigned integer -spec exsplus_next(exsplus_state()) -> {uint58(), exsplus_state()}. exsplus_next([S1|S0]) -> %% Note: members s0 and s1 are swapped here - S11 = S1 bxor ?BSL(58, S1, 24), - S12 = S11 bxor S0 bxor (S11 bsr 11) bxor (S0 bsr 41), - {?MASK(58, S0 + S12), [S0|S12]}. + NewS1 = ?exs_next(S0, S1, S1_1), + {?MASK(58, S0 + NewS1), [S0|NewS1]}. +%% %% Note: members s0 and s1 are swapped here +%% S11 = S1 bxor ?BSL(58, S1, 24), +%% S12 = S11 bxor S0 bxor (S11 bsr 11) bxor (S0 bsr 41), +%% {?MASK(58, S0 + S12), [S0|S12]}. + +-dialyzer({no_improper_lists, exsss_next/1}). +-spec exsss_next(exsplus_state()) -> {uint58(), exsplus_state()}. +exsss_next([S1|S0]) -> + %% Note: members s0 and s1 are swapped here + NewS1 = ?exs_next(S0, S1, S1_1), + {?scramble_starstar(S0, V_0, V_1), [S0|NewS1]}. +%% {?MASK(58, S0 + NewS1), [S0|NewS1]}. exsp_uniform({Alg, R0}) -> {I, R1} = exsplus_next(R0), @@ -687,18 +795,48 @@ exsp_uniform({Alg, R0}) -> %% randomness quality than the others {(I bsr (58-53)) * ?TWO_POW_MINUS53, {Alg, R1}}. +exsss_uniform({Alg, R0}) -> + {I, R1} = exsss_next(R0), + {(I bsr (58-53)) * ?TWO_POW_MINUS53, {Alg, R1}}. + exsp_uniform(Range, {Alg, R}) -> {V, R1} = exsplus_next(R), MaxMinusRange = ?BIT(58) - Range, ?uniform_range(Range, Alg, R1, V, MaxMinusRange, I). +exsss_uniform(Range, {Alg, R}) -> + {V, R1} = exsss_next(R), + MaxMinusRange = ?BIT(58) - Range, + ?uniform_range(Range, Alg, R1, V, MaxMinusRange, I). -%% This is the jump function for the exsplus generator, equivalent + +%% This is the jump function for the exs* generators, +%% i.e the Xorshift116 generators, equivalent %% to 2^64 calls to next/1; it can be used to generate 2^52 %% non-overlapping subsequences for parallel computations. %% Note: the jump function takes 116 times of the execution time of %% next/1. - +%% +%% #include <stdint.h> +%% +%% void jump(void) { +%% static const uint64_t JUMP[] = { 0x02f8ea6bc32c797, +%% 0x345d2a0f85f788c }; +%% int i, b; +%% uint64_t s0 = 0; +%% uint64_t s1 = 0; +%% for(i = 0; i < sizeof JUMP / sizeof *JUMP; i++) +%% for(b = 0; b < 58; b++) { +%% if (JUMP[i] & 1ULL << b) { +%% s0 ^= s[0]; +%% s1 ^= s[1]; +%% } +%% next(); +%% } +%% s[0] = s0; +%% s[1] = s1; +%% } +%% %% -define(JUMPCONST, 16#000d174a83e17de2302f8ea6bc32c797). %% split into 58-bit chunks %% and two iterative executions @@ -708,7 +846,8 @@ exsp_uniform(Range, {Alg, R}) -> -define(JUMPELEMLEN, 58). -dialyzer({no_improper_lists, exsplus_jump/1}). --spec exsplus_jump(state()) -> state(). +-spec exsplus_jump({alg_handler(), exsplus_state()}) -> + {alg_handler(), exsplus_state()}. exsplus_jump({Alg, S}) -> {S1, AS1} = exsplus_jump(S, [0|0], ?JUMPCONST1, ?JUMPELEMLEN), {_, AS2} = exsplus_jump(S1, AS1, ?JUMPCONST2, ?JUMPELEMLEN), @@ -735,6 +874,12 @@ exsplus_jump(S, [AS0|AS1], J, N) -> -opaque exs1024_state() :: {list(uint64()), list(uint64())}. +exs1024_seed(L) when is_list(L) -> + {seed64_nz(16, L), []}; +exs1024_seed(X) when is_integer(X) -> + {seed64(16, ?MASK(64, X)), []}; +%% +%% Seed from traditional triple, remain backwards compatible exs1024_seed({A1, A2, A3}) -> B1 = ?MASK(21, (?MASK(21, A1) + 1) * 2097131), B2 = ?MASK(21, (?MASK(21, A2) + 1) * 2097133), @@ -806,8 +951,8 @@ exs1024_next({[H], RL}) -> -define(JUMPTOTALLEN, 1024). -define(RINGLEN, 16). --spec exs1024_jump(state()) -> state(). - +-spec exs1024_jump({alg_handler(), exs1024_state()}) -> + {alg_handler(), exs1024_state()}. exs1024_jump({Alg, {L, RL}}) -> P = length(RL), AS = exs1024_jump({L, RL}, @@ -832,6 +977,195 @@ exs1024_jump({L, RL}, AS, JL, J, N, TN) -> end. %% ===================================================================== +%% exro928ss PRNG: Xoroshiro928** +%% +%% Reference URL: http://vigna.di.unimi.it/ftp/papers/ScrambledLinear.pdf +%% i.e the Xoroshiro1024 generator with ** scrambler +%% with {S, R, T} = {5, 7, 9} as recommended in the paper. +%% +%% {A, B, C} were tried out and selected as {44, 9, 45} +%% and the jump coefficients calculated. +%% +%% Standard jump function pseudocode: +%% +%% Jump constant j = 0xb10773cb...44085302f77130ca +%% Generator state: s +%% New generator state: t = 0 +%% foreach bit in j, low to high: +%% if the bit is one: +%% t ^= s +%% next s +%% s = t +%% +%% Generator used for reference value calculation: +%% +%% #include <stdint.h> +%% #include <stdio.h> +%% +%% int p = 0; +%% uint64_t s[16]; +%% +%% #define MASK(x) ((x) & ((UINT64_C(1) << 58) - 1)) +%% static __inline uint64_t rotl(uint64_t x, int n) { +%% return MASK(x << n) | (x >> (58 - n)); +%% } +%% +%% uint64_t next() { +%% const int q = p; +%% const uint64_t s0 = s[p = (p + 1) & 15]; +%% uint64_t s15 = s[q]; +%% +%% const uint64_t result_starstar = MASK(rotl(MASK(s0 * 5), 7) * 9); +%% +%% s15 ^= s0; +%% s[q] = rotl(s0, 44) ^ s15 ^ MASK(s15 << 9); +%% s[p] = rotl(s15, 45); +%% +%% return result_starstar; +%% } +%% +%% static const uint64_t jump_2pow512[15] = +%% { 0x44085302f77130ca, 0xba05381fdfd14902, 0x10a1de1d7d6813d2, +%% 0xb83fe51a1eb3be19, 0xa81b0090567fd9f0, 0x5ac26d5d20f9b49f, +%% 0x4ddd98ee4be41e01, 0x0657e19f00d4b358, 0xf02f778573cf0f0a, +%% 0xb45a3a8a3cef3cc0, 0x6e62a33cc2323831, 0xbcb3b7c4cc049c53, +%% 0x83f240c6007e76ce, 0xe19f5fc1a1504acd, 0x00000000b10773cb }; +%% +%% static const uint64_t jump_2pow20[15] = +%% { 0xbdb966a3daf905e6, 0x644807a56270cf78, 0xda90f4a806c17e9e, +%% 0x4a426866bfad3c77, 0xaf699c306d8e7566, 0x8ebc73c700b8b091, +%% 0xc081a7bf148531fb, 0xdc4d3af15f8a4dfd, 0x90627c014098f4b6, +%% 0x06df2eb1feaf0fb6, 0x5bdeb1a5a90f2e6b, 0xa480c5878c3549bd, +%% 0xff45ef33c82f3d48, 0xa30bebc15fefcc78, 0x00000000cb3d181c }; +%% +%% void jump(const uint64_t *jump) { +%% uint64_t j, t[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; +%% int m, n, k; +%% for (m = 0; m < 15; m++, jump++) { +%% for (n = 0, j = *jump; n < 64; n++, j >>= 1) { +%% if ((j & 1) != 0) { +%% for (k = 0; k < 16; k++) { +%% t[k] ^= s[(p + k) & 15]; +%% } +%% } +%% next(); +%% } +%% } +%% for (k = 0; k < 16; k++) { +%% s[(p + k) & 15] = t[k]; +%% } +%% } +%% +%% ===================================================================== + +-opaque exro928_state() :: {list(uint58()), list(uint58())}. + +-spec exro928_seed( + list(uint58()) | integer() | {integer(), integer(), integer()}) -> + exro928_state(). +exro928_seed(L) when is_list(L) -> + {seed58_nz(16, L), []}; +exro928_seed(X) when is_integer(X) -> + {seed58(16, ?MASK(64, X)), []}; +%% +%% Seed from traditional integer triple - mix into splitmix +exro928_seed({A1, A2, A3}) -> + {S0, X0} = seed58(?MASK(64, A1)), + {S1, X1} = seed58(?MASK(64, A2) bxor X0), + {S2, X2} = seed58(?MASK(64, A3) bxor X1), + {[S0,S1,S2|seed58(13, X2)], []}. + + +%% Update the state and calculate output word +-spec exro928ss_next(exro928_state()) -> {uint58(), exro928_state()}. +exro928ss_next({[S15,S0|Ss], Rs}) -> + SR = exro928_next_state(Ss, Rs, S15, S0), + %% + %% {S, R, T} = {5, 7, 9} + %% const uint64_t result_starstar = rotl(s0 * S, R) * T; + %% + {?scramble_starstar(S0, V_0, V_1), SR}; +%% %% The multiply by add shifted trick avoids creating bignums +%% %% which improves performance significantly +%% %% +%% V0 = ?MASK(58, S0 + ?BSL(58, S0, 2)), % V0 = S0 * 5 +%% V1 = ?ROTL(58, V0, 7), +%% V = ?MASK(58, V1 + ?BSL(58, V1, 3)), % V = V1 * 9 +%% {V, SR}; +exro928ss_next({[S15], Rs}) -> + exro928ss_next({[S15|lists:reverse(Rs)], []}). + +-spec exro928_next(exro928_state()) -> {{uint58(),uint58()}, exro928_state()}. +exro928_next({[S15,S0|Ss], Rs}) -> + SR = exro928_next_state(Ss, Rs, S15, S0), + {{S15,S0}, SR}; +exro928_next({[S15], Rs}) -> + exro928_next({[S15|lists:reverse(Rs)], []}). + +%% Just update the state +-spec exro928_next_state(exro928_state()) -> exro928_state(). +exro928_next_state({[S15,S0|Ss], Rs}) -> + exro928_next_state(Ss, Rs, S15, S0); +exro928_next_state({[S15], Rs}) -> + [S0|Ss] = lists:reverse(Rs), + exro928_next_state(Ss, [], S15, S0). + +exro928_next_state(Ss, Rs, S15, S0) -> + %% {A, B, C} = {44, 9, 45}, + %% s15 ^= s0; + %% NewS15: s[q] = rotl(s0, A) ^ s15 ^ (s15 << B); + %% NewS0: s[p] = rotl(s15, C); + %% + Q = S15 bxor S0, + NewS15 = ?ROTL(58, S0, 44) bxor Q bxor ?BSL(58, Q, 9), + NewS0 = ?ROTL(58, Q, 45), + {[NewS0|Ss], [NewS15|Rs]}. + + +exro928ss_uniform({Alg, SR}) -> + {V, NewSR} = exro928ss_next(SR), + {(V bsr (58-53)) * ?TWO_POW_MINUS53, {Alg, NewSR}}. + +exro928ss_uniform(Range, {Alg, SR}) -> + {V, NewSR} = exro928ss_next(SR), + MaxMinusRange = ?BIT(58) - Range, + ?uniform_range(Range, Alg, NewSR, V, MaxMinusRange, I). + + +-spec exro928_jump({alg_handler(), exro928_state()}) -> + {alg_handler(), exro928_state()}. +exro928_jump({Alg, SR}) -> + {Alg,exro928_jump_2pow512(SR)}. + +-spec exro928_jump_2pow512(exro928_state()) -> exro928_state(). +exro928_jump_2pow512(SR) -> + polyjump( + SR, fun exro928_next_state/1, + %% 2^512 + [16#4085302F77130CA, 16#54E07F7F4524091, + 16#5E1D7D6813D2BA0, 16#4687ACEF8644287, + 16#4567FD9F0B83FE5, 16#43E6D27EA06C024, + 16#641E015AC26D5D2, 16#6CD61377663B92F, + 16#70A0657E19F00D4, 16#43C0BDDE15CF3C3, + 16#745A3A8A3CEF3CC, 16#58A8CF308C8E0C6, + 16#7B7C4CC049C536E, 16#431801F9DB3AF2C, + 16#41A1504ACD83F24, 16#6C41DCF2F867D7F]). + +-spec exro928_jump_2pow20(exro928_state()) -> exro928_state(). +exro928_jump_2pow20(SR) -> + polyjump( + SR, fun exro928_next_state/1, + %% 2^20 + [16#5B966A3DAF905E6, 16#601E9589C33DE2F, + 16#74A806C17E9E644, 16#59AFEB4F1DF6A43, + 16#46D8E75664A4268, 16#42E2C246BDA670C, + 16#4531FB8EBC73C70, 16#537F702069EFC52, + 16#4B6DC4D3AF15F8A, 16#5A4189F0050263D, + 16#46DF2EB1FEAF0FB, 16#77AC696A43CB9AC, + 16#4C5878C3549BD5B, 16#7CCF20BCF522920, + 16#415FEFCC78FF45E, 16#72CF460728C2FAF]). + +%% ===================================================================== %% exrop PRNG: Xoroshiro116+ %% %% Reference URL: http://xorshift.di.unimi.it/ @@ -899,6 +1233,15 @@ exs1024_jump({L, RL}, AS, JL, J, N, TN) -> -opaque exrop_state() :: nonempty_improper_list(uint58(), uint58()). -dialyzer({no_improper_lists, exrop_seed/1}). + +exrop_seed(L) when is_list(L) -> + [S0,S1] = seed58_nz(2, L), + [S0|S1]; +exrop_seed(X) when is_integer(X) -> + [S0,S1] = seed58(2, ?MASK(64, X)), + [S0|S1]; +%% +%% Traditional integer triplet seed exrop_seed({A1, A2, A3}) -> [_|S1] = exrop_next_s( @@ -962,6 +1305,142 @@ exrop_jump([S__0|S__1] = _S, S0, S1, J, Js) -> end. %% ===================================================================== +%% Mask and fill state list, ensure not all zeros +%% ===================================================================== + +seed58_nz(N, Ss) -> + seed_nz(N, Ss, 58, false). + +seed64_nz(N, Ss) -> + seed_nz(N, Ss, 64, false). + +seed_nz(_N, [], _M, false) -> + erlang:error(zero_seed); +seed_nz(0, [_|_], _M, _NZ) -> + erlang:error(too_many_seed_integers); +seed_nz(0, [], _M, _NZ) -> + []; +seed_nz(N, [], M, true) -> + [0|seed_nz(N - 1, [], M, true)]; +seed_nz(N, [S|Ss], M, NZ) -> + if + is_integer(S) -> + R = ?MASK(M, S), + [R|seed_nz(N - 1, Ss, M, NZ orelse R =/= 0)]; + true -> + erlang:error(non_integer_seed) + end. + +%% ===================================================================== +%% Splitmix seeders, lowest bits of SplitMix64, zeros skipped +%% ===================================================================== + +-spec seed58(non_neg_integer(), uint64()) -> list(uint58()). +seed58(0, _X) -> + []; +seed58(N, X) -> + {Z,NewX} = seed58(X), + [Z|seed58(N - 1, NewX)]. +%% +seed58(X_0) -> + {Z0,X} = splitmix64_next(X_0), + case ?MASK(58, Z0) of + 0 -> + seed58(X); + Z -> + {Z,X} + end. + +-spec seed64(non_neg_integer(), uint64()) -> list(uint64()). +seed64(0, _X) -> + []; +seed64(N, X) -> + {Z,NewX} = seed64(X), + [Z|seed64(N - 1, NewX)]. +%% +seed64(X_0) -> + {Z,X} = ZX = splitmix64_next(X_0), + if + Z =:= 0 -> + seed64(X); + true -> + ZX + end. + +%% The SplitMix64 generator: +%% +%% uint64_t splitmix64_next() { +%% uint64_t z = (x += 0x9e3779b97f4a7c15); +%% z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9; +%% z = (z ^ (z >> 27)) * 0x94d049bb133111eb; +%% return z ^ (z >> 31); +%% } +%% +splitmix64_next(X_0) -> + X = ?MASK(64, X_0 + 16#9e3779b97f4a7c15), + Z_0 = ?MASK(64, (X bxor (X bsr 30)) * 16#bf58476d1ce4e5b9), + Z_1 = ?MASK(64, (Z_0 bxor (Z_0 bsr 27)) * 16#94d049bb133111eb), + {?MASK(64, Z_1 bxor (Z_1 bsr 31)),X}. + +%% ===================================================================== +%% Polynomial jump with a jump constant word list, +%% high bit in each word marking top of word, +%% SR is a {Forward, Reverse} queue tuple with Forward never empty +%% ===================================================================== + +polyjump({Ss, Rs} = SR, NextState, JumpConst) -> + %% Create new state accumulator T + Ts = lists:duplicate(length(Ss) + length(Rs), 0), + polyjump(SR, NextState, JumpConst, Ts). +%% +%% Foreach jump word +polyjump(_SR, _NextState, [], Ts) -> + %% Return new calculated state + {Ts, []}; +polyjump(SR, NextState, [J|Js], Ts) -> + polyjump(SR, NextState, Js, Ts, J). +%% +%% Foreach bit in jump word until top bit +polyjump(SR, NextState, Js, Ts, 1) -> + polyjump(SR, NextState, Js, Ts); +polyjump({Ss, Rs} = SR, NextState, Js, Ts, J) when J =/= 0 -> + NewSR = NextState(SR), + NewJ = J bsr 1, + case ?MASK(1, J) of + 0 -> + polyjump(NewSR, NextState, Js, Ts, NewJ); + 1 -> + %% Xor this state onto T + polyjump(NewSR, NextState, Js, xorzip_sr(Ts, Ss, Rs), NewJ) + end. + +xorzip_sr([], [], undefined) -> + []; +xorzip_sr(Ts, [], Rs) -> + xorzip_sr(Ts, lists:reverse(Rs), undefined); +xorzip_sr([T|Ts], [S|Ss], Rs) -> + [T bxor S|xorzip_sr(Ts, Ss, Rs)]. + +%% ===================================================================== + +format_jumpconst58(String) -> + ReOpts = [{newline,any},{capture,all_but_first,binary},global], + {match,Matches} = re:run(String, "0x([a-zA-Z0-9]+)", ReOpts), + format_jumcons58_matches(lists:reverse(Matches), 0). + +format_jumcons58_matches([], J) -> + format_jumpconst58_value(J); +format_jumcons58_matches([[Bin]|Matches], J) -> + NewJ = (J bsl 64) bor binary_to_integer(Bin, 16), + format_jumcons58_matches(Matches, NewJ). + +format_jumpconst58_value(0) -> + ok; +format_jumpconst58_value(J) -> + io:format("16#~s,~n", [integer_to_list(?MASK(58, J) bor ?BIT(58), 16)]), + format_jumpconst58_value(J bsr 58). + +%% ===================================================================== %% Ziggurat cont %% ===================================================================== -define(NOR_R, 3.6541528853610087963519472518). diff --git a/lib/stdlib/src/slave.erl b/lib/stdlib/src/slave.erl index 5e8c1a43ea..4791e3ebda 100644 --- a/lib/stdlib/src/slave.erl +++ b/lib/stdlib/src/slave.erl @@ -104,18 +104,17 @@ relay1(Pid) -> %% this to work is that the 'erl' program can be found in PATH. %% %% If the master and slave are on different hosts, start/N uses -%% the 'rsh' program to spawn an Erlang node on the other host. +%% the 'ssh' program to spawn an Erlang node on the other host. %% Alternative, if the master was started as %% 'erl -sname xxx -rsh my_rsh...', then 'my_rsh' will be used instead -%% of 'rsh' (this is useful for systems where the rsh program is named -%% 'remsh'). +%% of 'ssh' (this is useful for systems still using rsh or remsh). %% %% For this to work, the following conditions must be fulfilled: %% -%% 1. There must be an Rsh program on computer; if not an error +%% 1. There must be an ssh program on computer; if not an error %% is returned. %% -%% 2. The hosts must be configured to allowed 'rsh' access without +%% 2. The hosts must be configured to allow 'ssh' access without %% prompts for password. %% %% The slave node will have its filer and user server redirected @@ -286,7 +285,7 @@ register_unique_name(Number) -> %% Makes up the command to start the nodes. %% If the node should run on the local host, there is -%% no need to use rsh. +%% no need to use a remote shell. mk_cmd(Host, Name, Args, Waiter, Prog0) -> Prog = quote_progname(Prog0), @@ -342,9 +341,7 @@ do_quote_progname([Prog,Arg|Args]) -> lists:flatten(lists:map(fun(X) -> [" ",X] end, [Arg|Args])) end. -%% Give the user an opportunity to run another program, -%% than the "rsh". On HP-UX rsh is called remsh; thus HP users -%% must start erlang as erl -rsh remsh. +%% Give the user an opportunity to run another program than "ssh". %% %% Also checks that the given program exists. %% @@ -354,7 +351,7 @@ rsh() -> Rsh = case init:get_argument(rsh) of {ok, [[Prog]]} -> Prog; - _ -> "rsh" + _ -> "ssh" end, case os:find_executable(Rsh) of false -> {error, no_rsh}; diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index cd09872b87..9cd425db9a 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -108,7 +108,7 @@ dets]}, {applications, [kernel]}, {env, []}, - {runtime_dependencies, ["sasl-3.0","kernel-6.0","erts-10.0","crypto-3.3", + {runtime_dependencies, ["sasl-3.0","kernel-6.0","erts-@OTP-15128@","crypto-3.3", "compiler-5.0"]} ]}. diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index 2939e78d9d..a418754caf 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -128,7 +128,8 @@ length(CD) -> to_graphemes(CD0) -> case unicode_util:gc(CD0) of [GC|CD] -> [GC|to_graphemes(CD)]; - [] -> [] + [] -> []; + {error, Err} -> error({badarg, Err}) end. %% Compare two strings return boolean, assumes that the input are @@ -332,7 +333,10 @@ uppercase(<<CP1/utf8, Rest/binary>>=Orig) -> catch unchanged -> Orig end; uppercase(<<>>) -> - <<>>. + <<>>; +uppercase(Bin) -> + error({badarg, Bin}). + %% Lowercase all chars in Str -spec lowercase(String::unicode:chardata()) -> unicode:chardata(). @@ -346,7 +350,10 @@ lowercase(<<CP1/utf8, Rest/binary>>=Orig) -> catch unchanged -> Orig end; lowercase(<<>>) -> - <<>>. + <<>>; +lowercase(Bin) -> + error({badarg, Bin}). + %% Make a titlecase of the first char in Str -spec titlecase(String::unicode:chardata()) -> unicode:chardata(). @@ -375,7 +382,9 @@ casefold(<<CP1/utf8, Rest/binary>>=Orig) -> catch unchanged -> Orig end; casefold(<<>>) -> - <<>>. + <<>>; +casefold(Bin) -> + error({badarg, Bin}). -spec to_integer(String) -> {Int, Rest} | {'error', Reason} when String :: unicode:chardata(), @@ -544,7 +553,8 @@ length_1([CP1|[CP2|_]=Cont], N) when ?ASCII_LIST(CP1,CP2) -> length_1(Str, N) -> case unicode_util:gc(Str) of [] -> N; - [_|Rest] -> length_1(Rest, N+1) + [_|Rest] -> length_1(Rest, N+1); + {error, Err} -> error({badarg, Err}) end. length_b(<<CP2/utf8, Rest/binary>>, CP1, N) @@ -554,7 +564,8 @@ length_b(Bin0, CP1, N) -> [_|Bin1] = unicode_util:gc([CP1|Bin0]), case unicode_util:cp(Bin1) of [] -> N+1; - [CP3|Bin] -> length_b(Bin, CP3, N+1) + [CP3|Bin] -> length_b(Bin, CP3, N+1); + {error, Err} -> error({badarg, Err}) end. equal_1([A|AR], [B|BR]) when is_integer(A), is_integer(B) -> @@ -599,7 +610,8 @@ reverse_1([CP1|[CP2|_]=Cont], Acc) when ?ASCII_LIST(CP1,CP2) -> reverse_1(CD, Acc) -> case unicode_util:gc(CD) of [GC|Rest] -> reverse_1(Rest, [GC|Acc]); - [] -> Acc + [] -> Acc; + {error, Err} -> error({badarg, Err}) end. reverse_b(<<CP2/utf8, Rest/binary>>, CP1, Acc) @@ -609,7 +621,8 @@ reverse_b(Bin0, CP1, Acc) -> [GC|Bin1] = unicode_util:gc([CP1|Bin0]), case unicode_util:cp(Bin1) of [] -> [GC|Acc]; - [CP3|Bin] -> reverse_b(Bin, CP3, [GC|Acc]) + [CP3|Bin] -> reverse_b(Bin, CP3, [GC|Acc]); + {error, Err} -> error({badarg, Err}) end. slice_l0(<<CP1/utf8, Bin/binary>>, N) when N > 0 -> @@ -622,7 +635,8 @@ slice_l([CP1|[CP2|_]=Cont], N) when ?ASCII_LIST(CP1,CP2),N > 0 -> slice_l(CD, N) when N > 0 -> case unicode_util:gc(CD) of [_|Cont] -> slice_l(Cont, N-1); - [] -> [] + [] -> []; + {error, Err} -> error({badarg, Err}) end; slice_l(Cont, 0) -> Cont. @@ -634,7 +648,8 @@ slice_lb(Bin, CP1, N) -> if N > 1 -> case unicode_util:cp(Rest) of [CP2|Cont] -> slice_lb(Cont, CP2, N-1); - [] -> <<>> + [] -> <<>>; + {error, Err} -> error({badarg, Err}) end; N =:= 1 -> Rest @@ -647,7 +662,10 @@ slice_trail(Orig, N) when is_binary(Orig) -> Sz = byte_size(Orig) - Length, <<Keep:Sz/binary, _/binary>> = Orig, Keep; - _ -> <<>> + <<_, _/binary>> when N > 0 -> + error({badarg, Orig}); + _ -> + <<>> end; slice_trail(CD, N) when is_list(CD) -> slice_list(CD, N). @@ -657,7 +675,8 @@ slice_list([CP1|[CP2|_]=Cont], N) when ?ASCII_LIST(CP1,CP2),N > 0 -> slice_list(CD, N) when N > 0 -> case unicode_util:gc(CD) of [GC|Cont] -> append(GC, slice_list(Cont, N-1)); - [] -> [] + [] -> []; + {error, Err} -> error({badarg, Err}) end; slice_list(_, 0) -> []. @@ -668,7 +687,8 @@ slice_bin(CD, CP1, N) when N > 0 -> [_|Bin] = unicode_util:gc([CP1|CD]), case unicode_util:cp(Bin) of [CP2|Cont] -> slice_bin(Cont, CP2, N-1); - [] -> 0 + [] -> 0; + {error, Err} -> error({badarg, Err}) end; slice_bin(CD, CP1, 0) -> byte_size(CD)+byte_size(<<CP1/utf8>>). @@ -703,14 +723,18 @@ uppercase_bin(CP1, Bin, Changed) -> [] when Changed -> [CP1]; [] -> - throw(unchanged) + throw(unchanged); + {error, Err} -> + error({badarg, Err}) end; [Char|CPs] -> case unicode_util:cp(CPs) of [Next|Rest] -> [Char|uppercase_bin(Next, Rest, true)]; [] -> - [Char] + [Char]; + {error, Err} -> + error({badarg, Err}) end end. @@ -744,14 +768,18 @@ lowercase_bin(CP1, Bin, Changed) -> [] when Changed -> [CP1]; [] -> - throw(unchanged) + throw(unchanged); + {error, Err} -> + error({badarg, Err}) end; [Char|CPs] -> case unicode_util:cp(CPs) of [Next|Rest] -> [Char|lowercase_bin(Next, Rest, true)]; [] -> - [Char] + [Char]; + {error, Err} -> + error({badarg, Err}) end end. @@ -785,14 +813,18 @@ casefold_bin(CP1, Bin, Changed) -> [] when Changed -> [CP1]; [] -> - throw(unchanged) + throw(unchanged); + {error, Err} -> + error({badarg, Err}) end; [Char|CPs] -> case unicode_util:cp(CPs) of [Next|Rest] -> [Char|casefold_bin(Next, Rest, true)]; [] -> - [Char] + [Char]; + {error, Err} -> + error({badarg, Err}) end end. @@ -1247,18 +1279,20 @@ split_1(Bin, [_C|_]=Needle, Start, Where, Curr0, Acc) -> end end. -lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps, Ts) when is_integer(CP) -> +lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps0, Ts) when is_integer(CP) -> case lists:member(CP, CPs) of true -> [GC|Cs2] = unicode_util:gc(Cs0), case lists:member(GC, GCs) of true -> - lexemes_m(Cs2, Seps, Ts); + lexemes_m(Cs2, Seps0, Ts); false -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; false -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; @@ -1632,7 +1666,9 @@ bin_search_inv_1(<<CP1/utf8, BinRest/binary>>=Bin0, Cont, Sep) -> bin_search_inv_1(<<>>, Cont, _Sep) -> {nomatch, Cont}; bin_search_inv_1([], Cont, _Sep) -> - {nomatch, Cont}. + {nomatch, Cont}; +bin_search_inv_1(Bin, _, _) -> + error({badarg, Bin}). bin_search_inv_n(<<CP1/utf8, BinRest/binary>>=Bin0, Cont, Seps) -> @@ -1664,7 +1700,9 @@ bin_search_inv_n(<<CP1/utf8, BinRest/binary>>=Bin0, Cont, Seps) -> bin_search_inv_n(<<>>, Cont, _Sep) -> {nomatch, Cont}; bin_search_inv_n([], Cont, _Sep) -> - {nomatch, Cont}. + {nomatch, Cont}; +bin_search_inv_n(Bin, _, _) -> + error({badarg, Bin}). bin_search_str(Bin0, Start, [], SearchCPs) -> Compiled = binary:compile_pattern(unicode:characters_to_binary(SearchCPs)), diff --git a/lib/stdlib/src/sys.erl b/lib/stdlib/src/sys.erl index 0064414d6f..6ff9aa33b4 100644 --- a/lib/stdlib/src/sys.erl +++ b/lib/stdlib/src/sys.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -30,22 +30,30 @@ log_to_file/2, log_to_file/3, no_debug/1, no_debug/2, install/2, install/3, remove/2, remove/3]). -export([handle_system_msg/6, handle_system_msg/7, handle_debug/4, - print_log/1, get_debug/3, debug_options/1, suspend_loop_hib/6]). + print_log/1, get_log/1, get_debug/3, debug_options/1, suspend_loop_hib/6]). +-deprecated([{get_debug,3,eventually}]). %%----------------------------------------------------------------- %% Types %%----------------------------------------------------------------- --export_type([dbg_opt/0]). +-export_type([dbg_opt/0, dbg_fun/0, debug_option/0]). -type name() :: pid() | atom() | {'global', term()} | {'via', module(), term()}. -type system_event() :: {'in', Msg :: _} - | {'in', Msg :: _, From :: _} + | {'in', Msg :: _, State :: _} | {'out', Msg :: _, To :: _} | {'out', Msg :: _, To :: _, State :: _} - | term(). + | {'noreply', State :: _} + | {'continue', Continuation :: _} + | {'code_change', Event :: _, State :: _} + | {'postpone', Event :: _, State :: _, NextState :: _} + | {'consume', Event :: _, State :: _, NextState :: _} + | {'enter', State :: _} + | {'terminate', Reason :: _, State :: _} + | term(). -opaque dbg_opt() :: {'trace', 'true'} | {'log', {N :: non_neg_integer(), @@ -67,6 +75,16 @@ Event :: system_event(), Extra :: term()) -> any()). +-type debug_option() :: + 'trace' + | 'log' + | {'log', N :: pos_integer()} + | 'statistics' + | {'log_to_file', FileName :: file:name()} + | {'install', + {Func :: dbg_fun(), FuncState :: term()} + | {FuncId :: term(), Func :: dbg_fun(), FuncState :: term()}}. + %%----------------------------------------------------------------- %% System messages %%----------------------------------------------------------------- @@ -385,31 +403,41 @@ handle_system_msg(SysState, Msg, From, Parent, Mod, Debug, Misc, Hib) -> FormFunc :: format_fun(), Extra :: term(), Event :: system_event(). -handle_debug([{trace, true} | T], FormFunc, State, Event) -> +handle_debug([{trace, true} = DbgOpt | T], FormFunc, State, Event) -> print_event({Event, State, FormFunc}), - [{trace, true} | handle_debug(T, FormFunc, State, Event)]; -handle_debug([{log, {N, LogData}} | T], FormFunc, State, Event) -> - NLogData = [{Event, State, FormFunc} | trim(N, LogData)], - [{log, {N, NLogData}} | handle_debug(T, FormFunc, State, Event)]; -handle_debug([{log_to_file, Fd} | T], FormFunc, State, Event) -> + [DbgOpt | handle_debug(T, FormFunc, State, Event)]; +handle_debug([{log, NLog} | T], FormFunc, State, Event) -> + Item = {Event, State, FormFunc}, + [{log, nlog_put(Item, NLog)} | handle_debug(T, FormFunc, State, Event)]; +handle_debug([{log_to_file, Fd} = DbgOpt | T], FormFunc, State, Event) -> print_event(Fd, {Event, State, FormFunc}), - [{log_to_file, Fd} | handle_debug(T, FormFunc, State, Event)]; + [DbgOpt | handle_debug(T, FormFunc, State, Event)]; handle_debug([{statistics, StatData} | T], FormFunc, State, Event) -> NStatData = stat(Event, StatData), [{statistics, NStatData} | handle_debug(T, FormFunc, State, Event)]; handle_debug([{FuncId, {Func, FuncState}} | T], FormFunc, State, Event) -> - case catch Func(FuncState, Event, State) of + try Func(FuncState, Event, State) of + done -> handle_debug(T, FormFunc, State, Event); + NFuncState -> + [{FuncId, {Func, NFuncState}} | + handle_debug(T, FormFunc, State, Event)] + catch done -> handle_debug(T, FormFunc, State, Event); - {'EXIT', _} -> handle_debug(T, FormFunc, State, Event); NFuncState -> - [{FuncId, {Func, NFuncState}} | handle_debug(T, FormFunc, State, Event)] + [{FuncId, {Func, NFuncState}} | + handle_debug(T, FormFunc, State, Event)]; + _:_ -> handle_debug(T, FormFunc, State, Event) end; handle_debug([{Func, FuncState} | T], FormFunc, State, Event) -> - case catch Func(FuncState, Event, State) of + try Func(FuncState, Event, State) of done -> handle_debug(T, FormFunc, State, Event); - {'EXIT', _} -> handle_debug(T, FormFunc, State, Event); - NFuncState -> + NFuncState -> [{Func, NFuncState} | handle_debug(T, FormFunc, State, Event)] + catch + done -> handle_debug(T, FormFunc, State, Event); + NFuncState -> + [{Func, NFuncState} | handle_debug(T, FormFunc, State, Event)]; + _:_ -> handle_debug(T, FormFunc, State, Event) end; handle_debug([], _FormFunc, _State, _Event) -> []. @@ -526,19 +554,19 @@ debug_cmd({trace, true}, Debug) -> debug_cmd({trace, false}, Debug) -> {ok, remove_debug(trace, Debug)}; debug_cmd({log, true}, Debug) -> - {_N, Logs} = get_debug(log, Debug, {0, []}), - {ok, install_debug(log, {10, trim(10, Logs)}, Debug)}; -debug_cmd({log, {true, N}}, Debug) when is_integer(N), N > 0 -> - {_N, Logs} = get_debug(log, Debug, {0, []}), - {ok, install_debug(log, {N, trim(N, Logs)}, Debug)}; + NLog = get_debug(log, Debug, nlog_new()), + {ok, install_debug(log, nlog_new(NLog), Debug)}; +debug_cmd({log, {true, N}}, Debug) when is_integer(N), 1 =< N -> + NLog = get_debug(log, Debug, nlog_new(N)), + {ok, install_debug(log, nlog_new(N, NLog), Debug)}; debug_cmd({log, false}, Debug) -> {ok, remove_debug(log, Debug)}; debug_cmd({log, print}, Debug) -> print_log(Debug), {ok, Debug}; debug_cmd({log, get}, Debug) -> - {_N, Logs} = get_debug(log, Debug, {0, []}), - {{ok, lists:reverse(Logs)}, Debug}; + NLog = get_debug(log, Debug, nlog_new()), + {{ok, [Event || {Event, _State, _FormFunc} <- nlog_get(NLog)]}, Debug}; debug_cmd({log_to_file, false}, Debug) -> NDebug = close_log_file(Debug), {ok, NDebug}; @@ -595,9 +623,6 @@ stat({out, _Msg, _To}, {Time, Reds, In, Out}) -> {Time, Reds, In, Out+1}; stat({out, _Msg, _To, _State}, {Time, Reds, In, Out}) -> {Time, Reds, In, Out+1}; stat(_, StatData) -> StatData. -trim(N, LogData) -> - lists:sublist(LogData, 1, N-1). - %%----------------------------------------------------------------- %% Debug structure manipulating functions %%----------------------------------------------------------------- @@ -625,9 +650,14 @@ get_debug2(Item, Debug, Default) -> -spec print_log(Debug) -> 'ok' when Debug :: [dbg_opt()]. print_log(Debug) -> - {_N, Logs} = get_debug(log, Debug, {0, []}), - lists:foreach(fun print_event/1, - lists:reverse(Logs)). + NLog = get_debug(log, Debug, nlog_new()), + lists:foreach(fun print_event/1, nlog_get(NLog)). + +-spec get_log(Debug) -> [system_event()] when + Debug :: [dbg_opt()]. +get_log(Debug) -> + NLog = get_debug(log, Debug, nlog_new()), + [Event || {Event, _State, _FormFunc} <- nlog_get(NLog)]. close_log_file(Debug) -> case get_debug2(log_to_file, Debug, []) of @@ -639,6 +669,74 @@ close_log_file(Debug) -> end. %%----------------------------------------------------------------- +%% Keep the last N Log functions +%%----------------------------------------------------------------- +%% +%% Streamlined Okasaki queue as base for "keep the last N" log. +%% +%% To the reverse list head we cons new items. +%% The forward list contains elements in insertion order, +%% so the head is the oldest and the one to drop off +%% when the log is full. +%% +%% Here is how we can get away with only using one cons cell +%% to wrap the forward and reverse list, and the log size: +%% +%% A full log does not need a counter; we just cons one +%% and drop one: +%% +%% [ReverseList|ForwardList] +%% +%% A non-full log is filling up to N elements; +%% use a down counter instead of a list as first element: +%% +%% [RemainingToFullCount|ReverseList] + +nlog_new() -> + nlog_new(10). +%% +nlog_new([_|_] = NLog) -> + nlog_new(10, NLog); +nlog_new(N) -> + [N]. % Empty log size N >= 1 +%% +nlog_new(N, NLog) -> + lists:foldl( + fun (Item, NL) -> nlog_put(Item, NL) end, + nlog_new(N), + nlog_get(NLog)). + +%% +nlog_put(Item, NLog) -> + case NLog of + [R|FF] when is_list(R) -> + %% Full log + case FF of + [_|F] -> + %% Cons to reverse list, drop from forward list + [[Item|R]|F]; + [] -> + %% Create new forward list from reverse list, + %% create new empty reverse list + [_|F] = lists:reverse(R, [Item]), + [[]|F] + end; + [1|R] -> + %% Log now gets full + [[Item|R]]; + [J|R] -> + %% Filling up to N elements + [J - 1,Item|R] + end. + +nlog_get([[]|F]) -> + F; +nlog_get([[_|_] = R|F]) -> + F ++ lists:reverse(R); +nlog_get([_J|R]) -> + lists:reverse(R). + +%%----------------------------------------------------------------- %% Func: debug_options/1 %% Purpose: Initiate a debug structure. Called by a process that %% wishes to initiate the debug structure without the @@ -646,28 +744,16 @@ close_log_file(Debug) -> %% Returns: [debug_opts()] %%----------------------------------------------------------------- --spec debug_options(Options) -> [dbg_opt()] when - Options :: [Opt], - Opt :: 'trace' - | 'log' - | {'log', pos_integer()} - | 'statistics' - | {'log_to_file', FileName} - | {'install', FuncSpec}, - FileName :: file:name(), - FuncSpec :: {Func, FuncState} | {FuncId, Func, FuncState}, - FuncId :: term(), - Func :: dbg_fun(), - FuncState :: term(). +-spec debug_options([Opt :: debug_option()]) -> [dbg_opt()]. debug_options(Options) -> debug_options(Options, []). debug_options([trace | T], Debug) -> debug_options(T, install_debug(trace, true, Debug)); debug_options([log | T], Debug) -> - debug_options(T, install_debug(log, {10, []}, Debug)); + debug_options(T, install_debug(log, nlog_new(), Debug)); debug_options([{log, N} | T], Debug) when is_integer(N), N > 0 -> - debug_options(T, install_debug(log, {N, []}, Debug)); + debug_options(T, install_debug(log, nlog_new(N), Debug)); debug_options([statistics | T], Debug) -> debug_options(T, install_debug(statistics, init_stat(), Debug)); debug_options([{log_to_file, FileName} | T], Debug) -> diff --git a/lib/stdlib/src/uri_string.erl b/lib/stdlib/src/uri_string.erl index f07307c039..d33dc89af8 100644 --- a/lib/stdlib/src/uri_string.erl +++ b/lib/stdlib/src/uri_string.erl @@ -415,7 +415,7 @@ transcode(URIString, Options) when is_list(URIString) -> %% (application/x-www-form-urlencoded encoding algorithm) %%------------------------------------------------------------------------- -spec compose_query(QueryList) -> QueryString when - QueryList :: [{unicode:chardata(), unicode:chardata()}], + QueryList :: [{unicode:chardata(), unicode:chardata() | true}], QueryString :: uri_string() | error(). compose_query(List) -> @@ -423,7 +423,7 @@ compose_query(List) -> -spec compose_query(QueryList, Options) -> QueryString when - QueryList :: [{unicode:chardata(), unicode:chardata()}], + QueryList :: [{unicode:chardata(), unicode:chardata() | true}], Options :: [{encoding, atom()}], QueryString :: uri_string() | error(). @@ -435,6 +435,11 @@ compose_query(List, Options) -> throw:{error, Atom, RestData} -> {error, Atom, RestData} end. %% +compose_query([{Key,true}|Rest], Options, IsList, Acc) -> + Separator = get_separator(Rest), + K = form_urlencode(Key, Options), + IsListNew = IsList orelse is_list(Key), + compose_query(Rest, Options, IsListNew, <<Acc/binary,K/binary,Separator/binary>>); compose_query([{Key,Value}|Rest], Options, IsList, Acc) -> Separator = get_separator(Rest), K = form_urlencode(Key, Options), @@ -454,7 +459,7 @@ compose_query([], _Options, IsList, Acc) -> %%------------------------------------------------------------------------- -spec dissect_query(QueryString) -> QueryList when QueryString :: uri_string(), - QueryList :: [{unicode:chardata(), unicode:chardata()}] + QueryList :: [{unicode:chardata(), unicode:chardata() | true}] | error(). dissect_query(<<>>) -> []; @@ -1889,13 +1894,12 @@ dissect_query_key(<<$=,T/binary>>, IsList, Acc, Key, Value) -> dissect_query_value(T, IsList, Acc, Key, Value); dissect_query_key(<<"&#",T/binary>>, IsList, Acc, Key, Value) -> dissect_query_key(T, IsList, Acc, <<Key/binary,"&#">>, Value); -dissect_query_key(<<$&,_T/binary>>, _IsList, _Acc, _Key, _Value) -> - throw({error, missing_value, "&"}); +dissect_query_key(T = <<$&,_/binary>>, IsList, Acc, Key, <<>>) -> + dissect_query_value(T, IsList, Acc, Key, true); dissect_query_key(<<H,T/binary>>, IsList, Acc, Key, Value) -> dissect_query_key(T, IsList, Acc, <<Key/binary,H>>, Value); -dissect_query_key(B, _, _, _, _) -> - throw({error, missing_value, B}). - +dissect_query_key(T = <<>>, IsList, Acc, Key, <<>>) -> + dissect_query_value(T, IsList, Acc, Key, true). dissect_query_value(<<$&,T/binary>>, IsList, Acc, Key, Value) -> K = form_urldecode(IsList, Key), @@ -1908,9 +1912,10 @@ dissect_query_value(<<>>, IsList, Acc, Key, Value) -> V = form_urldecode(IsList, Value), lists:reverse([{K,V}|Acc]). - %% HTML 5.2 - 4.10.21.6 URL-encoded form data - WHATWG URL (10 Jan 2018) - UTF-8 %% HTML 5.0 - 4.10.22.6 URL-encoded form data - decoding (non UTF-8) +form_urldecode(_, true) -> + true; form_urldecode(true, B) -> Result = base10_decode(form_urldecode(B, <<>>)), convert_to_list(Result, utf8); diff --git a/lib/stdlib/test/Makefile b/lib/stdlib/test/Makefile index bbe3cefa42..712b1b92fb 100644 --- a/lib/stdlib/test/Makefile +++ b/lib/stdlib/test/Makefile @@ -99,11 +99,9 @@ MODULES= \ maps_SUITE \ zzz_SUITE -ERL_FILES= $(MODULES:%=%.erl) +ERTS_MODULES= erts_test_utils -TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) - -INSTALL_PROGS= $(TARGET_FILES) +ERL_FILES= $(MODULES:%=%.erl) $(ERTS_MODULES:%=$(ERL_TOP)/erts/emulator/test/%.erl) # ---------------------------------------------------- # Release directory specification @@ -128,7 +126,7 @@ COVERFILE=stdlib.cover # ---------------------------------------------------- make_emakefile: - $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) \ + $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) $(ERTS_MODULES) \ > $(EMAKEFILE) tests debug opt: make_emakefile diff --git a/lib/stdlib/test/beam_lib_SUITE.erl b/lib/stdlib/test/beam_lib_SUITE.erl index 6418dc7eb6..4b2694320e 100644 --- a/lib/stdlib/test/beam_lib_SUITE.erl +++ b/lib/stdlib/test/beam_lib_SUITE.erl @@ -35,7 +35,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, - normal/1, error/1, cmp/1, cmp_literals/1, strip/1, otp_6711/1, + normal/1, error/1, cmp/1, cmp_literals/1, strip/1, strip_add_chunks/1, otp_6711/1, building/1, md5/1, encrypted_abstr/1, encrypted_abstr_file/1]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -45,7 +45,7 @@ suite() -> {timetrap,{minutes,2}}]. all() -> - [error, normal, cmp, cmp_literals, strip, otp_6711, + [error, normal, cmp, cmp_literals, strip, strip_add_chunks, otp_6711, building, md5, encrypted_abstr, encrypted_abstr_file]. groups() -> @@ -401,6 +401,69 @@ strip(Conf) when is_list(Conf) -> Source5D1, BeamFile5D1]), ok. +strip_add_chunks(Conf) when is_list(Conf) -> + PrivDir = ?privdir, + {SourceD1, BeamFileD1} = make_beam(PrivDir, simple, member), + {Source2D1, BeamFile2D1} = make_beam(PrivDir, simple2, concat), + {Source3D1, BeamFile3D1} = make_beam(PrivDir, make_fun, make_fun), + {Source4D1, BeamFile4D1} = make_beam(PrivDir, constant, constant), + {Source5D1, BeamFile5D1} = make_beam(PrivDir, lines, lines), + + NoOfTables = erlang:system_info(ets_count), + P0 = pps(), + + %% strip binary + verify(not_a_beam_file, beam_lib:strip(<<>>)), + {ok, B1} = file:read_file(BeamFileD1), + {ok, {simple, NB1}} = beam_lib:strip(B1), + + BId1 = chunk_ids(B1), + NBId1 = chunk_ids(NB1), + true = length(BId1) > length(NBId1), + compare_chunks(B1, NB1, NBId1), + + %% Keep all the extra chunks + ExtraChunks = ["Abst" , "Dbgi" , "Attr" , "CInf" , "LocT" , "Atom" ], + {ok, {simple, AB1}} = beam_lib:strip(B1, ExtraChunks), + ABId1 = chunk_ids(AB1), + true = length(BId1) == length(ABId1), + compare_chunks(B1, AB1, ABId1), + + %% strip file - Keep extra chunks + verify(file_error, beam_lib:strip(foo)), + {ok, {simple, _}} = beam_lib:strip(BeamFileD1, ExtraChunks), + compare_chunks(B1, BeamFileD1, ABId1), + + %% strip_files + {ok, B2} = file:read_file(BeamFile2D1), + {ok, [{simple,_},{simple2,_}]} = beam_lib:strip_files([B1, B2], ExtraChunks), + {ok, [{simple,_},{simple2,_},{make_fun,_},{constant,_}]} = + beam_lib:strip_files([BeamFileD1, BeamFile2D1, BeamFile3D1, BeamFile4D1], ExtraChunks), + + %% check that each module can be loaded. + {module, simple} = code:load_abs(filename:rootname(BeamFileD1)), + {module, simple2} = code:load_abs(filename:rootname(BeamFile2D1)), + {module, make_fun} = code:load_abs(filename:rootname(BeamFile3D1)), + {module, constant} = code:load_abs(filename:rootname(BeamFile4D1)), + + %% check that line number information is still present after stripping + {module, lines} = code:load_abs(filename:rootname(BeamFile5D1)), + {'EXIT',{badarith,[{lines,t,1,Info}|_]}} = (catch lines:t(atom)), + false = code:purge(lines), + true = code:delete(lines), + {ok, {lines,BeamFile5D1}} = beam_lib:strip(BeamFile5D1), + {module, lines} = code:load_abs(filename:rootname(BeamFile5D1)), + {'EXIT',{badarith,[{lines,t,1,Info}|_]}} = (catch lines:t(atom)), + + true = (P0 == pps()), + NoOfTables = erlang:system_info(ets_count), + + delete_files([SourceD1, BeamFileD1, + Source2D1, BeamFile2D1, + Source3D1, BeamFile3D1, + Source4D1, BeamFile4D1, + Source5D1, BeamFile5D1]), + ok. otp_6711(Conf) when is_list(Conf) -> {'EXIT',{function_clause,_}} = (catch {a, beam_lib:info(3)}), @@ -729,6 +792,7 @@ make_beam(Dir, Module, F) -> FileBase = filename:join(Dir, atom_to_list(Module)), Source = FileBase ++ ".erl", BeamFile = FileBase ++ ".beam", + file:delete(BeamFile), simple_file(Source, Module, F), {ok, _} = compile:file(Source, [{outdir,Dir}, debug_info, report]), {Source, BeamFile}. diff --git a/lib/stdlib/test/binary_module_SUITE.erl b/lib/stdlib/test/binary_module_SUITE.erl index e0811f19cf..9b2033ec4a 100644 --- a/lib/stdlib/test/binary_module_SUITE.erl +++ b/lib/stdlib/test/binary_module_SUITE.erl @@ -755,8 +755,9 @@ list_to_bin(Config) when is_list(Config) -> copy(Config) when is_list(Config) -> <<1,2,3>> = binary:copy(<<1,2,3>>), RS = random_string({1,10000}), - RS = RS2 = binary:copy(RS), - false = erts_debug:same(RS,RS2), + RS2 = binary:copy(RS), + true = RS =:= RS2, + false = erts_debug:same(RS, RS2), <<>> = ?MASK_ERROR(binary:copy(<<1,2,3>>,0)), badarg = ?MASK_ERROR(binary:copy(<<1,2,3:3>>,2)), badarg = ?MASK_ERROR(binary:copy([],0)), diff --git a/lib/stdlib/test/epp_SUITE.erl b/lib/stdlib/test/epp_SUITE.erl index 0ac99ad03a..a90beed4f3 100644 --- a/lib/stdlib/test/epp_SUITE.erl +++ b/lib/stdlib/test/epp_SUITE.erl @@ -1372,7 +1372,7 @@ otp_8562(Config) when is_list(Config) -> otp_8911(Config) when is_list(Config) -> case test_server:is_cover() of true -> - {skip, "Testing cover, so can not run when cover is already running"}; + {skip, "Testing cover, so cannot run when cover is already running"}; false -> do_otp_8911(Config) end. diff --git a/lib/stdlib/test/erl_eval_SUITE.erl b/lib/stdlib/test/erl_eval_SUITE.erl index f4019d477b..2436c8091c 100644 --- a/lib/stdlib/test/erl_eval_SUITE.erl +++ b/lib/stdlib/test/erl_eval_SUITE.erl @@ -1156,7 +1156,17 @@ simple() -> {A}. simple1() -> - erlang:error(simple). + %% If the compiler could see that this function would always + %% throw an error exception, it would rewrite simple() like this: + %% + %% simple() -> simple1(). + %% + %% That would change the stacktrace. To prevent the compiler from + %% doing that optimization, we must obfuscate the code. + case get(a_key_that_is_not_defined) of + undefined -> erlang:error(simple); + WillNeverHappen -> WillNeverHappen + end. %% Simple cases, just to cover some code. funs(Config) when is_list(Config) -> diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index d18c7c255c..fe98a3796d 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -68,7 +68,7 @@ non_latin1_module/1, otp_14323/1, stacktrace_syntax/1, otp_14285/1, otp_14378/1, - external_funs/1]). + external_funs/1,otp_15456/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -89,7 +89,8 @@ all() -> maps, maps_type, maps_parallel_match, otp_11851, otp_11879, otp_13230, record_errors, otp_11879_cont, non_latin1_module, otp_14323, - stacktrace_syntax, otp_14285, otp_14378, external_funs]. + stacktrace_syntax, otp_14285, otp_14378, external_funs, + otp_15456]. groups() -> [{unused_vars_warn, [], @@ -2108,17 +2109,93 @@ otp_5362(Config) when is_list(Config) -> {calendar,local_time_to_universal_time_dst,1}, "a future release"}}]}}, {call_removed_function, - <<"t(X) -> regexp:match(X).">>, + <<"t(X) -> erlang:hash(X, 10000).">>, [], {warnings, - [{1,erl_lint,{removed,{regexp,match,1}, - "removed in R15; use the re module instead"}}]}} + [{1,erl_lint,{removed,{erlang,hash,2},{erlang,phash2,2},"20.0"}}]}}, + + {nowarn_call_removed_function_1, + <<"t(X) -> erlang:hash(X, 10000).">>, + [{nowarn_removed,{erlang,hash,2}}], + []}, + + {nowarn_call_removed_function_2, + <<"t(X) -> os_mon_mib:any_function_really(erlang:hash(X, 10000)).">>, + [nowarn_removed], + []}, + + {call_removed_module, + <<"t(X) -> os_mon_mib:any_function_really(X).">>, + [], + {warnings,[{1,erl_lint, + {removed,{os_mon_mib,any_function_really,1}, + "was removed in 22.0"}}]}}, + + {nowarn_call_removed_module, + <<"t(X) -> os_mon_mib:any_function_really(X).">>, + [{nowarn_removed,os_mon_mib}], + []} ], [] = run(Config, Ts), ok. +%% OTP-15456. All compiler options can now be given in the option list +%% (as opposed to only in files). +otp_15456(Config) when is_list(Config) -> + Ts = [ + %% {nowarn_deprecated_function,[{M,F,A}]} can now be given + %% in the option list as well as in an attribute. + %% Wherever it occurs, it is not affected by + %% warn_deprecated_function. + {otp_15456_1, + <<"-compile({nowarn_deprecated_function,{erlang,now,0}}). + -export([foo/0]). + + foo() -> + {erlang:now(), random:seed0(), random:seed(1, 2, 3), + random:uniform(), random:uniform(42)}. + ">>, + {[{nowarn_deprecated_function,{random,seed0,0}}, + {nowarn_deprecated_function,[{random,uniform,0}, + {random,uniform,1}]}, + %% There should be no warnings when attempting to + %% turn of warnings for functions that are not + %% deprecated or not used in the module. + {nowarn_deprecated_function,{random,uniform_s,1}}, + {nowarn_deprecated_function,{erlang,abs,1}}, + warn_deprecated_function]}, + {warnings,[{5,erl_lint, + {deprecated,{random,seed,3}, + "the 'random' module is deprecated; " + "use the 'rand' module instead"}}]}}, + + %% {nowarn_unused_function,[{M,F,A}]} can be given + %% in the option list as well as in an attribute. + %% It was incorrectly documented to only work when + %% given in an attribute. + {otp_15456_2, + <<"-compile({nowarn_unused_function,foo/0}). + foo() -> ok. + bar() -> ok. + foobar() -> ok. + barf(_) -> ok. + other() -> ok. + ">>, + {[{nowarn_unused_function,[{bar,0},{foobar,0}]}, + {nowarn_unused_function,{barf,1}}, + %% There should be no warnings when attempting to + %% turn of warnings for unused functions that are not + %% defined in the module. + {nowarn_unused_function,{not_defined_in_module,1}}, + warn_unused_function]}, + {warnings,[{6,erl_lint,{unused_function,{other,0}}}]} + }], + + [] = run(Config, Ts), + ok. + %% OTP-5371. Aliases for bit syntax expressions are no longer allowed. otp_5371(Config) when is_list(Config) -> Ts = [{otp_5371_1, @@ -2731,7 +2808,7 @@ bif_clash(Config) when is_list(Config) -> [], {errors,[{2,erl_lint,{call_to_redefined_old_bif,{size,1}}}],[]}}, - %% Verify that warnings can not be turned off in the old way. + %% Verify that warnings cannot be turned off in the old way. {clash2, <<"-export([t/1,size/1]). t(X) -> @@ -3499,10 +3576,12 @@ basic_errors(Config) -> {illegal_record_info, <<"f1() -> record_info(42, record). - f2() -> record_info(shoe_size, record).">>, + f2() -> record_info(shoe_size, record). + f3() -> fun record_info/2.">>, [], {errors,[{1,erl_lint,illegal_record_info}, - {2,erl_lint,illegal_record_info}],[]}}, + {2,erl_lint,illegal_record_info}, + {3,erl_lint,illegal_record_info}],[]}}, {illegal_expr, <<"f() -> a:b.">>, diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl index f5d80e7e68..3eb1670806 100644 --- a/lib/stdlib/test/erl_pp_SUITE.erl +++ b/lib/stdlib/test/erl_pp_SUITE.erl @@ -47,11 +47,12 @@ hook/1, neg_indent/1, maps_syntax/1, + quoted_atom_types/1, otp_6321/1, otp_6911/1, otp_6914/1, otp_8150/1, otp_8238/1, otp_8473/1, otp_8522/1, otp_8567/1, otp_8664/1, otp_9147/1, otp_10302/1, otp_10820/1, otp_11100/1, otp_11861/1, pr_1014/1, - otp_13662/1, otp_14285/1, otp_15592/1]). + otp_13662/1, otp_14285/1, otp_15592/1, otp_15751/1, otp_15755/1]). %% Internal export. -export([ehook/6]). @@ -74,14 +75,14 @@ groups() -> [{expr, [], [func, call, recs, try_catch, if_then, receive_after, bits, head_tail, cond1, block, case1, ops, - messages, maps_syntax + messages, maps_syntax, quoted_atom_types ]}, {attributes, [], [misc_attrs, import_export, dialyzer_attrs]}, {tickets, [], [otp_6321, otp_6911, otp_6914, otp_8150, otp_8238, otp_8473, otp_8522, otp_8567, otp_8664, otp_9147, otp_10302, otp_10820, otp_11100, otp_11861, pr_1014, otp_13662, - otp_14285, otp_15592]}]. + otp_14285, otp_15592, otp_15751, otp_15755]}]. init_per_suite(Config) -> Config. @@ -473,10 +474,10 @@ cond1(Config) when is_list(Config) -> [{tuple,5,[{atom,5,x},{atom,5,y}]}]}]}, CChars = flat_expr1(C), "cond\n" - " {foo,bar} ->\n" - " [a,b];\n" + " {foo, bar} ->\n" + " [a, b];\n" " true ->\n" - " {x,y}\n" + " {x, y}\n" "end" = CChars, ok. @@ -711,7 +712,7 @@ otp_6321(Config) when is_list(Config) -> Str = "S = hopp, {hej, S}. ", {done, {ok, Tokens, _EndLine}, ""} = erl_scan:tokens("", Str, _L=1), {ok, Exprs} = erl_parse:parse_exprs(Tokens), - "S = hopp, {hej,S}" = lists:flatten(erl_pp:exprs(Exprs)), + "S = hopp, {hej, S}" = lists:flatten(erl_pp:exprs(Exprs)), ok. %% OTP_6911. More newlines. @@ -912,6 +913,21 @@ maps_syntax(Config) when is_list(Config) -> ok = pp_forms(F), ok. +quoted_atom_types(Config) when is_list(Config) -> + Q = [{quote_singleton_atom_types, true}], + U = [{encoding,unicode}], + L = [{encoding,latin1}], + F = "-type t() :: a | a().", + "-type t() :: 'a' | a().\n" = + lists:flatten(parse_and_pp_forms(F, Q ++ L)), + "-type t() :: 'a' | a().\n" = + lists:flatten(parse_and_pp_forms(F, Q ++ U)), + UF = "-type t() :: '\x{400}' | '\x{400}'().", + "-type t() :: '\\x{400}' | '\\x{400}'().\n" = + lists:flatten(parse_and_pp_forms(UF, Q ++ L)), + "-type t() :: '\x{400}' | '\x{400}'().\n" = + lists:flatten(parse_and_pp_forms(UF, Q ++ U)), + ok. %% OTP_8567. Avoid duplicated 'undefined' in record field types. otp_8567(Config) when is_list(Config) -> @@ -1096,7 +1112,7 @@ otp_11861(Config) when is_list(Config) -> A3 = erl_anno:new(3), "-optional_callbacks([bar/0]).\n" = pf({attribute,A3,optional_callbacks,[{bar,0}]}), - "-optional_callbacks([{bar,1,bad}]).\n" = + "-optional_callbacks([{bar, 1, bad}]).\n" = pf({attribute,A3,optional_callbacks,[{bar,1,bad}]}), ok. @@ -1172,6 +1188,79 @@ otp_15592(_Config) -> "56789012345678901234:f(<<>>)">>), ok. +otp_15751(_Config) -> + ok = pp_expr(<<"try foo:bar() + catch + Reason : Stacktrace -> + {Reason, Stacktrace} + end">>), + ok = pp_expr(<<"try foo:bar() + catch + throw: Reason : Stacktrace -> + {Reason, Stacktrace} + end">>), + ok = pp_expr(<<"try foo:bar() + catch + Reason : _ -> + Reason + end">>), + ok = pp_expr(<<"try foo:bar() + catch + throw: Reason : _ -> + Reason + end">>), + ok = pp_expr(<<"try foo:bar() + catch + Reason -> + Reason + end">>), + ok = pp_expr(<<"try foo:bar() + catch + throw: Reason -> + Reason + end">>), + ok. + +otp_15755(_Config) -> + "[{a, b}, c, {d, e} | t]" = + flat_parse_and_pp_expr("[{a, b}, c, {d, e} | t]", 0, []), + "[{a, b},\n c, d,\n {d, e},\n 1, 2.0,\n {d, e},\n <<>>, {},\n {d, e},\n" + " [], [],\n {d, e} |\n t]" = + flat_parse_and_pp_expr("[{a,b},c,d,{d,e},1,2.0,{d,e},<<>>," + "{},{d,e},[],[],{d,e}|t]", 0, []), + "[{a, b},\n c, d,\n {d, e},\n 1, 2.0,\n {d, e},\n <<>>, {},\n {d, e},\n" + " [], [], d, e | t]" = + flat_parse_and_pp_expr("[{a,b},c,d,{d,e},1,2.0,{d,e},<<>>," + "{},{d,e},[],[],d,e|t]", 0, []), + + "-type t() :: + a | b | c | a | b | a | b | a | b | a | b | a | b | a | b | + a | b | a | b | a | b.\n" = + lists:flatten(parse_and_pp_forms( + "-type t() :: a | b | c| a | b | a | b | a | b | a |" + " b | a | b | a | b | a | b | a | b |a | b.", [])), + + "-type t() :: + {dict, 0, 16, 16, 8, 80, 48, + {[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], + []}, + {{[], [], [], [], [], [], [], [], [], [], [], [], [], [], []}}}.\n" = + lists:flatten(parse_and_pp_forms( + "-type t() :: {dict,0,16,16,8,80,48," + "{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}," + "{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}.", [])), + + "-type t() :: + {{a}, + 0, 16, + {16}, + 8, 80, 48, a, b, e, f, 'sf s sdf', [], {}, + {[]}}.\n" = + lists:flatten(parse_and_pp_forms( + "-type t() :: {{a}, 0, 16, {16}, 8, 80, 48, a, b, e, f," + " 'sf s sdf', [], {}, {[]}}.", [])), + ok. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% compile(Config, Tests) -> @@ -1303,6 +1392,9 @@ pp_expr(List, Options) when is_list(List) -> not_ok end. +flat_parse_and_pp_expr(String, Indent, Options) -> + lists:flatten(parse_and_pp_expr(String, Indent, Options)). + parse_and_pp_expr(String, Indent, Options) -> StringDot = lists:flatten(String) ++ ".", erl_pp:expr(parse_expr(StringDot), Indent, Options). diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 433b812fd5..dd49288417 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -24,7 +24,7 @@ -export([default/1,setbag/1,badnew/1,verybadnew/1,named/1,keypos2/1, privacy/1]). -export([empty/1,badinsert/1]). --export([time_lookup/1,badlookup/1,lookup_order/1]). +-export([badlookup/1,lookup_order/1]). -export([delete_elem/1,delete_tab/1,delete_large_tab/1, delete_large_named_table/1, evil_delete/1,baddelete/1,match_delete/1,table_leak/1]). @@ -42,6 +42,8 @@ select_bound_chunk/1, t_delete_all_objects/1, t_insert_list/1, t_test_ms/1, t_select_delete/1,t_select_replace/1,t_select_replace_next_bug/1,t_ets_dets/1]). +-export([test_table_size_concurrency/1,test_table_memory_concurrency/1, + test_delete_table_while_size_snapshot/1, test_delete_table_while_size_snapshot_helper/0]). -export([ordered/1, ordered_match/1, interface_equality/1, fixtable_next/1, fixtable_insert/1, rename/1, rename_unnamed/1, evil_rename/1, @@ -56,6 +58,7 @@ -export([t_match_spec_run/1]). -export([t_bucket_disappears/1]). -export([t_named_select/1]). +-export([select_fixtab_owner_change/1]). -export([otp_5340/1]). -export([otp_6338/1]). -export([otp_6842_select_1000/1]). @@ -67,7 +70,12 @@ meta_lookup_named_read/1, meta_lookup_named_write/1, meta_newdel_unnamed/1, meta_newdel_named/1]). -export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1, + smp_ordered_iteration/1, smp_select_replace/1, otp_8166/1, otp_8732/1, delete_unfix_race/1]). +-export([throughput_benchmark/0, + throughput_benchmark/1, + test_throughput_benchmark/1, + long_throughput_benchmark/1]). -export([exit_large_table_owner/1, exit_many_large_table_owner/1, exit_many_tables_owner/1, @@ -90,6 +98,7 @@ -include_lib("stdlib/include/ms_transform.hrl"). % ets:fun2ms -include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). -define(m(A,B), assert_eq(A,B)). -define(heap_binary_size, 64). @@ -128,13 +137,14 @@ all() -> t_insert_list, t_test_ms, t_select_delete, t_select_replace, t_select_replace_next_bug, t_ets_dets, memory, t_select_reverse, t_bucket_disappears, - t_named_select, + t_named_select, select_fixtab_owner_change, select_fail, t_insert_new, t_repair_continuation, otp_5340, otp_6338, otp_6842_select_1000, otp_7665, select_mbuf_trapping, otp_8732, meta_wb, grow_shrink, grow_pseudo_deleted, shrink_pseudo_deleted, {group, meta_smp}, smp_insert, - smp_fixed_delete, smp_unfix_fix, smp_select_replace, + smp_fixed_delete, smp_unfix_fix, smp_select_replace, + smp_ordered_iteration, smp_select_delete, otp_8166, exit_large_table_owner, exit_many_large_table_owner, exit_many_tables_owner, exit_many_many_tables_owner, write_concurrency, heir, @@ -146,14 +156,20 @@ all() -> massive_ets_all, take, whereis_table, - delete_unfix_race]. + delete_unfix_race, + test_throughput_benchmark, + {group, benchmark}, + test_table_size_concurrency, + test_table_memory_concurrency, + test_delete_table_while_size_snapshot]. + groups() -> [{new, [], [default, setbag, badnew, verybadnew, named, keypos2, privacy]}, {insert, [], [empty, badinsert]}, - {lookup, [], [time_lookup, badlookup, lookup_order]}, + {lookup, [], [badlookup, lookup_order]}, {lookup_element, [], [lookup_element_mult]}, {delete, [], [delete_elem, delete_tab, delete_large_tab, @@ -174,7 +190,9 @@ groups() -> {meta_smp, [], [meta_lookup_unnamed_read, meta_lookup_unnamed_write, meta_lookup_named_read, meta_lookup_named_write, - meta_newdel_unnamed, meta_newdel_named]}]. + meta_newdel_unnamed, meta_newdel_named]}, + {benchmark, [], + [long_throughput_benchmark]}]. init_per_suite(Config) -> erts_debug:set_internal_state(available_internal_state, true), @@ -187,9 +205,61 @@ end_per_suite(_Config) -> catch erts_debug:set_internal_state(available_internal_state, false), ok. +init_per_group(benchmark, Config) -> + P = self(), + %% Spawn owner of ETS table that is alive until end_per_group is run + EtsProcess = + spawn( + fun()-> + Tab = ets:new(ets_benchmark_result_summary_tab, [public]), + P ! {the_table, Tab}, + receive + kill -> ok + end + end), + Tab = receive {the_table, T} -> T end, + CounterNames = [nr_of_benchmarks, + total_throughput, + nr_of_set_benchmarks, + total_throughput_set, + nr_of_ordered_set_benchmarks, + total_throughput_ordered_set], + lists:foreach(fun(CtrName) -> + ets:insert(Tab, {CtrName, 0.0}) + end, + CounterNames), + [{ets_benchmark_result_summary_tab, Tab}, + {ets_benchmark_result_summary_tab_process, EtsProcess} | Config]; init_per_group(_GroupName, Config) -> Config. +end_per_group(benchmark, Config) -> + T = proplists:get_value(ets_benchmark_result_summary_tab, Config), + EtsProcess = proplists:get_value(ets_benchmark_result_summary_tab_process, Config), + Report = + fun(NOfBenchmarksCtr, TotThroughoutCtr, Name) -> + Average = + ets:lookup_element(T, TotThroughoutCtr, 2) / + ets:lookup_element(T, NOfBenchmarksCtr, 2), + io:format("~p ~p~n", [Name, Average]), + ct_event:notify( + #event{name = benchmark_data, + data = [{suite,"ets_bench"}, + {name, Name}, + {value, Average}]}) + end, + Report(nr_of_benchmarks, + total_throughput, + "Average Throughput"), + Report(nr_of_set_benchmarks, + total_throughput_set, + "Average Throughput Set"), + Report(nr_of_ordered_set_benchmarks, + total_throughput_ordered_set, + "Average Throughput Ordered Set"), + ets:delete(T), + EtsProcess ! kill, + Config; end_per_group(_GroupName, Config) -> Config. @@ -245,7 +315,64 @@ t_named_select_do(Opts) -> verify_etsmem(EtsMem). +%% Verify select and friends release fixtab as they should +%% even when owneship is changed between traps. +select_fixtab_owner_change(_Config) -> + T = ets:new(xxx, [protected]), + NKeys = 2000, + [ets:insert(T,{K,K band 7}) || K <- lists:seq(1,NKeys)], + + %% Buddy and Papa will ping-pong table ownership between them + %% and the aim is to give Buddy the table when he is + %% in the middle of a yielding select* call. + {Buddy,_} = spawn_opt(fun() -> sfoc_buddy_loop(T, 1, undefined) end, + [link,monitor]), + sfoc_papa_loop(T, Buddy), + + receive {'DOWN', _, process, Buddy, _} -> ok end, + ets:delete(T), + ok. + +sfoc_buddy_loop(T, I, State0) -> + receive + {'ETS-TRANSFER', T, Papa, _} -> + ets:give_away(T, Papa, State0), + case State0 of + done -> + ok; + _ -> + State1 = sfoc_traverse(T, I, State0), + %% Verify no fixation left + {I, false} = {I, ets:info(T, safe_fixed_monotonic_time)}, + sfoc_buddy_loop(T, I+1, State1) + end + end. + +sfoc_papa_loop(T, Buddy) -> + ets:give_away(T, Buddy, "Catch!"), + receive + {'ETS-TRANSFER', T, Buddy, State} -> + case State of + done -> + ok; + _ -> + sfoc_papa_loop(T, Buddy) + end + end. + +sfoc_traverse(T, 1, S) -> + ets:select(T, [{{'$1',7}, [], ['$1']}]), S; +sfoc_traverse(T, 2, S) -> + 0 = ets:select_count(T, [{{'$1',7}, [], [false]}]), S; +sfoc_traverse(T, 3, _) -> + Limit = ets:info(T, size) div 2, + {_, Continuation} = ets:select(T, [{{'$1',7}, [], ['$1']}], + Limit), + Continuation; +sfoc_traverse(_T, 4, Continuation) -> + _ = ets:select(Continuation), + done. %% Check ets:match_spec_run/2. t_match_spec_run(Config) when is_list(Config) -> @@ -707,7 +834,11 @@ adjust_xmem([_T1,_T2,_T3,_T4], {A0,B0,C0,D0} = _Mem0, EstCnt) -> {TabSz, EstSz} = erts_debug:get_internal_state('DbTable_words'), HTabSz = TabSz + EstCnt*EstSz, - {A0+TabSz, B0+HTabSz, C0+HTabSz, D0+HTabSz}. + OrdSetExtra = case erlang:system_info(wordsize) of + 8 -> 40; % larger stack on 64 bit architectures + _ -> 0 + end, + {A0+TabSz+OrdSetExtra, B0+HTabSz, C0+HTabSz, D0+HTabSz}. %% Misc. whitebox tests t_whitebox(Config) when is_list(Config) -> @@ -745,7 +876,7 @@ select_bound_chunk(_Config) -> repeat_for_opts(fun select_bound_chunk_do/1, [all_types]). select_bound_chunk_do(Opts) -> - T = ets:new(x, Opts), + T = ets_new(x, Opts), ets:insert(T, [{key, 1}]), {[{key, 1}], '$end_of_table'} = ets:select(T, [{{key,1},[],['$_']}], 100000), ok. @@ -791,38 +922,42 @@ check_badarg({'EXIT', {badarg, [{M,F,A,_} | _]}}, M, F, Args) -> %% Test ets:delete_all_objects/1. t_delete_all_objects(Config) when is_list(Config) -> EtsMem = etsmem(), - repeat_for_opts(fun t_delete_all_objects_do/1), + repeat_for_opts_all_set_table_types(fun t_delete_all_objects_do/1), verify_etsmem(EtsMem). get_kept_objects(T) -> case ets:info(T,stats) of - false -> - 0; {_,_,_,_,_,_,KO} -> - KO + KO; + _ -> + 0 end. t_delete_all_objects_do(Opts) -> - T=ets_new(x,Opts), - filltabint(T,4000), + KeyRange = 4000, + T=ets_new(x, Opts, KeyRange), + filltabint(T,KeyRange), O=ets:first(T), ets:next(T,O), ets:safe_fixtable(T,true), true = ets:delete_all_objects(T), '$end_of_table' = ets:next(T,O), 0 = ets:info(T,size), - 4000 = get_kept_objects(T), + case ets:info(T,type) of + ordered_set -> ok; + _ -> KeyRange = get_kept_objects(T) + end, ets:safe_fixtable(T,false), 0 = ets:info(T,size), 0 = get_kept_objects(T), - filltabint(T,4000), - 4000 = ets:info(T,size), + filltabint(T, KeyRange), + KeyRange = ets:info(T,size), true = ets:delete_all_objects(T), 0 = ets:info(T,size), ets:delete(T), %% Test delete_all_objects is atomic - T2 = ets:new(t_delete_all_objects, [public | Opts]), + T2 = ets_new(t_delete_all_objects, [public | Opts]), Self = self(), Inserters = [spawn_link(fun() -> inserter(T2, 100*1000, 1, Self) end) || _ <- [1,2,3,4]], [receive {Ipid, running} -> ok end || Ipid <- Inserters], @@ -2374,17 +2509,29 @@ write_concurrency(Config) when is_list(Config) -> Yes6 = ets_new(foo,[duplicate_bag,protected,{write_concurrency,true}]), No3 = ets_new(foo,[duplicate_bag,private,{write_concurrency,true}]), - No4 = ets_new(foo,[ordered_set,public,{write_concurrency,true}]), - No5 = ets_new(foo,[ordered_set,protected,{write_concurrency,true}]), - No6 = ets_new(foo,[ordered_set,private,{write_concurrency,true}]), - - No7 = ets_new(foo,[public,{write_concurrency,false}]), - No8 = ets_new(foo,[protected,{write_concurrency,false}]), + Yes7 = ets_new(foo,[ordered_set,public,{write_concurrency,true}]), + Yes8 = ets_new(foo,[ordered_set,protected,{write_concurrency,true}]), + Yes9 = ets_new(foo,[ordered_set,{write_concurrency,true}]), + Yes10 = ets_new(foo,[{write_concurrency,true},ordered_set,public]), + Yes11 = ets_new(foo,[{write_concurrency,true},ordered_set,protected]), + Yes12 = ets_new(foo,[set,{write_concurrency,false}, + {write_concurrency,true},ordered_set,public]), + Yes13 = ets_new(foo,[private,public,set,{write_concurrency,false}, + {write_concurrency,true},ordered_set]), + No4 = ets_new(foo,[ordered_set,private,{write_concurrency,true}]), + No5 = ets_new(foo,[ordered_set,public,{write_concurrency,false}]), + No6 = ets_new(foo,[ordered_set,protected,{write_concurrency,false}]), + No7 = ets_new(foo,[ordered_set,private,{write_concurrency,false}]), + + No8 = ets_new(foo,[public,{write_concurrency,false}]), + No9 = ets_new(foo,[protected,{write_concurrency,false}]), YesMem = ets:info(Yes1,memory), NoHashMem = ets:info(No1,memory), + YesTreeMem = ets:info(Yes7,memory), NoTreeMem = ets:info(No4,memory), - io:format("YesMem=~p NoHashMem=~p NoTreeMem=~p\n",[YesMem,NoHashMem,NoTreeMem]), + io:format("YesMem=~p NoHashMem=~p NoTreeMem=~p YesTreeMem=~p\n",[YesMem,NoHashMem, + NoTreeMem,YesTreeMem]), YesMem = ets:info(Yes2,memory), YesMem = ets:info(Yes3,memory), @@ -2393,13 +2540,24 @@ write_concurrency(Config) when is_list(Config) -> YesMem = ets:info(Yes6,memory), NoHashMem = ets:info(No2,memory), NoHashMem = ets:info(No3,memory), + YesTreeMem = ets:info(Yes7,memory), + YesTreeMem = ets:info(Yes8,memory), + YesTreeMem = ets:info(Yes9,memory), + YesTreeMem = ets:info(Yes10,memory), + YesTreeMem = ets:info(Yes11,memory), + YesTreeMem = ets:info(Yes12,memory), + YesTreeMem = ets:info(Yes13,memory), + NoTreeMem = ets:info(No4,memory), NoTreeMem = ets:info(No5,memory), NoTreeMem = ets:info(No6,memory), - NoHashMem = ets:info(No7,memory), + NoTreeMem = ets:info(No7,memory), NoHashMem = ets:info(No8,memory), + NoHashMem = ets:info(No9,memory), true = YesMem > NoHashMem, true = YesMem > NoTreeMem, + true = YesMem > YesTreeMem, + true = YesTreeMem < NoTreeMem, {'EXIT',{badarg,_}} = (catch ets_new(foo,[public,{write_concurrency,foo}])), {'EXIT',{badarg,_}} = (catch ets_new(foo,[public,{write_concurrency}])), @@ -2407,8 +2565,8 @@ write_concurrency(Config) when is_list(Config) -> {'EXIT',{badarg,_}} = (catch ets_new(foo,[public,write_concurrency])), lists:foreach(fun(T) -> ets:delete(T) end, - [Yes1,Yes2,Yes3,Yes4,Yes5,Yes6, - No1,No2,No3,No4,No5,No6,No7,No8]), + [Yes1,Yes2,Yes3,Yes4,Yes5,Yes6,Yes7,Yes8,Yes9,Yes10,Yes11,Yes12,Yes13, + No1,No2,No3,No4,No5,No6,No7,No8,No9]), verify_etsmem(EtsMem), ok. @@ -3053,41 +3211,43 @@ pick_all_backwards(T) -> %% Small test case for both set and bag type ets tables. setbag(Config) when is_list(Config) -> EtsMem = etsmem(), - Set = ets_new(set,[set]), - Bag = ets_new(bag,[bag]), - Key = {foo,bar}, - - %% insert some value - ets:insert(Set,{Key,val1}), - ets:insert(Bag,{Key,val1}), - - %% insert new value for same key again - ets:insert(Set,{Key,val2}), - ets:insert(Bag,{Key,val2}), - - %% check - [{Key,val2}] = ets:lookup(Set,Key), - [{Key,val1},{Key,val2}] = ets:lookup(Bag,Key), - - true = ets:delete(Set), - true = ets:delete(Bag), + lists:foreach(fun(SetType) -> + Set = ets_new(SetType,[SetType]), + Bag = ets_new(bag,[bag]), + Key = {foo,bar}, + + %% insert some value + ets:insert(Set,{Key,val1}), + ets:insert(Bag,{Key,val1}), + + %% insert new value for same key again + ets:insert(Set,{Key,val2}), + ets:insert(Bag,{Key,val2}), + + %% check + [{Key,val2}] = ets:lookup(Set,Key), + [{Key,val1},{Key,val2}] = ets:lookup(Bag,Key), + + true = ets:delete(Set), + true = ets:delete(Bag) + end, [set, cat_ord_set,stim_cat_ord_set,ordered_set]), verify_etsmem(EtsMem). %% Test case to check proper return values for illegal ets_new() calls. badnew(Config) when is_list(Config) -> EtsMem = etsmem(), - {'EXIT',{badarg,_}} = (catch ets_new(12,[])), - {'EXIT',{badarg,_}} = (catch ets_new({a,b},[])), - {'EXIT',{badarg,_}} = (catch ets_new(name,[foo])), - {'EXIT',{badarg,_}} = (catch ets_new(name,{bag})), - {'EXIT',{badarg,_}} = (catch ets_new(name,bag)), + {'EXIT',{badarg,_}} = (catch ets:new(12,[])), + {'EXIT',{badarg,_}} = (catch ets:new({a,b},[])), + {'EXIT',{badarg,_}} = (catch ets:new(name,[foo])), + {'EXIT',{badarg,_}} = (catch ets:new(name,{bag})), + {'EXIT',{badarg,_}} = (catch ets:new(name,bag)), verify_etsmem(EtsMem). %% OTP-2314. Test case to check that a non-proper list does not %% crash the emulator. verybadnew(Config) when is_list(Config) -> EtsMem = etsmem(), - {'EXIT',{badarg,_}} = (catch ets_new(verybad,[set|protected])), + {'EXIT',{badarg,_}} = (catch ets:new(verybad,[set|protected])), verify_etsmem(EtsMem). %% Small check to see if named tables work. @@ -3103,11 +3263,13 @@ named(Config) when is_list(Config) -> %% Test case to check if specified keypos works. keypos2(Config) when is_list(Config) -> EtsMem = etsmem(), - Tab = make_table(foo, - [set,{keypos,2}], - [{val,key}, {val2,key}]), - [{val2,key}] = ets:lookup(Tab,key), - true = ets:delete(Tab), + lists:foreach(fun(SetType) -> + Tab = make_table(foo, + [SetType,{keypos,2}], + [{val,key}, {val2,key}]), + [{val2,key}] = ets:lookup(Tab,key), + true = ets:delete(Tab) + end, [set, cat_ord_set,stim_cat_ord_set,ordered_set]), verify_etsmem(EtsMem). %% Privacy check. Check that a named(public/private/protected) table @@ -3199,7 +3361,7 @@ rotate_tuple(Tuple, N) -> %% Check lookup in an empty table and lookup of a non-existing key. empty(Config) when is_list(Config) -> - repeat_for_opts(fun empty_do/1). + repeat_for_opts_all_table_types(fun empty_do/1). empty_do(Opts) -> EtsMem = etsmem(), @@ -3212,7 +3374,7 @@ empty_do(Opts) -> %% Check proper return values for illegal insert operations. badinsert(Config) when is_list(Config) -> - repeat_for_opts(fun badinsert_do/1). + repeat_for_opts_all_table_types(fun badinsert_do/1). badinsert_do(Opts) -> EtsMem = etsmem(), @@ -3232,31 +3394,6 @@ badinsert_do(Opts) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Test lookup timing. -time_lookup(Config) when is_list(Config) -> - %% just for timing, really - EtsMem = etsmem(), - Values = repeat_for_opts(fun time_lookup_do/1), - verify_etsmem(EtsMem), - {comment,lists:flatten(io_lib:format( - "~p ets lookups/s",[Values]))}. - -time_lookup_do(Opts) -> - Tab = ets_new(foo,Opts), - fill_tab(Tab,foo), - ets:insert(Tab,{{a,key},foo}), - N = 100000, - {Time,_} = timer:tc(fun() -> time_lookup_many(N, Tab) end), - Seconds = Time / 1000000, - true = ets:delete(Tab), - round(N / Seconds). % lookups/s - -time_lookup_many(0, _Tab) -> - ok; -time_lookup_many(N, Tab) -> - ets:lookup(Tab, {a,key}), - time_lookup_many(N-1, Tab). - %% Check proper return values from bad lookups in existing/non existing %% ets tables. badlookup(Config) when is_list(Config) -> @@ -3433,19 +3570,29 @@ delete_tab_do(Opts) -> %% Check that ets:delete/1 works and that other processes can run. delete_large_tab(Config) when is_list(Config) -> - ct:timetrap({minutes,30}), %% valgrind needs a lot - Data = [{erlang:phash2(I, 16#ffffff),I} || I <- lists:seq(1, 200000)], + ct:timetrap({minutes,60}), %% valgrind needs a lot + KeyRange = 16#ffffff, + Data = [{erlang:phash2(I, KeyRange),I} || I <- lists:seq(1, 200000)], EtsMem = etsmem(), - repeat_for_opts(fun(Opts) -> delete_large_tab_do(Opts,Data) end), + repeat_for_opts(fun(Opts) -> delete_large_tab_do(key_range(Opts,KeyRange), + Data) end), verify_etsmem(EtsMem). delete_large_tab_do(Opts,Data) -> delete_large_tab_1(foo_hash, Opts, Data, false), delete_large_tab_1(foo_tree, [ordered_set | Opts], Data, false), - delete_large_tab_1(foo_hash, Opts, Data, true). + delete_large_tab_1(foo_tree, [stim_cat_ord_set | Opts], Data, false), + delete_large_tab_1(foo_hash_fix, Opts, Data, true). delete_large_tab_1(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + delete_large_tab_2(Name, Flags, Data, Fix) + end. + +delete_large_tab_2(Name, Flags, Data, Fix) -> Tab = ets_new(Name, Flags), ets:insert(Tab, Data), @@ -3504,18 +3651,30 @@ delete_large_tab_1(Name, Flags, Data, Fix) -> %% Delete a large name table and try to create a new table with %% the same name in another process. delete_large_named_table(Config) when is_list(Config) -> - Data = [{erlang:phash2(I, 16#ffffff),I} || I <- lists:seq(1, 200000)], + KeyRange = 16#ffffff, + Data = [{erlang:phash2(I, KeyRange),I} || I <- lists:seq(1, 200000)], EtsMem = etsmem(), - repeat_for_opts(fun(Opts) -> delete_large_named_table_do(Opts,Data) end), + repeat_for_opts(fun(Opts) -> + delete_large_named_table_do(key_range(Opts,KeyRange), + Data) + end), verify_etsmem(EtsMem), ok. delete_large_named_table_do(Opts,Data) -> delete_large_named_table_1(foo_hash, [named_table | Opts], Data, false), delete_large_named_table_1(foo_tree, [ordered_set,named_table | Opts], Data, false), + delete_large_named_table_1(foo_tree, [stim_cat_ord_set,named_table | Opts], Data, false), delete_large_named_table_1(foo_hash, [named_table | Opts], Data, true). delete_large_named_table_1(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + delete_large_named_table_2(Name, Flags, Data, Fix) + end. + +delete_large_named_table_2(Name, Flags, Data, Fix) -> Tab = ets_new(Name, Flags), ets:insert(Tab, Data), @@ -3539,8 +3698,12 @@ delete_large_named_table_1(Name, Flags, Data, Fix) -> %% Delete a large table, and kill the process during the delete. evil_delete(Config) when is_list(Config) -> - Data = [{I,I*I} || I <- lists:seq(1, 100000)], - repeat_for_opts(fun(Opts) -> evil_delete_do(Opts,Data) end). + KeyRange = 100000, + Data = [{I,I*I} || I <- lists:seq(1, KeyRange)], + repeat_for_opts(fun(Opts) -> + evil_delete_do(key_range(Opts,KeyRange), + Data) + end). evil_delete_do(Opts,Data) -> EtsMem = etsmem(), @@ -3550,16 +3713,27 @@ evil_delete_do(Opts,Data) -> verify_etsmem(EtsMem), evil_delete_owner(foo_tree, [ordered_set | Opts], Data, false), verify_etsmem(EtsMem), + evil_delete_owner(foo_catree, [stim_cat_ord_set | Opts], Data, false), + verify_etsmem(EtsMem), TabA = evil_delete_not_owner(foo_hash, Opts, Data, false), verify_etsmem(EtsMem), TabB = evil_delete_not_owner(foo_hash, Opts, Data, true), verify_etsmem(EtsMem), TabC = evil_delete_not_owner(foo_tree, [ordered_set | Opts], Data, false), verify_etsmem(EtsMem), + TabD = evil_delete_not_owner(foo_catree, [stim_cat_ord_set | Opts], Data, false), + verify_etsmem(EtsMem), lists:foreach(fun(T) -> undefined = ets:info(T) end, - [TabA,TabB,TabC]). + [TabA,TabB,TabC,TabD]). evil_delete_not_owner(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + evil_delete_not_owner_1(Name, Flags, Data, Fix) + end. + +evil_delete_not_owner_1(Name, Flags, Data, Fix) -> io:format("Not owner: ~p, fix = ~p", [Name,Fix]), Tab = ets_new(Name, [public|Flags]), ets:insert(Tab, Data), @@ -3585,6 +3759,13 @@ evil_delete_not_owner(Name, Flags, Data, Fix) -> Tab. evil_delete_owner(Name, Flags, Data, Fix) -> + case is_redundant_opts_combo(Flags) of + true -> skip; + false -> + evil_delete_owner_1(Name, Flags, Data, Fix) + end. + +evil_delete_owner_1(Name, Flags, Data, Fix) -> Fun = fun() -> Tab = ets_new(Name, [public|Flags]), ets:insert(Tab, Data), @@ -3772,7 +3953,7 @@ verify_rescheduling_exit(Config, ForEachData, Flags, Fix, NOTabs, NOProcs) -> %% Make sure that slots for ets tables are cleared properly. table_leak(Config) when is_list(Config) -> - repeat_for_opts(fun(Opts) -> table_leak_1(Opts,20000) end). + repeat_for_opts_all_non_stim_table_types(fun(Opts) -> table_leak_1(Opts,20000) end). table_leak_1(_,0) -> ok; table_leak_1(Opts,N) -> @@ -3835,7 +4016,7 @@ match_delete3_do(Opts) -> %% Test ets:first/1 & ets:next/2. firstnext(Config) when is_list(Config) -> - repeat_for_opts(fun firstnext_do/1). + repeat_for_opts_all_set_table_types(fun firstnext_do/1). firstnext_do(Opts) -> EtsMem = etsmem(), @@ -3857,15 +4038,20 @@ firstnext_collect(Tab,Key,List) -> %% Tests ets:first/1 & ets:next/2. firstnext_concurrent(Config) when is_list(Config) -> - register(master, self()), - ets_init(?MODULE, 20), - [dynamic_go() || _ <- lists:seq(1, 2)], - receive - after 5000 -> ok - end. + lists:foreach( + fun(TableType) -> + register(master, self()), + TableName = list_to_atom(atom_to_list(?MODULE) ++ atom_to_list(TableType)), + ets_init(TableName, 20, TableType), + [dynamic_go(TableName) || _ <- lists:seq(1, 2)], + receive + after 5000 -> ok + end, + unregister(master) + end, repeat_for_opts_atom2list(ord_set_types)). -ets_init(Tab, N) -> - ets_new(Tab, [named_table,public,ordered_set]), +ets_init(Tab, N, TableType) -> + ets_new(Tab, [named_table,public,TableType]), cycle(Tab, lists:seq(1,N+1)). cycle(_Tab, [H|T]) when H > length(T)-> ok; @@ -3873,9 +4059,9 @@ cycle(Tab, L) -> ets:insert(Tab,list_to_tuple(L)), cycle(Tab, tl(L)++[hd(L)]). -dynamic_go() -> my_spawn_link(fun dynamic_init/0). +dynamic_go(TableName) -> my_spawn_link(fun() -> dynamic_init(TableName) end). -dynamic_init() -> [dyn_lookup(?MODULE) || _ <- lists:seq(1, 10)]. +dynamic_init(TableName) -> [dyn_lookup(TableName) || _ <- lists:seq(1, 10)]. dyn_lookup(T) -> dyn_lookup(T, ets:first(T)). @@ -3893,7 +4079,7 @@ dyn_lookup(T, K) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% slot(Config) when is_list(Config) -> - repeat_for_opts(fun slot_do/1). + repeat_for_opts_all_set_table_types(fun slot_do/1). slot_do(Opts) -> EtsMem = etsmem(), @@ -3901,6 +4087,11 @@ slot_do(Opts) -> fill_tab(Tab,foo), Elts = ets:info(Tab,size), Elts = slot_loop(Tab,0,0), + case ets:info(Tab, type) of + ordered_set -> + '$end_of_table' = ets:slot(Tab,Elts); + _ -> ok + end, true = ets:delete(Tab), verify_etsmem(EtsMem). @@ -3918,7 +4109,7 @@ slot_loop(Tab,SlotNo,EltsSoFar) -> match1(Config) when is_list(Config) -> - repeat_for_opts(fun match1_do/1). + repeat_for_opts_all_set_table_types(fun match1_do/1). match1_do(Opts) -> EtsMem = etsmem(), @@ -3981,7 +4172,7 @@ match2_do(Opts) -> %% Some ets:match_object tests. match_object(Config) when is_list(Config) -> - repeat_for_opts(fun match_object_do/1). + repeat_for_opts_all_set_table_types(fun match_object_do/1). match_object_do(Opts) -> EtsMem = etsmem(), @@ -4081,23 +4272,16 @@ match_object_do(Opts) -> %% Tests that db_match_object does not generate a `badarg' when %% resuming a search with no previous matches. match_object2(Config) when is_list(Config) -> - repeat_for_opts(fun match_object2_do/1). + repeat_for_opts_all_table_types(fun match_object2_do/1). match_object2_do(Opts) -> EtsMem = etsmem(), - Tab = ets_new(foo, [bag, {keypos, 2} | Opts]), - fill_tab2(Tab, 0, 13005), % match_db_object does 1000 + KeyRange = 13005, + Tab = ets_new(foo, [{keypos, 2} | Opts], KeyRange), + fill_tab2(Tab, 0, KeyRange), % match_db_object does 1000 % elements per pass, might % change in the future. - case catch ets:match_object(Tab, {hej, '$1'}) of - {'EXIT', _} -> - ets:delete(Tab), - ct:fail("match_object EXIT:ed"); - [] -> - io:format("Nothing matched."); - List -> - io:format("Matched:~p~n",[List]) - end, + [] = ets:match_object(Tab, {hej, '$1'}), ets:delete(Tab), verify_etsmem(EtsMem). @@ -4106,18 +4290,21 @@ match_object2_do(Opts) -> %% OTP-3319. Test tab2list. tab2list(Config) when is_list(Config) -> - EtsMem = etsmem(), - Tab = make_table(foo, - [ordered_set], - [{a,b}, {c,b}, {b,b}, {a,c}]), - [{a,c},{b,b},{c,b}] = ets:tab2list(Tab), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_all_ord_set_table_types( + fun(Opts) -> + EtsMem = etsmem(), + Tab = make_table(foo, + Opts, + [{a,b}, {c,b}, {b,b}, {a,c}]), + [{a,c},{b,b},{c,b}] = ets:tab2list(Tab), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end). %% Simple general small test. If this fails, ets is in really bad %% shape. misc1(Config) when is_list(Config) -> - repeat_for_opts(fun misc1_do/1). + repeat_for_opts_all_table_types(fun misc1_do/1). misc1_do(Opts) -> EtsMem = etsmem(), @@ -4135,7 +4322,7 @@ misc1_do(Opts) -> %% Check the safe_fixtable function. safe_fixtable(Config) when is_list(Config) -> - repeat_for_opts(fun safe_fixtable_do/1). + repeat_for_opts_all_table_types(fun safe_fixtable_do/1). safe_fixtable_do(Opts) -> EtsMem = etsmem(), @@ -4193,10 +4380,42 @@ safe_fixtable_do(Opts) -> %% Tests ets:info result for required tuples. info(Config) when is_list(Config) -> - repeat_for_opts(fun info_do/1). + repeat_for_opts_all_table_types(fun info_do/1). info_do(Opts) -> EtsMem = etsmem(), + TableType = lists:foldl( + fun(Item, Curr) -> + case Item of + set -> set; + ordered_set -> ordered_set; + cat_ord_set -> ordered_set; + stim_cat_ord_set -> ordered_set; + bag -> bag; + duplicate_bag -> duplicate_bag; + _ -> Curr + end + end, set, Opts), + PublicOrCurr = + fun(Curr) -> + case lists:member({write_concurrency, false}, Opts) or + lists:member(private, Opts) or + lists:member(protected, Opts) of + true -> Curr; + false -> public + end + end, + Protection = lists:foldl( + fun(Item, Curr) -> + case Item of + public -> public; + protected -> protected; + private -> private; + cat_ord_set -> PublicOrCurr(Curr); %% Special items + stim_cat_ord_set -> PublicOrCurr(Curr); + _ -> Curr + end + end, protected, Opts), MeMyselfI=self(), ThisNode=node(), Tab = ets_new(foobar, [{keypos, 2} | Opts]), @@ -4210,9 +4429,9 @@ info_do(Opts) -> {value, {size, 0}} = lists:keysearch(size, 1, Res), {value, {node, ThisNode}} = lists:keysearch(node, 1, Res), {value, {named_table, false}} = lists:keysearch(named_table, 1, Res), - {value, {type, set}} = lists:keysearch(type, 1, Res), + {value, {type, TableType}} = lists:keysearch(type, 1, Res), {value, {keypos, 2}} = lists:keysearch(keypos, 1, Res), - {value, {protection, protected}} = + {value, {protection, Protection}} = lists:keysearch(protection, 1, Res), {value, {id, Tab}} = lists:keysearch(id, 1, Res), true = ets:delete(Tab), @@ -4224,6 +4443,131 @@ info_do(Opts) -> undefined = ets:info(non_existing_table_xxyy,safe_fixed), verify_etsmem(EtsMem). +size_loop(_T, 0, _, _) -> + ok; +size_loop(T, I, PrevSize, WhatToTest) -> + Size = ets:info(T, WhatToTest), + case Size < PrevSize of + true -> ct:fail("Bad ets:info/2"); + _ -> ok + end, + size_loop(T, I -1, Size, WhatToTest). + +add_loop(_T, 0) -> + ok; +add_loop(T, I) -> + ets:insert(T, {I}), + add_loop(T, I -1). + + +test_table_counter_concurrency(WhatToTest) -> + IntStatePrevOn = + erts_debug:set_internal_state(available_internal_state, true), + ItemsToAdd = 1000000, + SizeLoopSize = 1000, + T = ets:new(k, [public, ordered_set, {write_concurrency, true}]), + erts_debug:set_internal_state(ets_debug_random_split_join, {T, false}), + 0 = ets:info(T, size), + P = self(), + SpawnedSizeProcs = + [spawn_link(fun() -> + size_loop(T, SizeLoopSize, 0, WhatToTest), + P ! done + end) + || _ <- lists:seq(1, 6)], + spawn_link(fun() -> + add_loop(T, ItemsToAdd), + P ! done_add + end), + [receive + done -> ok; + done_add -> ok + end + || _ <- [ok|SpawnedSizeProcs]], + case WhatToTest =:= size of + true -> + ItemsToAdd = ets:info(T, size); + _ -> + ok + end, + erts_debug:set_internal_state(available_internal_state, IntStatePrevOn), + ok. + +test_table_size_concurrency(Config) when is_list(Config) -> + test_table_counter_concurrency(size). + +test_table_memory_concurrency(Config) when is_list(Config) -> + test_table_counter_concurrency(memory). + +%% Tests that calling the ets:delete operation on a table T with +%% decentralized counters works while ets:info(T, size) operations are +%% active +test_delete_table_while_size_snapshot(Config) when is_list(Config) -> + %% Run test case in a slave node as other test suites in stdlib + %% depend on that pids are ordered in creation order which is no + %% longer the case when many processes have been started before + Node = start_slave(), + ok = rpc:call(Node, ?MODULE, test_delete_table_while_size_snapshot_helper, []), + test_server:stop_node(Node), + ok. + +test_delete_table_while_size_snapshot_helper()-> + TopParent = self(), + repeat_par( + fun() -> + Table = ets:new(t, [public, ordered_set, + {write_concurrency, true}]), + Parent = self(), + NrOfSizeProcs = 100, + Pids = [ spawn(fun()-> size_process(Table, Parent) end) + || _ <- lists:seq(1, NrOfSizeProcs)], + timer:sleep(1), + ets:delete(Table), + [receive + table_gone -> ok; + Problem -> TopParent ! Problem + end || _ <- Pids] + end, + 15000), + receive + Problem -> throw(Problem) + after 0 -> ok + end. + +size_process(Table, Parent) -> + try ets:info(Table, size) of + N when is_integer(N) -> + size_process(Table, Parent); + undefined -> Parent ! table_gone; + E -> Parent ! {got_unexpected, E} + catch + E -> Parent ! {got_unexpected_exception, E} + end. + +start_slave() -> + MicroSecs = erlang:monotonic_time(), + Name = "ets_" ++ integer_to_list(MicroSecs), + Pa = filename:dirname(code:which(?MODULE)), + {ok, Node} = test_server:start_node(list_to_atom(Name), slave, [{args, "-pa " ++ Pa}]), + Node. + +repeat_par(FunToRepeat, NrOfTimes) -> + repeat_par_help(FunToRepeat, NrOfTimes, NrOfTimes). + +repeat_par_help(_FunToRepeat, 0, OrgNrOfTimes) -> + repeat(fun()-> receive done -> ok end end, OrgNrOfTimes); +repeat_par_help(FunToRepeat, NrOfTimes, OrgNrOfTimes) -> + Parent = self(), + case NrOfTimes rem 5 of + 0 -> timer:sleep(1); + _ -> ok + end, + spawn(fun()-> + FunToRepeat(), + Parent ! done + end), + repeat_par_help(FunToRepeat, NrOfTimes-1, OrgNrOfTimes). + %% Test various duplicate_bags stuff. dups(Config) when is_list(Config) -> repeat_for_opts(fun dups_do/1). @@ -4257,20 +4601,29 @@ dups_do(Opts) -> %% Test the ets:tab2file function on an empty ets table. tab2file(Config) when is_list(Config) -> FName = filename:join([proplists:get_value(priv_dir, Config),"tab2file_case"]), - tab2file_do(FName, []), - tab2file_do(FName, [{sync,true}]), - tab2file_do(FName, [{sync,false}]), - {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [{sync,yes}])), - {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [sync])), + tab2file_do(FName, [], set), + tab2file_do(FName, [], ordered_set), + tab2file_do(FName, [], cat_ord_set), + tab2file_do(FName, [], stim_cat_ord_set), + tab2file_do(FName, [{sync,true}], set), + tab2file_do(FName, [{sync,false}], set), + {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [{sync,yes}], set)), + {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [sync], set)), ok. -tab2file_do(FName, Opts) -> +tab2file_do(FName, Opts, TableType) -> %% Write an empty ets table to a file, read back and check properties. - Tab = ets_new(ets_SUITE_foo_tab, [named_table, set, public, + Tab = ets_new(ets_SUITE_foo_tab, [named_table, TableType, public, {keypos, 2}, compressed, {write_concurrency,true}, {read_concurrency,true}]), + ActualTableType = + case TableType of + cat_ord_set -> ordered_set; + stim_cat_ord_set -> ordered_set; + _ -> TableType + end, catch file:delete(FName), Res = ets:tab2file(Tab, FName, Opts), true = ets:delete(Tab), @@ -4281,7 +4634,7 @@ tab2file_do(FName, Opts) -> public = ets:info(Tab2, protection), true = ets:info(Tab2, named_table), 2 = ets:info(Tab2, keypos), - set = ets:info(Tab2, type), + ActualTableType = ets:info(Tab2, type), true = ets:info(Tab2, compressed), Smp = erlang:system_info(smp_support), Smp = ets:info(Tab2, read_concurrency), @@ -4294,14 +4647,15 @@ tab2file_do(FName, Opts) -> tab2file2(Config) when is_list(Config) -> repeat_for_opts(fun(Opts) -> tab2file2_do(Opts, Config) - end, [[set,bag],compressed]). + end, [[stim_cat_ord_set,cat_ord_set,set,bag],compressed]). tab2file2_do(Opts, Config) -> EtsMem = etsmem(), - Tab = ets_new(ets_SUITE_foo_tab, [named_table, private, - {keypos, 2} | Opts]), + KeyRange = 10000, + Tab = ets_new(ets_SUITE_foo_tab, [named_table, private, {keypos, 2} | Opts], + KeyRange), FName = filename:join([proplists:get_value(priv_dir, Config),"tab2file2_case"]), - ok = fill_tab2(Tab, 0, 10000), % Fill up the table (grucho mucho!) + ok = fill_tab2(Tab, 0, KeyRange), % Fill up the table (grucho mucho!) Len = length(ets:tab2list(Tab)), Mem = ets:info(Tab, memory), Type = ets:info(Tab, type), @@ -4355,13 +4709,14 @@ fill_tab2(Tab, Val, Num) -> %% Test verification of tables with object count extended_info. tabfile_ext1(Config) when is_list(Config) -> - repeat_for_opts(fun(Opts) -> tabfile_ext1_do(Opts, Config) end). + repeat_for_opts_all_set_table_types(fun(Opts) -> tabfile_ext1_do(Opts, Config) end). tabfile_ext1_do(Opts,Config) -> FName = filename:join([proplists:get_value(priv_dir, Config),"nisse.dat"]), FName2 = filename:join([proplists:get_value(priv_dir, Config),"countflip.dat"]), - L = lists:seq(1,10), - T = ets_new(x,Opts), + KeyRange = 10, + L = lists:seq(1,KeyRange), + T = ets_new(x,Opts,KeyRange), Name = make_ref(), [ets:insert(T,{X,integer_to_list(X)}) || X <- L], ok = ets:tab2file(T,FName,[{extended_info,[object_count]}]), @@ -4393,13 +4748,14 @@ tabfile_ext1_do(Opts,Config) -> %% Test verification of tables with md5sum extended_info. tabfile_ext2(Config) when is_list(Config) -> - repeat_for_opts(fun(Opts) -> tabfile_ext2_do(Opts,Config) end). + repeat_for_opts_all_set_table_types(fun(Opts) -> tabfile_ext2_do(Opts,Config) end). tabfile_ext2_do(Opts,Config) -> FName = filename:join([proplists:get_value(priv_dir, Config),"olle.dat"]), FName2 = filename:join([proplists:get_value(priv_dir, Config),"bitflip.dat"]), - L = lists:seq(1,10), - T = ets_new(x,Opts), + KeyRange = 10, + L = lists:seq(1, KeyRange), + T = ets_new(x, Opts, KeyRange), Name = make_ref(), [ets:insert(T,{X,integer_to_list(X)}) || X <- L], ok = ets:tab2file(T,FName,[{extended_info,[md5sum]}]), @@ -4430,71 +4786,77 @@ tabfile_ext2_do(Opts,Config) -> %% Test verification of (named) tables without extended info. tabfile_ext3(Config) when is_list(Config) -> - FName = filename:join([proplists:get_value(priv_dir, Config),"namn.dat"]), - FName2 = filename:join([proplists:get_value(priv_dir, Config),"ncountflip.dat"]), - L = lists:seq(1,10), - Name = make_ref(), - ?MODULE = ets_new(?MODULE,[named_table]), - [ets:insert(?MODULE,{X,integer_to_list(X)}) || X <- L], - ets:tab2file(?MODULE,FName), - {error,cannot_create_table} = ets:file2tab(FName), - true = ets:delete(?MODULE), - {ok,?MODULE} = ets:file2tab(FName), - true = ets:delete(?MODULE), - disk_log:open([{name,Name},{file,FName}]), - {_,[H2|T2]} = disk_log:chunk(Name,start), - disk_log:close(Name), - NewT2=lists:keydelete(8,1,T2), - file:delete(FName2), - disk_log:open([{name,Name},{file,FName2},{mode,read_write}]), - disk_log:log_terms(Name,[H2|NewT2]), - disk_log:close(Name), - 9 = length(ets:tab2list(element(2,ets:file2tab(FName2)))), - true = ets:delete(?MODULE), - {error,invalid_object_count} = ets:file2tab(FName2,[{verify,true}]), - {'EXIT',_} = (catch ets:delete(?MODULE)), - {ok,_} = ets:tabfile_info(FName2), - {ok,_} = ets:tabfile_info(FName), - file:delete(FName), - file:delete(FName2), + repeat_for_all_set_table_types( + fun(Opts) -> + FName = filename:join([proplists:get_value(priv_dir, Config),"namn.dat"]), + FName2 = filename:join([proplists:get_value(priv_dir, Config),"ncountflip.dat"]), + L = lists:seq(1,10), + Name = make_ref(), + ?MODULE = ets_new(?MODULE,[named_table|Opts]), + [ets:insert(?MODULE,{X,integer_to_list(X)}) || X <- L], + ets:tab2file(?MODULE,FName), + {error,cannot_create_table} = ets:file2tab(FName), + true = ets:delete(?MODULE), + {ok,?MODULE} = ets:file2tab(FName), + true = ets:delete(?MODULE), + disk_log:open([{name,Name},{file,FName}]), + {_,[H2|T2]} = disk_log:chunk(Name,start), + disk_log:close(Name), + NewT2=lists:keydelete(8,1,T2), + file:delete(FName2), + disk_log:open([{name,Name},{file,FName2},{mode,read_write}]), + disk_log:log_terms(Name,[H2|NewT2]), + disk_log:close(Name), + 9 = length(ets:tab2list(element(2,ets:file2tab(FName2)))), + true = ets:delete(?MODULE), + {error,invalid_object_count} = ets:file2tab(FName2,[{verify,true}]), + {'EXIT',_} = (catch ets:delete(?MODULE)), + {ok,_} = ets:tabfile_info(FName2), + {ok,_} = ets:tabfile_info(FName), + file:delete(FName), + file:delete(FName2) + end), ok. %% Tests verification of large table with md5 sum. tabfile_ext4(Config) when is_list(Config) -> - FName = filename:join([proplists:get_value(priv_dir, Config),"bauta.dat"]), - LL = lists:seq(1,10000), - TL = ets_new(x,[]), - Name2 = make_ref(), - [ets:insert(TL,{X,integer_to_list(X)}) || X <- LL], - ok = ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), - {ok, Name2} = disk_log:open([{name, Name2}, {file, FName}, - {mode, read_only}]), - {C,[_|_]} = disk_log:chunk(Name2,start), - {_,[_|_]} = disk_log:chunk(Name2,C), - disk_log:close(Name2), - true = lists:sort(ets:tab2list(TL)) =:= - lists:sort(ets:tab2list(element(2,ets:file2tab(FName)))), - Res = [begin - {ok,FD} = file:open(FName,[binary,read,write]), - {ok, Bin} = file:pread(FD,0,1000), - <<B1:N/binary,Ch:8,B2/binary>> = Bin, - Ch2 = (Ch + 1) rem 255, - Bin2 = <<B1/binary,Ch2:8,B2/binary>>, - ok = file:pwrite(FD,0,Bin2), - ok = file:close(FD), - X = case ets:file2tab(FName) of - {ok,TL2} -> - true = lists:sort(ets:tab2list(TL)) =/= - lists:sort(ets:tab2list(TL2)); - _ -> - totally_broken - end, - {error,Y} = ets:file2tab(FName,[{verify,true}]), - ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), - {X,Y} - end || N <- lists:seq(500,600)], - io:format("~p~n",[Res]), - file:delete(FName), + repeat_for_all_set_table_types( + fun(Opts) -> + FName = filename:join([proplists:get_value(priv_dir, Config),"bauta.dat"]), + LL = lists:seq(1,10000), + TL = ets_new(x,Opts), + Name2 = make_ref(), + [ets:insert(TL,{X,integer_to_list(X)}) || X <- LL], + ok = ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), + {ok, Name2} = disk_log:open([{name, Name2}, {file, FName}, + {mode, read_only}]), + {C,[_|_]} = disk_log:chunk(Name2,start), + {_,[_|_]} = disk_log:chunk(Name2,C), + disk_log:close(Name2), + true = lists:sort(ets:tab2list(TL)) =:= + lists:sort(ets:tab2list(element(2,ets:file2tab(FName)))), + Res = [begin + {ok,FD} = file:open(FName,[binary,read,write]), + {ok, Bin} = file:pread(FD,0,1000), + <<B1:N/binary,Ch:8,B2/binary>> = Bin, + Ch2 = (Ch + 1) rem 255, + Bin2 = <<B1/binary,Ch2:8,B2/binary>>, + ok = file:pwrite(FD,0,Bin2), + ok = file:close(FD), + X = case ets:file2tab(FName) of + {ok,TL2} -> + true = lists:sort(ets:tab2list(TL)) =/= + lists:sort(ets:tab2list(TL2)); + _ -> + totally_broken + end, + {error,Y} = ets:file2tab(FName,[{verify,true}]), + ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), + {X,Y} + end || N <- lists:seq(500,600)], + io:format("~p~n",[Res]), + file:delete(FName) + end), ok. %% Test that no disk_log is left open when file has been corrupted. @@ -4558,13 +4920,14 @@ make_sub_binary(List, Num) when is_list(List) -> %% Perform multiple lookups for every key in a large table. heavy_lookup(Config) when is_list(Config) -> - repeat_for_opts(fun heavy_lookup_do/1). + repeat_for_opts_all_set_table_types(fun heavy_lookup_do/1). heavy_lookup_do(Opts) -> EtsMem = etsmem(), - Tab = ets_new(foobar_table, [set, protected, {keypos, 2} | Opts]), - ok = fill_tab2(Tab, 0, 7000), - _ = [do_lookup(Tab, 6999) || _ <- lists:seq(1, 50)], + KeyRange = 7000, + Tab = ets_new(foobar_table, [{keypos, 2} | Opts], KeyRange), + ok = fill_tab2(Tab, 0, KeyRange), + _ = [do_lookup(Tab, KeyRange-1) || _ <- lists:seq(1, 50)], true = ets:delete(Tab), verify_etsmem(EtsMem). @@ -4581,15 +4944,16 @@ do_lookup(Tab, N) -> %% Perform multiple lookups for every element in a large table. heavy_lookup_element(Config) when is_list(Config) -> - repeat_for_opts(fun heavy_lookup_element_do/1). + repeat_for_opts_all_set_table_types(fun heavy_lookup_element_do/1). heavy_lookup_element_do(Opts) -> EtsMem = etsmem(), - Tab = ets_new(foobar_table, [set, protected, {keypos, 2} | Opts]), - ok = fill_tab2(Tab, 0, 7000), + KeyRange = 7000, + Tab = ets_new(foobar_table, [{keypos, 2} | Opts], KeyRange), + ok = fill_tab2(Tab, 0, KeyRange), %% lookup ALL elements 50 times Laps = 50 div syrup_factor(), - _ = [do_lookup_element(Tab, 6999, 1) || _ <- lists:seq(1, Laps)], + _ = [do_lookup_element(Tab, KeyRange-1, 1) || _ <- lists:seq(1, Laps)], true = ets:delete(Tab), verify_etsmem(EtsMem). @@ -4608,15 +4972,15 @@ do_lookup_element(Tab, N, M) -> heavy_concurrent(Config) when is_list(Config) -> - ct:timetrap({minutes,30}), %% valgrind needs a lot of time - repeat_for_opts(fun do_heavy_concurrent/1). + ct:timetrap({minutes,120}), %% valgrind needs a lot of time + repeat_for_opts_all_set_table_types(fun do_heavy_concurrent/1). do_heavy_concurrent(Opts) -> - Size = 10000, + KeyRange = 10000, Laps = 10000 div syrup_factor(), EtsMem = etsmem(), - Tab = ets_new(blupp, [set, public, {keypos, 2} | Opts]), - ok = fill_tab2(Tab, 0, Size), + Tab = ets_new(blupp, [public, {keypos, 2} | Opts], KeyRange), + ok = fill_tab2(Tab, 0, KeyRange), Procs = lists:map( fun (N) -> my_spawn_link( @@ -4649,48 +5013,68 @@ do_heavy_concurrent_proc(Tab, N, Offs) -> fold_empty(Config) when is_list(Config) -> - EtsMem = etsmem(), - Tab = make_table(a, [], []), - [] = ets:foldl(fun(_X) -> exit(hej) end, [], Tab), - [] = ets:foldr(fun(_X) -> exit(hej) end, [], Tab), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_opts_all_set_table_types( + fun(Opts) -> + EtsMem = etsmem(), + Tab = make_table(a, Opts, []), + [] = ets:foldl(fun(_X) -> exit(hej) end, [], Tab), + [] = ets:foldr(fun(_X) -> exit(hej) end, [], Tab), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end), + ok. foldl(Config) when is_list(Config) -> - EtsMem = etsmem(), - L = [{a,1}, {c,3}, {b,2}], - LS = lists:sort(L), - Tab = make_table(a, [bag], L), - LS = lists:sort(ets:foldl(fun(E,A) -> [E|A] end, [], Tab)), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_opts_all_table_types( + fun(Opts) -> + EtsMem = etsmem(), + L = [{a,1}, {c,3}, {b,2}], + LS = lists:sort(L), + Tab = make_table(a, Opts, L), + LS = lists:sort(ets:foldl(fun(E,A) -> [E|A] end, [], Tab)), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end), + ok. foldr(Config) when is_list(Config) -> - EtsMem = etsmem(), - L = [{a,1}, {c,3}, {b,2}], - LS = lists:sort(L), - Tab = make_table(a, [bag], L), - LS = lists:sort(ets:foldr(fun(E,A) -> [E|A] end, [], Tab)), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_opts_all_table_types( + fun(Opts) -> + EtsMem = etsmem(), + L = [{a,1}, {c,3}, {b,2}], + LS = lists:sort(L), + Tab = make_table(a, Opts, L), + LS = lists:sort(ets:foldr(fun(E,A) -> [E|A] end, [], Tab)), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end), + ok. foldl_ordered(Config) when is_list(Config) -> - EtsMem = etsmem(), - L = [{a,1}, {c,3}, {b,2}], - LS = lists:sort(L), - Tab = make_table(a, [ordered_set], L), - LS = lists:reverse(ets:foldl(fun(E,A) -> [E|A] end, [], Tab)), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_opts_all_ord_set_table_types( + fun(Opts) -> + EtsMem = etsmem(), + L = [{a,1}, {c,3}, {b,2}], + LS = lists:sort(L), + Tab = make_table(a, Opts, L), + LS = lists:reverse(ets:foldl(fun(E,A) -> [E|A] end, [], Tab)), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end), + ok. foldr_ordered(Config) when is_list(Config) -> - EtsMem = etsmem(), - L = [{a,1}, {c,3}, {b,2}], - LS = lists:sort(L), - Tab = make_table(a, [ordered_set], L), - LS = ets:foldr(fun(E,A) -> [E|A] end, [], Tab), - true = ets:delete(Tab), - verify_etsmem(EtsMem). + repeat_for_opts_all_ord_set_table_types( + fun(Opts) -> + EtsMem = etsmem(), + L = [{a,1}, {c,3}, {b,2}], + LS = lists:sort(L), + Tab = make_table(a, Opts, L), + LS = ets:foldr(fun(E,A) -> [E|A] end, [], Tab), + true = ets:delete(Tab), + verify_etsmem(EtsMem) + end), + ok. %% Test ets:member BIF. member(Config) when is_list(Config) -> @@ -4875,6 +5259,7 @@ filltabint(Tab,0) -> filltabint(Tab,N) -> ets:insert(Tab,{N,integer_to_list(N)}), filltabint(Tab,N-1). + filltabint2(Tab,0) -> Tab; filltabint2(Tab,N) -> @@ -5089,27 +5474,31 @@ gen_dets_filename(Config,N) -> "testdets_" ++ integer_to_list(N) ++ ".dets"). otp_6842_select_1000(Config) when is_list(Config) -> - Tab = ets_new(xxx,[ordered_set]), - [ets:insert(Tab,{X,X}) || X <- lists:seq(1,10000)], - AllTrue = lists:duplicate(10,true), - AllTrue = - [ length( - element(1, - ets:select(Tab,[{'_',[],['$_']}],X*1000))) =:= - X*1000 || X <- lists:seq(1,10) ], - Sequences = [[1000,1000,1000,1000,1000,1000,1000,1000,1000,1000], - [2000,2000,2000,2000,2000], - [3000,3000,3000,1000], - [4000,4000,2000], - [5000,5000], - [6000,4000], - [7000,3000], - [8000,2000], - [9000,1000], - [10000]], - AllTrue = [ check_seq(Tab, ets:select(Tab,[{'_',[],['$_']}],hd(L)),L) || - L <- Sequences ], - ets:delete(Tab), + repeat_for_opts_all_ord_set_table_types( + fun(Opts) -> + KeyRange = 10000, + Tab = ets_new(xxx, Opts, KeyRange), + [ets:insert(Tab,{X,X}) || X <- lists:seq(1,KeyRange)], + AllTrue = lists:duplicate(10,true), + AllTrue = + [ length( + element(1, + ets:select(Tab,[{'_',[],['$_']}],X*1000))) =:= + X*1000 || X <- lists:seq(1,10) ], + Sequences = [[1000,1000,1000,1000,1000,1000,1000,1000,1000,1000], + [2000,2000,2000,2000,2000], + [3000,3000,3000,1000], + [4000,4000,2000], + [5000,5000], + [6000,4000], + [7000,3000], + [8000,2000], + [9000,1000], + [10000]], + AllTrue = [ check_seq(Tab, ets:select(Tab,[{'_',[],['$_']}],hd(L)),L) || + L <- Sequences ], + ets:delete(Tab) + end), ok. check_seq(_,'$end_of_table',[]) -> @@ -5121,17 +5510,21 @@ check_seq(A,B,C) -> false. otp_6338(Config) when is_list(Config) -> - L = binary_to_term(<<131,108,0,0,0,2,104,2,108,0,0,0,2,103,100,0,19,112,112, - 98,49,95,98,115,49,50,64,98,108,97,100,101,95,48,95,53, - 0,0,33,50,0,0,0,4,1,98,0,0,23,226,106,100,0,4,101,120, - 105,116,104,2,108,0,0,0,2,104,2,100,0,3,115,98,109,100, - 0,19,112,112,98,50,95,98,115,49,50,64,98,108,97,100, - 101,95,48,95,56,98,0,0,18,231,106,100,0,4,114,101,99, - 118,106>>), - T = ets_new(xxx,[ordered_set]), - lists:foreach(fun(X) -> ets:insert(T,X) end,L), - [[4839,recv]] = ets:match(T,{[{sbm,ppb2_bs12@blade_0_8},'$1'],'$2'}), - ets:delete(T). + repeat_for_opts_all_ord_set_table_types( + fun(Opts) -> + L = binary_to_term(<<131,108,0,0,0,2,104,2,108,0,0,0,2,103,100,0,19,112,112, + 98,49,95,98,115,49,50,64,98,108,97,100,101,95,48,95,53, + 0,0,33,50,0,0,0,4,1,98,0,0,23,226,106,100,0,4,101,120, + 105,116,104,2,108,0,0,0,2,104,2,100,0,3,115,98,109,100, + 0,19,112,112,98,50,95,98,115,49,50,64,98,108,97,100, + 101,95,48,95,56,98,0,0,18,231,106,100,0,4,114,101,99, + 118,106>>), + T = ets_new(xxx,Opts), + lists:foreach(fun(X) -> ets:insert(T,X) end,L), + [[4839,recv]] = ets:match(T,{[{sbm,ppb2_bs12@blade_0_8},'$1'],'$2'}), + ets:delete(T) + end), + ok. %% OTP-15660: Verify select not doing excessive trapping %% when process have mbuf heap fragments. @@ -5256,7 +5649,7 @@ otp_7665_act(Tab,Min,Max,DelNr) -> %% Whitebox testing of meta name table hashing. meta_wb(Config) when is_list(Config) -> EtsMem = etsmem(), - repeat_for_opts(fun meta_wb_do/1), + repeat_for_opts_all_non_stim_table_types(fun meta_wb_do/1), verify_etsmem(EtsMem). @@ -5325,13 +5718,16 @@ colliding_names(Name) -> %% OTP_6913: Grow and shrink. grow_shrink(Config) when is_list(Config) -> - EtsMem = etsmem(), - - Set = ets_new(a, [set]), - grow_shrink_0(0, 3071, 3000, 5000, Set), - ets:delete(Set), - - verify_etsmem(EtsMem). + repeat_for_all_set_table_types( + fun(Opts) -> + EtsMem = etsmem(), + + Set = ets_new(a, Opts, 5000), + grow_shrink_0(0, 3071, 3000, 5000, Set), + ets:delete(Set), + + verify_etsmem(EtsMem) + end). grow_shrink_0(N, _, _, Max, _) when N >= Max -> ok; @@ -5355,7 +5751,7 @@ grow_shrink_3(N, ShrinkTo, T) -> true = ets:delete(T, N), grow_shrink_3(N-1, ShrinkTo, T). -%% Grow a table that still contains pseudo-deleted objects. +%% Grow a hash table that still contains pseudo-deleted objects. grow_pseudo_deleted(Config) when is_list(Config) -> only_if_smp(fun() -> grow_pseudo_deleted_do() end). @@ -5408,7 +5804,7 @@ grow_pseudo_deleted_do(Type) -> ets:delete(T), process_flag(scheduler,0). -%% Shrink a table that still contains pseudo-deleted objects. +%% Shrink a hash table that still contains pseudo-deleted objects. shrink_pseudo_deleted(Config) when is_list(Config) -> only_if_smp(fun()->shrink_pseudo_deleted_do() end). @@ -5528,10 +5924,16 @@ meta_newdel_named(Config) when is_list(Config) -> %% Concurrent insert's on same table. smp_insert(Config) when is_list(Config) -> - ets_new(smp_insert,[named_table,public,{write_concurrency,true}]), + repeat_for_opts(fun smp_insert_do/1, + [[set,ordered_set,stim_cat_ord_set]]). + +smp_insert_do(Opts) -> + KeyRange = 10000, + ets_new(smp_insert,[named_table,public,{write_concurrency,true}|Opts], + KeyRange), InitF = fun(_) -> ok end, - ExecF = fun(_) -> true = ets:insert(smp_insert,{rand:uniform(10000)}) - end, + ExecF = fun(_) -> true = ets:insert(smp_insert,{rand:uniform(KeyRange)}) + end, FiniF = fun(_) -> ok end, run_smp_workers(InitF,ExecF,FiniF,100000), verify_table_load(smp_insert), @@ -5539,7 +5941,7 @@ smp_insert(Config) when is_list(Config) -> %% Concurrent deletes on same fixated table. smp_fixed_delete(Config) when is_list(Config) -> - only_if_smp(fun()->smp_fixed_delete_do() end). + only_if_smp(fun() -> smp_fixed_delete_do() end). smp_fixed_delete_do() -> T = ets_new(foo,[public,{write_concurrency,true}]), @@ -5550,17 +5952,20 @@ smp_fixed_delete_do() -> Buckets = num_of_buckets(T), InitF = fun([ProcN,NumOfProcs|_]) -> {ProcN,NumOfProcs} end, ExecF = fun({Key,_}) when Key > NumOfObjs -> - [end_of_work]; - ({Key,Increment}) -> - true = ets:delete(T,Key), - {Key+Increment,Increment} - end, + [end_of_work]; + ({Key,Increment}) -> + true = ets:delete(T,Key), + {Key+Increment,Increment} + end, FiniF = fun(_) -> ok end, run_sched_workers(InitF,ExecF,FiniF,NumOfObjs), 0 = ets:info(T,size), true = ets:info(T,fixed), Buckets = num_of_buckets(T), - NumOfObjs = get_kept_objects(T), + case ets:info(T,type) of + set -> NumOfObjs = get_kept_objects(T); + _ -> ok + end, ets:safe_fixtable(T,false), %% Will fail as unfix does not shrink the table: %%Mem = ets:info(T,memory), @@ -5608,7 +6013,12 @@ delete_unfix_race(Config) when is_list(Config) -> verify_etsmem(EtsMem). num_of_buckets(T) -> - element(1,ets:info(T,stats)). + case ets:info(T,type) of + set -> element(1,ets:info(T,stats)); + bag -> element(1,ets:info(T,stats)); + duplicate_bag -> element(1,ets:info(T,stats)); + _ -> ok + end. %% Fixate hash table while other process is busy doing unfix. smp_unfix_fix(Config) when is_list(Config) -> @@ -5773,106 +6183,126 @@ otp_8166_zombie_creator(T,Deleted) -> verify_table_load(T) -> - Stats = ets:info(T,stats), - {Buckets,AvgLen,StdDev,ExpSD,_MinLen,_MaxLen,_} = Stats, - ok = if - AvgLen > 1.2 -> - io:format("Table overloaded: Stats=~p\n~p\n", - [Stats, ets:info(T)]), - false; - - Buckets>256, AvgLen < 0.47 -> - io:format("Table underloaded: Stats=~p\n~p\n", - [Stats, ets:info(T)]), - false; - - StdDev > ExpSD*2 -> - io:format("Too large standard deviation (poor hashing?)," - " stats=~p\n~p\n",[Stats, ets:info(T)]), - false; - - true -> - io:format("Stats = ~p\n",[Stats]), - ok - end. + case ets:info(T,type) of + ordered_set -> ok; + _ -> + Stats = ets:info(T,stats), + {Buckets,AvgLen,StdDev,ExpSD,_MinLen,_MaxLen,_} = Stats, + ok = if + AvgLen > 1.2 -> + io:format("Table overloaded: Stats=~p\n~p\n", + [Stats, ets:info(T)]), + false; + + Buckets>256, AvgLen < 0.47 -> + io:format("Table underloaded: Stats=~p\n~p\n", + [Stats, ets:info(T)]), + false; + + StdDev > ExpSD*2 -> + io:format("Too large standard deviation (poor hashing?)," + " stats=~p\n~p\n",[Stats, ets:info(T)]), + false; + + true -> + io:format("Stats = ~p\n",[Stats]), + ok + end + end. %% ets:select on a tree with NIL key object. otp_8732(Config) when is_list(Config) -> - Tab = ets_new(noname,[ordered_set]), - filltabstr(Tab,999), - ets:insert(Tab,{[],"nasty NIL object"}), - [] = ets:match(Tab,{'_',nomatch}), %% Will hang if bug not fixed + repeat_for_all_ord_set_table_types( + fun(Opts) -> + KeyRange = 999, + KeyFun = fun(K) -> integer_to_list(K) end, + Tab = ets_new(noname,Opts, KeyRange, KeyFun), + filltabstr(Tab, KeyRange), + ets:insert(Tab,{[],"nasty NIL object"}), + [] = ets:match(Tab,{'_',nomatch}) %% Will hang if bug not fixed + end), ok. %% Run concurrent select_delete (and inserts) on same table. smp_select_delete(Config) when is_list(Config) -> - T = ets_new(smp_select_delete,[named_table,public,{write_concurrency,true}]), - Mod = 17, - Zeros = erlang:make_tuple(Mod,0), - InitF = fun(_) -> Zeros end, - ExecF = fun(Diffs0) -> - case rand:uniform(20) of - 1 -> - Mod = 17, - Eq = rand:uniform(Mod) - 1, - Deleted = ets:select_delete(T, - [{{'_', '$1'}, - [{'=:=', {'rem', '$1', Mod}, Eq}], - [true]}]), - Diffs1 = setelement(Eq+1, Diffs0, - element(Eq+1,Diffs0) - Deleted), - Diffs1; - _ -> - Key = rand:uniform(10000), - Eq = Key rem Mod, - case ets:insert_new(T,{Key,Key}) of - true -> - Diffs1 = setelement(Eq+1, Diffs0, - element(Eq+1,Diffs0)+1), - Diffs1; - false -> Diffs0 - end - end - end, - FiniF = fun(Result) -> Result end, - Results = run_sched_workers(InitF,ExecF,FiniF,20000), - TotCnts = lists:foldl(fun(Diffs, Sum) -> add_lists(Sum,tuple_to_list(Diffs)) end, - lists:duplicate(Mod, 0), Results), - io:format("TotCnts = ~p\n",[TotCnts]), - LeftInTab = lists:foldl(fun(N,Sum) -> Sum+N end, - 0, TotCnts), - io:format("LeftInTab = ~p\n",[LeftInTab]), - LeftInTab = ets:info(T,size), - lists:foldl(fun(Cnt,Eq) -> - WasCnt = ets:select_count(T, - [{{'_', '$1'}, - [{'=:=', {'rem', '$1', Mod}, Eq}], - [true]}]), - io:format("~p: ~p =?= ~p\n",[Eq,Cnt,WasCnt]), - Cnt = WasCnt, - Eq+1 - end, - 0, TotCnts), - %% May fail as select_delete does not shrink table (enough) - %%verify_table_load(T), - LeftInTab = ets:select_delete(T, [{{'$1','$1'}, [], [true]}]), - 0 = ets:info(T,size), - false = ets:info(T,fixed), - ets:delete(T). + repeat_for_opts(fun smp_select_delete_do/1, + [[set,ordered_set,stim_cat_ord_set], + read_concurrency, compressed]). + +smp_select_delete_do(Opts) -> + KeyRange = 10000, + begin % indentation + T = ets_new(smp_select_delete,[named_table,public,{write_concurrency,true}|Opts], + KeyRange), + Mod = 17, + Zeros = erlang:make_tuple(Mod,0), + InitF = fun(_) -> Zeros end, + ExecF = fun(Diffs0) -> + case rand:uniform(20) of + 1 -> + Mod = 17, + Eq = rand:uniform(Mod) - 1, + Deleted = ets:select_delete(T, + [{{'_', '$1'}, + [{'=:=', {'rem', '$1', Mod}, Eq}], + [true]}]), + Diffs1 = setelement(Eq+1, Diffs0, + element(Eq+1,Diffs0) - Deleted), + Diffs1; + _ -> + Key = rand:uniform(KeyRange), + Eq = Key rem Mod, + case ets:insert_new(T,{Key,Key}) of + true -> + Diffs1 = setelement(Eq+1, Diffs0, + element(Eq+1,Diffs0)+1), + Diffs1; + false -> Diffs0 + end + end + end, + FiniF = fun(Result) -> Result end, + Results = run_sched_workers(InitF,ExecF,FiniF,20000), + TotCnts = lists:foldl(fun(Diffs, Sum) -> add_lists(Sum,tuple_to_list(Diffs)) end, + lists:duplicate(Mod, 0), Results), + io:format("TotCnts = ~p\n",[TotCnts]), + LeftInTab = lists:foldl(fun(N,Sum) -> Sum+N end, + 0, TotCnts), + io:format("LeftInTab = ~p\n",[LeftInTab]), + LeftInTab = ets:info(T,size), + lists:foldl(fun(Cnt,Eq) -> + WasCnt = ets:select_count(T, + [{{'_', '$1'}, + [{'=:=', {'rem', '$1', Mod}, Eq}], + [true]}]), + io:format("~p: ~p =?= ~p\n",[Eq,Cnt,WasCnt]), + Cnt = WasCnt, + Eq+1 + end, + 0, TotCnts), + %% May fail as select_delete does not shrink table (enough) + %%verify_table_load(T), + LeftInTab = ets:select_delete(T, [{{'$1','$1'}, [], [true]}]), + 0 = ets:info(T,size), + false = ets:info(T,fixed), + ets:delete(T) + end, % indentation + ok. smp_select_replace(Config) when is_list(Config) -> repeat_for_opts(fun smp_select_replace_do/1, - [[set,ordered_set,duplicate_bag]]). + [[set,ordered_set,stim_cat_ord_set,duplicate_bag]]). smp_select_replace_do(Opts) -> + KeyRange = 20, T = ets_new(smp_select_replace, - [public, {write_concurrency, true} | Opts]), - ObjCount = 20, + [public, {write_concurrency, true} | Opts], + KeyRange), InitF = fun (_) -> 0 end, ExecF = fun (Cnt0) -> - CounterId = rand:uniform(ObjCount), + CounterId = rand:uniform(KeyRange), Match = [{{'$1', '$2'}, [{'=:=', '$1', CounterId}], [{{'$1', {'+', '$2', 1}}}]}], @@ -5896,15 +6326,143 @@ smp_select_replace_do(Opts) -> FinalCounts = ets:select(T, [{{'_', '$1'}, [], ['$1']}]), Total = lists:sum(FinalCounts), Total = lists:sum(Results), - ObjCount = ets:select_delete(T, [{{'_', '_'}, [], [true]}]), + KeyRange = ets:select_delete(T, [{{'_', '_'}, [], [true]}]), 0 = ets:info(T, size), true = ets:delete(T), ok. +%% Iterate ordered_set with write_concurrency +%% and make sure we hit all "stable" long lived keys +%% while "volatile" objects are randomly inserted and deleted. +smp_ordered_iteration(Config) when is_list(Config) -> + repeat_for_opts(fun smp_ordered_iteration_do/1, + [[cat_ord_set,stim_cat_ord_set]]). + + +smp_ordered_iteration_do(Opts) -> + KeyRange = 1000, + OffHeap = erts_test_utils:mk_ext_pid({a@b,1}, 4711, 1), + KeyFun = fun(K, Type) -> + {K div 10, K rem 10, Type, OffHeap} + end, + StimKeyFun = fun(K) -> + KeyFun(K, element(rand:uniform(3), + {stable, other, volatile})) + end, + T = ets_new(smp_ordered_iteration, [public, {write_concurrency,true} | Opts], + KeyRange, StimKeyFun), + NStable = KeyRange div 4, + prefill_table(T, KeyRange, NStable, fun(K) -> {KeyFun(K, stable), 0} end), + NStable = ets:info(T, size), + NVolatile = KeyRange div 2, + prefill_table(T, KeyRange, NVolatile, fun(K) -> {KeyFun(K, volatile), 0} end), + + InitF = fun (_) -> #{insert => 0, delete => 0, + select_delete_bk => 0, select_delete_pbk => 0, + select_replace_bk => 0, select_replace_pbk => 0} + end, + ExecF = fun (Counters) -> + K = rand:uniform(KeyRange), + Key = KeyFun(K, volatile), + Acc = case rand:uniform(22) of + R when R =< 10 -> + ets:insert(T, {Key}), + incr_counter(insert, Counters); + R when R =< 15 -> + ets:delete(T, Key), + incr_counter(delete, Counters); + R when R =< 19 -> + %% Delete bound key + ets:select_delete(T, [{{Key, '_'}, [], [true]}]), + incr_counter(select_delete_bk, Counters); + R when R =< 20 -> + %% Delete partially bound key + ets:select_delete(T, [{{{K div 10, '_', volatile, '_'}, '_'}, [], [true]}]), + incr_counter(select_delete_pbk, Counters); + R when R =< 21 -> + %% Replace bound key + ets:select_replace(T, [{{Key, '$1'}, [], + [{{{const,Key}, {'+','$1',1}}}]}]), + incr_counter(select_replace_bk, Counters); + _ -> + %% Replace partially bound key + ets:select_replace(T, [{{{K div 10, '_', volatile, '_'}, '$1'}, [], + [{{{element,1,'$_'}, {'+','$1',1}}}]}]), + incr_counter(select_replace_pbk, Counters) + end, + receive stop -> + [end_of_work | Acc] + after 0 -> + Acc + end + end, + FiniF = fun (Acc) -> Acc end, + Pids = run_sched_workers(InitF, ExecF, FiniF, infinite), + timer:send_after(1000, stop), + + Log2ChunkMax = math:log2(NStable*2), + Rounds = fun Loop(N) -> + MS = [{{{'_', '_', stable, '_'}, '_'}, [], [true]}], + NStable = ets:select_count(T, MS), + NStable = count_stable(T, next, ets:first(T), 0), + NStable = count_stable(T, prev, ets:last(T), 0), + NStable = length(ets:select(T, MS)), + NStable = length(ets:select_reverse(T, MS)), + Chunk = round(math:pow(2, rand:uniform()*Log2ChunkMax)), + NStable = ets_select_chunks_count(T, MS, Chunk), + receive stop -> N + after 0 -> Loop(N+1) + end + end (1), + [P ! stop || P <- Pids], + Results = wait_pids(Pids), + io:format("Ops = ~p\n", [maps_sum(Results)]), + io:format("Diff = ~p\n", [ets:info(T,size) - NStable - NVolatile]), + io:format("Stats = ~p\n", [ets:info(T,stats)]), + io:format("Rounds = ~p\n", [Rounds]), + true = ets:delete(T), + + %% Verify no leakage of offheap key data + ok = erts_test_utils:check_node_dist(), + ok. + +incr_counter(Name, Counters) -> + Counters#{Name => maps:get(Name, Counters, 0) + 1}. + +count_stable(T, Next, {_, _, stable, _}=Key, N) -> + count_stable(T, Next, ets:Next(T, Key), N+1); +count_stable(T, Next, {_, _, volatile, _}=Key, N) -> + count_stable(T, Next, ets:Next(T, Key), N); +count_stable(_, _, '$end_of_table', N) -> + N. + +ets_select_chunks_count(T, MS, Chunk) -> + ets_select_chunks_count(ets:select(T, MS, Chunk), 0). + +ets_select_chunks_count('$end_of_table', N) -> + N; +ets_select_chunks_count({List, Continuation}, N) -> + ets_select_chunks_count(ets:select(Continuation), + length(List) + N). + +maps_sum([Ma | Tail]) when is_map(Ma) -> + maps_sum([lists:sort(maps:to_list(Ma)) | Tail]); +maps_sum([La, Mb | Tail]) -> + Lab = lists:zipwith(fun({K,Va}, {K,Vb}) -> {K,Va+Vb} end, + La, + lists:sort(maps:to_list(Mb))), + maps_sum([Lab | Tail]); +maps_sum([L]) -> + L. + + + + %% Test different types. types(Config) when is_list(Config) -> init_externals(), - repeat_for_opts(fun types_do/1, [[set,ordered_set],compressed]). + repeat_for_opts(fun types_do/1, [repeat_for_opts_atom2list(set_types), + compressed]). types_do(Opts) -> EtsMem = etsmem(), @@ -5931,7 +6489,7 @@ types_do(Opts) -> %% OTP-9932: Memory overwrite when inserting large integers in compressed bag. %% Will crash with segv on 64-bit opt if not fixed. otp_9932(Config) when is_list(Config) -> - T = ets:new(xxx, [bag, compressed]), + T = ets_new(xxx, [bag, compressed]), Fun = fun(N) -> Key = {1316110174588445 bsl N,1316110174588583 bsl N}, S = {Key, Key}, @@ -5947,48 +6505,56 @@ otp_9932(Config) when is_list(Config) -> %% vm-deadlock caused by race between ets:delete and others on %% write_concurrency table. otp_9423(Config) when is_list(Config) -> - InitF = fun(_) -> {0,0} end, - ExecF = fun({S,F}) -> - receive - stop -> - io:format("~p got stop\n", [self()]), - [end_of_work | {"Succeded=",S,"Failed=",F}] - after 0 -> - %%io:format("~p (~p) doing lookup\n", [self(), {S,F}]), - try ets:lookup(otp_9423, key) of - [] -> {S+1,F} - catch - error:badarg -> {S,F+1} - end - end - end, - FiniF = fun(R) -> R end, - case run_smp_workers(InitF, ExecF, FiniF, infinite, 1) of - Pids when is_list(Pids) -> - %%[P ! start || P <- Pids], - repeat(fun() -> ets:new(otp_9423, [named_table, public, {write_concurrency,true}]), - ets:delete(otp_9423) - end, 10000), - [P ! stop || P <- Pids], - wait_pids(Pids), - ok; + repeat_for_all_non_stim_set_table_types( + fun(Opts) -> + InitF = fun(_) -> {0,0} end, + ExecF = fun({S,F}) -> + receive + stop -> + io:format("~p got stop\n", [self()]), + [end_of_work | {"Succeded=",S,"Failed=",F}] + after 0 -> + %%io:format("~p (~p) doing lookup\n", [self(), {S,F}]), + try ets:lookup(otp_9423, key) of + [] -> {S+1,F} + catch + error:badarg -> {S,F+1} + end + end + end, + FiniF = fun(R) -> R end, + case run_smp_workers(InitF, ExecF, FiniF, infinite, 1) of + Pids when is_list(Pids) -> + %%[P ! start || P <- Pids], + repeat(fun() -> ets_new(otp_9423, [named_table, public, + {write_concurrency,true}|Opts]), + ets:delete(otp_9423) + end, 10000), + [P ! stop || P <- Pids], + wait_pids(Pids), + ok; + + Skipped -> Skipped + end + end). - Skipped -> Skipped - end. %% Corrupted binary in compressed table otp_10182(Config) when is_list(Config) -> - Bin = <<"aHR0cDovL2hvb3RzdWl0ZS5jb20vYy9wcm8tYWRyb2xsLWFi">>, - Key = {test, Bin}, - Value = base64:decode(Bin), - In = {Key,Value}, - Db = ets:new(undefined, [set, protected, {read_concurrency, true}, compressed]), - ets:insert(Db, In), - [Out] = ets:lookup(Db, Key), - io:format("In : ~p\nOut: ~p\n", [In,Out]), - ets:delete(Db), - In = Out. + repeat_for_opts_all_table_types( + fun(Opts) -> + Bin = <<"aHR0cDovL2hvb3RzdWl0ZS5jb20vYy9wcm8tYWRyb2xsLWFi">>, + Key = {test, Bin}, + Value = base64:decode(Bin), + In = {Key,Value}, + Db = ets_new(undefined, Opts), + ets:insert(Db, In), + [Out] = ets:lookup(Db, Key), + io:format("In : ~p\nOut: ~p\n", [In,Out]), + ets:delete(Db), + In = Out + end). %% Test that ets:all include/exclude tables that we know are created/deleted ets_all(Config) when is_list(Config) -> @@ -6079,19 +6645,23 @@ take(Config) when is_list(Config) -> ets:insert(T1, {{'not',<<"immediate">>},ok}), [{{'not',<<"immediate">>},ok}] = ets:take(T1, {'not',<<"immediate">>}), %% Same with ordered tables. - T2 = ets_new(b, [ordered_set]), - [] = ets:take(T2, foo), - ets:insert(T2, {foo,bar}), - [] = ets:take(T2, bar), - [{foo,bar}] = ets:take(T2, foo), - [] = ets:tab2list(T2), - ets:insert(T2, {{'not',<<"immediate">>},ok}), - [{{'not',<<"immediate">>},ok}] = ets:take(T2, {'not',<<"immediate">>}), - %% Arithmetically-equal keys. - ets:insert(T2, [{1.0,float},{2,integer}]), - [{1.0,float}] = ets:take(T2, 1), - [{2,integer}] = ets:take(T2, 2.0), - [] = ets:tab2list(T2), + repeat_for_all_ord_set_table_types( + fun(Opts) -> + T2 = ets_new(b, Opts), + [] = ets:take(T2, foo), + ets:insert(T2, {foo,bar}), + [] = ets:take(T2, bar), + [{foo,bar}] = ets:take(T2, foo), + [] = ets:tab2list(T2), + ets:insert(T2, {{'not',<<"immediate">>},ok}), + [{{'not',<<"immediate">>},ok}] = ets:take(T2, {'not',<<"immediate">>}), + %% Arithmetically-equal keys. + ets:insert(T2, [{1.0,float},{2,integer}]), + [{1.0,float}] = ets:take(T2, 1), + [{2,integer}] = ets:take(T2, 2.0), + [] = ets:tab2list(T2), + ets:delete(T2) + end), %% Same with bag. T3 = ets_new(c, [bag]), ets:insert(T3, [{1,1},{1,2},{3,3}]), @@ -6099,7 +6669,6 @@ take(Config) when is_list(Config) -> [{3,3}] = ets:take(T3, 3), [] = ets:tab2list(T3), ets:delete(T1), - ets:delete(T2), ets:delete(T3), ok. @@ -6134,9 +6703,518 @@ whereis_table(Config) when is_list(Config) -> ok. -%% -%% Utility functions: -%% + +%% The following help functions are used by +%% throughput_benchmark. They are declared on the top level beacuse +%% declaring them as function local funs cause a scalability issue. +get_op([{_,O}], _RandNum) -> + O; +get_op([{Prob,O}|Rest], RandNum) -> + case RandNum < Prob of + true -> O; + false -> get_op(Rest, RandNum) + end. +do_op(Table, ProbHelpTab, Range, Operations) -> + RandNum = rand:uniform(), + Op = get_op(ProbHelpTab, RandNum), + #{ Op := TheOp} = Operations, + TheOp(Table, Range). +do_work(WorksDoneSoFar, Table, ProbHelpTab, Range, Operations) -> + receive + stop -> WorksDoneSoFar + after + 0 -> do_op(Table, ProbHelpTab, Range, Operations), + do_work(WorksDoneSoFar + 1, Table, ProbHelpTab, Range, Operations) + end. + +prefill_table(T, KeyRange, Num, ObjFun) -> + Seed = rand:uniform(KeyRange), + %%io:format("prefill_table: Seed = ~p\n", [Seed]), + RState = unique_rand_start(KeyRange, Seed), + prefill_table_loop(T, RState, Num, ObjFun). + +prefill_table_loop(_, _, 0, _) -> + ok; +prefill_table_loop(T, RS0, N, ObjFun) -> + {Key, RS1} = unique_rand_next(RS0), + ets:insert(T, ObjFun(Key)), + prefill_table_loop(T, RS1, N-1, ObjFun). + +-record(ets_throughput_bench_config, + {benchmark_duration_ms = 3000, + recover_time_ms = 1000, + thread_counts = not_set, + key_ranges = [1000000], + scenarios = + [ + [ + {0.5, insert}, + {0.5, delete} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.8, lookup} + ], + [ + {0.01, insert}, + {0.01, delete}, + {0.98, lookup} + ], + [ + {1.0, lookup} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq10} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq100} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq1000} + ], + [ + {1.0, nextseq1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, partial_select1000} + ] + ], + table_types = + [ + [ordered_set, public], + [ordered_set, public, {write_concurrency, true}], + [ordered_set, public, {read_concurrency, true}], + [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], + [set, public], + [set, public, {write_concurrency, true}], + [set, public, {read_concurrency, true}], + [set, public, {write_concurrency, true}, {read_concurrency, true}] + ], + etsmem_fun = fun() -> ok end, + verify_etsmem_fun = fun(_) -> true end, + notify_res_fun = fun(_Name, _Throughput) -> ok end, + print_result_paths_fun = + fun(ResultPath, _LatestResultPath) -> + Comment = + io_lib:format("<a href=\"file:///~s\">Result visualization</a>",[ResultPath]), + {comment, Comment} + end + }). + +stdout_notify_res(ResultPath, LatestResultPath) -> + io:format("Result Location: /~s~n", [ResultPath]), + io:format("Latest Result Location: ~s~n", [LatestResultPath]). + +throughput_benchmark() -> + throughput_benchmark( + #ets_throughput_bench_config{ + print_result_paths_fun = fun stdout_notify_res/2}). + +throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = BenchmarkDurationMs, + recover_time_ms = RecoverTimeMs, + thread_counts = ThreadCountsOpt, + key_ranges = KeyRanges, + scenarios = Scenarios, + table_types = TableTypes, + etsmem_fun = ETSMemFun, + verify_etsmem_fun = VerifyETSMemFun, + notify_res_fun = NotifyResFun, + print_result_paths_fun = PrintResultPathsFun}) -> + NrOfSchedulers = erlang:system_info(schedulers), + %% Definitions of operations that are supported by the benchmark + NextSeqOp = + fun (T, KeyRange, SeqSize) -> + Start = rand:uniform(KeyRange), + Last = + lists:foldl( + fun(_, Prev) -> + case Prev of + '$end_of_table'-> ok; + _ -> + try ets:next(T, Prev) of + Normal -> Normal + catch + error:badarg -> + % sets (not ordered_sets) cannot handle when the argument + % to next is not in the set + rand:uniform(KeyRange) + end + end + end, + Start, + lists:seq(1, SeqSize)), + case Last =:= -1 of + true -> io:format("Will never be printed"); + false -> ok + end + end, + PartialSelectOp = + fun (T, KeyRange, SeqSize) -> + Start = rand:uniform(KeyRange), + Last = Start + SeqSize, + case -1 =:= ets:select_count(T, + ets:fun2ms(fun({X}) when X > Start andalso X =< Last -> true end)) of + true -> io:format("Will never be printed"); + false -> ok + end + + end, + %% Mapping benchmark operation names to their corresponding functions that do them + Operations = + #{insert => + fun(T,KeyRange) -> + Num = rand:uniform(KeyRange), + ets:insert(T, {Num}) + end, + delete => + fun(T,KeyRange) -> + Num = rand:uniform(KeyRange), + ets:delete(T, Num) + end, + lookup => + fun(T,KeyRange) -> + Num = rand:uniform(KeyRange), + ets:lookup(T, Num) + end, + nextseq10 => + fun(T,KeyRange) -> NextSeqOp(T,KeyRange,10) end, + nextseq100 => + fun(T,KeyRange) -> NextSeqOp(T,KeyRange,100) end, + nextseq1000 => + fun(T,KeyRange) -> NextSeqOp(T,KeyRange,1000) end, + selectAll => + fun(T,_KeyRange) -> + case -1 =:= ets:select_count(T, ets:fun2ms(fun(_X) -> true end)) of + true -> io:format("Will never be printed"); + false -> ok + end + end, + partial_select1000 => + fun(T,KeyRange) -> PartialSelectOp(T,KeyRange,1000) end + }, + %% Helper functions + CalculateThreadCounts = fun Calculate([Count|Rest]) -> + case Count > NrOfSchedulers of + true -> lists:reverse(Rest); + false -> Calculate([Count*2,Count|Rest]) + end + end, + CalculateOpsProbHelpTab = + fun Calculate([{_, OpName}], _) -> + [{1.0, OpName}]; + Calculate([{OpPropability, OpName}|Res], Current) -> + NewCurrent = Current + OpPropability, + [{NewCurrent, OpName}| Calculate(Res, NewCurrent)] + end, + RenderScenario = + fun R([], StringSoFar) -> + StringSoFar; + R([{Fraction, Operation}], StringSoFar) -> + io_lib:format("~s ~f% ~p",[StringSoFar, Fraction * 100.0, Operation]); + R([{Fraction, Operation}|Rest], StringSoFar) -> + R(Rest, + io_lib:format("~s ~f% ~p, ",[StringSoFar, Fraction * 100.0, Operation])) + end, + SafeFixTableIfRequired = + fun(Table, Scenario, On) -> + case set =:= ets:info(Table, type) of + true -> + HasNotRequiringOp = + lists:search( + fun({_,nextseq10}) -> true; + ({_,nextseq100}) -> true; + ({_,nextseq1000}) -> true; + (_) -> false + end, Scenario), + case HasNotRequiringOp of + false -> ok; + _ -> ets:safe_fixtable(Table, On) + end; + false -> ok + end + end, + DataHolder = + fun DataHolderFun(Data)-> + receive + {get_data, Pid} -> Pid ! {ets_bench_data, Data}; + D -> DataHolderFun([Data,D]) + end + end, + DataHolderPid = spawn_link(fun()-> DataHolder([]) end), + PrintData = + fun (Str, List) -> + io:format(Str, List), + DataHolderPid ! io_lib:format(Str, List) + end, + GetData = + fun () -> + DataHolderPid ! {get_data, self()}, + receive {ets_bench_data, Data} -> Data end + end, + %% Function that runs a benchmark instance and returns the number + %% of operations that were performed + RunBenchmark = + fun({NrOfProcs, TableConfig, Scenario, Range, Duration}) -> + ProbHelpTab = CalculateOpsProbHelpTab(Scenario, 0), + Table = ets:new(t, TableConfig), + Nobj = Range div 2, + prefill_table(Table, Range, Nobj, fun(K) -> {K} end), + Nobj = ets:info(Table, size), + SafeFixTableIfRequired(Table, Scenario, true), + ParentPid = self(), + Worker = + fun() -> + receive start -> ok end, + WorksDone = + do_work(0, Table, ProbHelpTab, Range, Operations), + ParentPid ! WorksDone + end, + ChildPids = + lists:map(fun(_N) ->spawn_link(Worker)end, lists:seq(1, NrOfProcs)), + lists:foreach(fun(Pid) -> Pid ! start end, ChildPids), + timer:sleep(Duration), + lists:foreach(fun(Pid) -> Pid ! stop end, ChildPids), + TotalWorksDone = lists:foldl( + fun(_, Sum) -> + receive + Count -> Sum + Count + end + end, 0, ChildPids), + SafeFixTableIfRequired(Table, Scenario, false), + ets:delete(Table), + TotalWorksDone + end, + RunBenchmarkInSepProcess = + fun(ParameterTuple) -> + P = self(), + spawn_link(fun()-> P ! {bench_result, RunBenchmark(ParameterTuple)} end), + Result = receive {bench_result, Res} -> Res end, + timer:sleep(RecoverTimeMs), + Result + end, + RunBenchmarkAndReport = + fun(ThreadCount, + TableType, + Scenario, + KeyRange, + Duration) -> + Result = RunBenchmarkInSepProcess({ThreadCount, + TableType, + Scenario, + KeyRange, + Duration}), + Throughput = Result/(Duration/1000.0), + PrintData("; ~f",[Throughput]), + Name = io_lib:format("Scenario: ~w, Key Range Size: ~w, " + "# of Processes: ~w, Table Type: ~w", + [Scenario, KeyRange, ThreadCount, TableType]), + NotifyResFun(Name, Throughput) + end, + ThreadCounts = + case ThreadCountsOpt of + not_set -> + CalculateThreadCounts([1]); + _ -> ThreadCountsOpt + end, + %% Run the benchmark + PrintData("# Each instance of the benchmark runs for ~w seconds:~n", [BenchmarkDurationMs/1000]), + PrintData("# The result of a benchmark instance is presented as a number representing~n",[]), + PrintData("# the number of operations performed per second:~n~n~n",[]), + PrintData("# To plot graphs for the results below:~n",[]), + PrintData("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n",[]), + PrintData("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n",[]), + PrintData("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n",[]), + PrintData("# step 1 and press the Render button~n~n",[]), + PrintData("#BENCHMARK STARTED$~n",[]), + EtsMem = ETSMemFun(), + %% The following loop runs all benchmark scenarios and prints the results (i.e, operations/second) + lists:foreach( + fun(KeyRange) -> + lists:foreach( + fun(Scenario) -> + PrintData("Scenario: ~s | Key Range Size: ~w$~n", + [RenderScenario(Scenario, ""), KeyRange]), + lists:foreach( + fun(ThreadCount) -> + PrintData("; ~w",[ThreadCount]) + end, + ThreadCounts), + PrintData("$~n",[]), + lists:foreach( + fun(TableType) -> + PrintData("~w ",[TableType]), + lists:foreach( + fun(ThreadCount) -> + RunBenchmarkAndReport(ThreadCount, + TableType, + Scenario, + KeyRange, + BenchmarkDurationMs) + end, + ThreadCounts), + PrintData("$~n",[]) + end, + TableTypes) + end, + Scenarios) + end, + KeyRanges), + PrintData("~n#BENCHMARK ENDED$~n~n",[]), + VerifyETSMemFun(EtsMem), + DataDir = filename:join(filename:dirname(code:which(?MODULE)), "ets_SUITE_data"), + TemplatePath = filename:join(DataDir, "visualize_throughput.html"), + {ok, Template} = file:read_file(TemplatePath), + OutputData = string:replace(Template, "#bench_data_placeholder", GetData()), + OutputPath1 = filename:join(DataDir, "ets_bench_result.html"), + {{Year, Month, Day}, {Hour, Minute, Second}} = calendar:now_to_datetime(erlang:timestamp()), + StrTime = lists:flatten(io_lib:format("~4..0w-~2..0w-~2..0wT~2..0w:~2..0w:~2..0w",[Year,Month,Day,Hour,Minute,Second])), + OutputPath2 = filename:join(DataDir, io_lib:format("ets_bench_result_~s.html", [StrTime])), + file:write_file(OutputPath1, OutputData), + file:write_file(OutputPath2, OutputData), + PrintResultPathsFun(OutputPath2, OutputPath1). + +test_throughput_benchmark(Config) when is_list(Config) -> + throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = 100, + recover_time_ms = 0, + thread_counts = [1, erlang:system_info(schedulers)], + key_ranges = [50000], + etsmem_fun = fun etsmem/0, + verify_etsmem_fun = fun verify_etsmem/1}). + +long_throughput_benchmark(Config) when is_list(Config) -> + N = erlang:system_info(schedulers), + throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = 3000, + recover_time_ms = 1000, + thread_counts = [1, N div 2, N], + key_ranges = [1000000], + scenarios = + [ + [ + {0.5, insert}, + {0.5, delete} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.8, lookup} + ], + [ + {0.01, insert}, + {0.01, delete}, + {0.98, lookup} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq100} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, partial_select1000} + ] + ], + table_types = + [ + [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], + [set, public, {write_concurrency, true}, {read_concurrency, true}] + ], + etsmem_fun = fun etsmem/0, + verify_etsmem_fun = fun verify_etsmem/1, + notify_res_fun = + fun(Name, Throughput) -> + SummaryTable = + proplists:get_value(ets_benchmark_result_summary_tab, Config), + AddToSummaryCounter = + case SummaryTable of + undefined -> + fun(_, _) -> + ok + end; + Tab -> + fun(CounterName, ToAdd) -> + OldVal = ets:lookup_element(Tab, CounterName, 2), + NewVal = OldVal + ToAdd, + ets:insert(Tab, {CounterName, NewVal}) + end + end, + Record = + fun(NoOfBenchsCtr, TotThrputCtr) -> + AddToSummaryCounter(NoOfBenchsCtr, 1), + AddToSummaryCounter(TotThrputCtr, Throughput) + end, + Record(nr_of_benchmarks, total_throughput), + case string:find(Name, "ordered_set") of + nomatch -> + Record(nr_of_set_benchmarks, total_throughput_set); + _ -> + Record(nr_of_ordered_set_benchmarks, + total_throughput_ordered_set) + end, + ct_event:notify( + #event{name = benchmark_data, + data = [{suite,"ets_bench"}, + {name, Name}, + {value,Throughput}]}) + end + }). add_lists(L1,L2) -> add_lists(L1,L2,[]). @@ -6197,8 +7275,11 @@ wait_pids(Pids, Acc) -> {Pid,Result} -> true = lists:member(Pid,Pids), Others = lists:delete(Pid,Pids), - io:format("wait_pid got ~p from ~p, still waiting for ~p\n",[Result,Pid,Others]), + %%io:format("wait_pid got ~p from ~p\n",[Result,Pid]), wait_pids(Others,[Result | Acc]) + after 60*1000 -> + io:format("Still waiting for workers ~p\n",[Pids]), + wait_pids(Pids, Acc) end. @@ -6222,48 +7303,25 @@ wait_for_memory_deallocations() -> wait_for_memory_deallocations() end. - etsmem() -> - wait_for_memory_deallocations(), - - AllTabs = lists:map(fun(T) -> {T,ets:info(T,name),ets:info(T,size), - ets:info(T,memory),ets:info(T,type)} - end, ets:all()), - - EtsAllocInfo = erlang:system_info({allocator,ets_alloc}), - ErlangMemoryEts = try erlang:memory(ets) catch error:notsup -> notsup end, - - Mem = - {ErlangMemoryEts, - case EtsAllocInfo of - false -> undefined; - MemInfo -> - CS = lists:foldl( - fun ({instance, _, L}, Acc) -> - {value,{mbcs,MBCS}} = lists:keysearch(mbcs, 1, L), - {value,{sbcs,SBCS}} = lists:keysearch(sbcs, 1, L), - NewAcc = [MBCS, SBCS | Acc], - case lists:keysearch(mbcs_pool, 1, L) of - {value,{mbcs_pool, MBCS_POOL}} -> - [MBCS_POOL|NewAcc]; - _ -> NewAcc - end - end, - [], - MemInfo), - lists:foldl( - fun(L, {Bl0,BlSz0}) -> - {value,BlTup} = lists:keysearch(blocks, 1, L), - blocks = element(1, BlTup), - Bl = element(2, BlTup), - {value,BlSzTup} = lists:keysearch(blocks_size, 1, L), - blocks_size = element(1, BlSzTup), - BlSz = element(2, BlSzTup), - {Bl0+Bl,BlSz0+BlSz} - end, {0,0}, CS) - end}, - {Mem,AllTabs}. + % The following is done twice to avoid an inconsistent memory + % "snapshot" (see verify_etsmem/2). + lists:foldl( + fun(_,_) -> + wait_for_memory_deallocations(), + + AllTabs = lists:map(fun(T) -> {T,ets:info(T,name),ets:info(T,size), + ets:info(T,memory),ets:info(T,type)} + end, ets:all()), + EtsAllocSize = erts_debug:alloc_blocks_size(ets_alloc), + ErlangMemoryEts = try erlang:memory(ets) catch error:notsup -> notsup end, + + Mem = {ErlangMemoryEts, EtsAllocSize}, + {Mem, AllTabs} + end, + not_used, + lists:seq(1,2)). verify_etsmem(MI) -> wait_for_test_procs(), @@ -6284,15 +7342,15 @@ verify_etsmem({MemInfo,AllTabs}, Try) -> end; {MemInfo2, AllTabs2} -> - io:format("Expected: ~p", [MemInfo]), - io:format("Actual: ~p", [MemInfo2]), - io:format("Changed tables before: ~p\n",[AllTabs -- AllTabs2]), - io:format("Changed tables after: ~p\n", [AllTabs2 -- AllTabs]), + io:format("#Expected: ~p", [MemInfo]), + io:format("#Actual: ~p", [MemInfo2]), + io:format("#Changed tables before: ~p\n",[AllTabs -- AllTabs2]), + io:format("#Changed tables after: ~p\n", [AllTabs2 -- AllTabs]), case Try < 2 of true -> - io:format("\nThis discrepancy could be caused by an " + io:format("\n#This discrepancy could be caused by an " "inconsistent memory \"snapshot\"" - "\nTry again...\n", []), + "\n#Try again...\n", []), verify_etsmem({MemInfo, AllTabs}, Try+1); false -> ct:fail("Failed memory check") @@ -6766,22 +7824,49 @@ make_unaligned_sub_binary(List) -> repeat_for_opts(F) -> repeat_for_opts(F, [write_concurrency, read_concurrency, compressed]). +repeat_for_opts_all_table_types(F) -> + repeat_for_opts(F, [all_types, write_concurrency, read_concurrency, compressed]). + +repeat_for_opts_all_non_stim_table_types(F) -> + repeat_for_opts(F, [all_non_stim_types, write_concurrency, read_concurrency, compressed]). + +repeat_for_opts_all_set_table_types(F) -> + repeat_for_opts(F, [set_types, write_concurrency, read_concurrency, compressed]). + +repeat_for_all_set_table_types(F) -> + repeat_for_opts(F, [set_types]). + +repeat_for_all_ord_set_table_types(F) -> + repeat_for_opts(F, [ord_set_types]). + +repeat_for_all_non_stim_set_table_types(F) -> + repeat_for_opts(F, [all_non_stim_set_types]). + +repeat_for_opts_all_ord_set_table_types(F) -> + repeat_for_opts(F, [ord_set_types, write_concurrency, read_concurrency, compressed]). + repeat_for_opts(F, OptGenList) when is_function(F, 1) -> repeat_for_opts(F, OptGenList, []). repeat_for_opts(F, [], Acc) -> lists:foldl(fun(Opts, RV_Acc) -> OptList = lists:filter(fun(E) -> E =/= void end, Opts), - io:format("Calling with options ~p\n",[OptList]), - RV = F(OptList), - case RV_Acc of - {comment,_} -> RV_Acc; - _ -> case RV of - {comment,_} -> RV; - _ -> [RV | RV_Acc] - end - end - end, [], Acc); + case is_redundant_opts_combo(OptList) of + true -> + %%io:format("Ignoring redundant options ~p\n",[OptList]), + ok; + false -> + io:format("Calling with options ~p\n",[OptList]), + RV = F(OptList), + case RV_Acc of + {comment,_} -> RV_Acc; + _ -> case RV of + {comment,_} -> RV; + _ -> [RV | RV_Acc] + end + end + end + end, [], Acc); repeat_for_opts(F, [OptList | Tail], []) when is_list(OptList) -> repeat_for_opts(F, Tail, [[Opt] || Opt <- OptList]); repeat_for_opts(F, [OptList | Tail], AccList) when is_list(OptList) -> @@ -6789,14 +7874,127 @@ repeat_for_opts(F, [OptList | Tail], AccList) when is_list(OptList) -> repeat_for_opts(F, [Atom | Tail], AccList) when is_atom(Atom) -> repeat_for_opts(F, [repeat_for_opts_atom2list(Atom) | Tail ], AccList). -repeat_for_opts_atom2list(all_types) -> [set,ordered_set,bag,duplicate_bag]; +repeat_for_opts_atom2list(set_types) -> [set,ordered_set,stim_cat_ord_set,cat_ord_set]; +repeat_for_opts_atom2list(ord_set_types) -> [ordered_set,stim_cat_ord_set,cat_ord_set]; +repeat_for_opts_atom2list(all_types) -> [set,ordered_set,stim_cat_ord_set,cat_ord_set,bag,duplicate_bag]; +repeat_for_opts_atom2list(all_non_stim_types) -> [set,ordered_set,cat_ord_set,bag,duplicate_bag]; +repeat_for_opts_atom2list(all_non_stim_set_types) -> [set,ordered_set,cat_ord_set]; repeat_for_opts_atom2list(write_concurrency) -> [{write_concurrency,false},{write_concurrency,true}]; repeat_for_opts_atom2list(read_concurrency) -> [{read_concurrency,false},{read_concurrency,true}]; repeat_for_opts_atom2list(compressed) -> [compressed,void]. -ets_new(Name, Opts) -> - %%ets:new(Name, [compressed | Opts]). - ets:new(Name, Opts). +is_redundant_opts_combo(Opts) -> + (lists:member(stim_cat_ord_set, Opts) orelse + lists:member(cat_ord_set, Opts)) + andalso + (lists:member({write_concurrency, false}, Opts) orelse + lists:member(private, Opts) orelse + lists:member(protected, Opts)). + +%% Add fake table option with info about key range. +%% Will be consumed by ets_new and used for stim_cat_ord_set. +key_range(Opts, KeyRange) -> + [{key_range, KeyRange} | Opts]. + +ets_new(Name, Opts0) -> + {KeyRange, Opts1} = case lists:keytake(key_range, 1, Opts0) of + {value, {key_range, KR}, Rest1} -> + {KR, Rest1}; + false -> + {1000*1000, Opts0} + end, + ets_new(Name, Opts1, KeyRange). + +ets_new(Name, Opts, KeyRange) -> + ets_new(Name, Opts, KeyRange, fun id/1). + +ets_new(Name, Opts0, KeyRange, KeyFun) -> + {CATree, Stimulate, RevOpts} = + lists:foldl(fun(cat_ord_set, {false, false, Lacc}) -> + {true, false, [ordered_set | Lacc]}; + (stim_cat_ord_set, {false, false, Lacc}) -> + {true, true, [ordered_set | Lacc]}; + (Other, {CAT, STIM, Lacc}) -> + {CAT, STIM, [Other | Lacc]} + end, + {false, false, []}, + Opts0), + Opts = lists:reverse(RevOpts), + EtsNewHelper = + fun (UseOpts) -> + case get(ets_new_opts) of + UseOpts -> + silence; %% suppress identical table opts spam + _ -> + put(ets_new_opts, UseOpts), + io:format("ets:new(~p, ~p)~n", [Name, UseOpts]) + end, + ets:new(Name, UseOpts) + end, + case CATree andalso + (not lists:member({write_concurrency, false}, Opts)) andalso + (not lists:member(private, Opts)) andalso + (not lists:member(protected, Opts)) of + true -> + NewOpts1 = + case lists:member({write_concurrency, true}, Opts) of + true -> Opts; + false -> [{write_concurrency, true}|Opts] + end, + NewOpts2 = + case lists:member(public, NewOpts1) of + true -> NewOpts1; + false -> [public|NewOpts1] + end, + T = EtsNewHelper(NewOpts2), + case Stimulate of + false -> ok; + true -> stimulate_contention(T, KeyRange, KeyFun) + end, + T; + false -> + EtsNewHelper(Opts) + end. + +% The purpose of this function is to stimulate fine grained locking in +% tables of types ordered_set with the write_concurrency options +% turned on. The erts_debug feature 'ets_force_split' is used to easier +% generate a routing tree with fine grained locking without having to +% provoke lots of actual lock contentions. +stimulate_contention(Tid, KeyRange, KeyFun) -> + T = case Tid of + A when is_atom(A) -> ets:whereis(A); + _ -> Tid + end, + erts_debug:set_internal_state(ets_force_split, {T, true}), + Num = case KeyRange > 50 of + true -> 50; + false -> KeyRange + end, + Seed = rand:uniform(KeyRange), + %%io:format("prefill_table: Seed = ~p\n", [Seed]), + RState = unique_rand_start(KeyRange, Seed), + stim_inserter_loop(T, RState, Num, KeyFun), + Num = ets:info(T, size), + ets:match_delete(T, {'$1','$1','$1'}), + 0 = ets:info(T, size), + erts_debug:set_internal_state(ets_force_split, {T, false}), + case ets:info(T,stats) of + {0, _, _} -> + io:format("No routing nodes in table?\n" + "Debug feature 'ets_force_split' does not seem to work.\n", []), + ct:fail("No ets_force_split?"); + Stats -> + io:format("stimulated ordered_set: ~p\n", [Stats]) + end. + +stim_inserter_loop(_, _, 0, _) -> + ok; +stim_inserter_loop(T, RS0, N, KeyFun) -> + {K, RS1} = unique_rand_next(RS0), + Key = KeyFun(K), + ets:insert(T, {Key, Key, Key}), + stim_inserter_loop(T, RS1, N-1, KeyFun). do_tc(Do, Report) -> T1 = erlang:monotonic_time(), @@ -6810,3 +8008,50 @@ syrup_factor() -> valgrind -> 20; _ -> 1 end. + + +%% +%% This is a pseudo random number generator for UNIQUE integers. +%% All integers between 1 and Max will be generated before it repeat itself. +%% It's a variant of this one using quadratic residues by Jeff Preshing: +%% http://preshing.com/20121224/how-to-generate-a-sequence-of-unique-random-integers/ +%% +unique_rand_start(Max, Seed) -> + L = lists:dropwhile(fun(P) -> P < Max end, + primes_3mod4()), + [P | _] = case L of + [] -> + error("Random range too large"); + _ -> + L + end, + 3 = P rem 4, + {0, {Max, P, Seed}}. + +unique_rand_next({N, {Max, P, Seed}=Const}) -> + case dquad(P, N, Seed) + 1 of + RND when RND > Max -> % Too large, skip + unique_rand_next({N+1, Const}); + RND -> + {RND, {N+1, Const}} + end. + +%% A one-to-one relation between all integers 0 =< X < Prime +%% if Prime rem 4 == 3. +quad(Prime, X) -> + Rem = X*X rem Prime, + case 2*X < Prime of + true -> + Rem; + false -> + Prime - Rem + end. + +dquad(Prime, X, Seed) -> + quad(Prime, (quad(Prime, X) + Seed) rem Prime). + +%% Primes where P rem 4 == 3. +primes_3mod4() -> + [103, 211, 503, 1019, 2003, 5003, 10007, 20011, 50023, + 100003, 200003, 500083, 1000003, 2000003, 5000011, + 10000019, 20000003, 50000047, 100000007]. diff --git a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html new file mode 100644 index 0000000000..27d6849c60 --- /dev/null +++ b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html @@ -0,0 +1,253 @@ +<!doctype html> +<html lang="en"> + +<!-- %% --> +<!-- %% %CopyrightBegin% --> +<!-- %% --> +<!-- %% Copyright Ericsson AB and Kjell Winblad 1996-2018. All Rights Reserved. --> +<!-- %% --> +<!-- %% Licensed under the Apache License, Version 2.0 (the "License"); --> +<!-- %% you may not use this file except in compliance with the License. --> +<!-- %% You may obtain a copy of the License at --> +<!-- %% --> +<!-- %% http://www.apache.org/licenses/LICENSE-2.0 --> +<!-- %% --> +<!-- %% Unless required by applicable law or agreed to in writing, software --> +<!-- %% distributed under the License is distributed on an "AS IS" BASIS, --> +<!-- %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --> +<!-- %% See the License for the specific language governing permissions and --> +<!-- %% limitations under the License. --> +<!-- %% --> +<!-- %% %CopyrightEnd% --> +<!-- %% --> +<!-- %% Author: Kjell Winblad --> +<!-- %% --> + + <head> + <meta charset="utf-8"> + <title>ETS Benchmark Result Viewer</title> + </head> + + <body> + <div id="insertPlaceholder"></div> + <h1>ETS Benchmark Result Viewer</h1> + <p> + This page generates graphs from data produced by the ETS benchmark which is defined in the function <code>ets_SUITE:throughput_benchmark/0</code> (see "<code>$ERL_TOP/lib/stdlib/test/ets_SUITE.erl</code>"). + </p> + <p> + Note that one can paste results from several benchmark runs into the field below. Results from the same scenario but from different benchmark runs will be relabeled and ploted in the same graph automatically. This makes comparisons of different ETS versions easy. + </p> + <p> + Note also that that lines can be hidden by clicking on the corresponding label. + </p> + Paste the generated data in the field below and press the Render button: + <br> + <textarea id="dataField" rows="4" cols="50">#bench_data_placeholder</textarea> + <br> + <input type="checkbox" id="barPlot"> Bar Plot + <br> + <input type="checkbox" id="sameSpacing" checked> Same X Spacing Between Points + <br> + <input type="checkbox" class="showCheck" value="[ordered_set,public]" checked> Show <code>[ordered_set,public]</code> + <br> + <input type="checkbox" class="showCheck" value="[ordered_set,public,{write_concurrency,true}]" checked> Show <code>[ordered_set,public,{write_concurrency,true}]</code> + <br> + <input type="checkbox" class="showCheck" value="[ordered_set,public,{read_concurrency,true}]" checked> Show <code>[ordered_set,public,{read_concurrency,true}]</code> + <br> + <input type="checkbox" class="showCheck" value="[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]</code> + <br> + <input type="checkbox" class="showCheck" value="[set,public]" checked> Show <code>[set,public]</code> + <br> + <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true}]</code> + <br> + <input type="checkbox" class="showCheck" value="[set,public,{read_concurrency,true}]" checked> Show <code>[set,public,{read_concurrency,true}]</code> + <br> + <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true},{read_concurrency,true}]</code> + <br> + <button id="renderButton" type="button">Render</button> + + <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" + integrity="sha256-3edrmyuQ0w65f8gfBsqowzjJe2iM6n0nKciPUp8y+7E=" + crossorigin="anonymous"></script> + <script> + var loading = false; + function toggleLoadingScreen(){ + if(loading){ + $("#loading").remove(); + loading = false; + }else{ + $('<div id="loading">'+ + '<span style="position: fixed; top: 50%;left: 50%;color: white;"><b>Loading...</b></span>'+ + '</div>') + .css({position: "fixed", + top: 0, + left: 0, + width: "100%", + height: "100%", + 'background-color': "#000", + filter:"alpha(opacity=50)", + '-moz-opacity':"0.5", + '-khtml-opacity': "0.5", + opacity: "0.5", + 'z-index': "10000"}) + .appendTo(document.body); + loading = true; + + } + } + //Start loading screen before downloading plotly which is quite large + toggleLoadingScreen(); + </script> + <script src="https://cdn.plot.ly/plotly-1.5.0.min.js"></script> + <script> + String.prototype.replaceAll = function(search, replacement) { + var target = this; + return target.split(search).join(replacement); + }; + String.prototype.myTrim = function() { + var target = this; + return target.replace(/^\s+|\s+$/g, ''); + }; + function plotGraph(lines, sameSpacing, barPlot, prefix) { + var xvals = null; + var data = []; + while(lines.length > 0 && + (lines[0].myTrim() == "" || + lines[0].myTrim().indexOf(";") !== -1)){ + var line = lines.shift().myTrim(); + if(line == "" || line.startsWith("#")){ + continue; + } else if(line.startsWith(";")) { + xvals = line.split(";") + xvals.shift(); // Remove first + xvals = $.map(xvals, function (i){ + if(sameSpacing){ + return "_"+i.myTrim(); + }else{ + return parseInt(i.myTrim(), 10); + } + }); + }else{ + line = line.split(";") + var label = prefix + line.shift().myTrim(); + var yvals = $.map(line, function (i){ + return parseFloat(i.myTrim(), 10); + }); + var trace = { + x: xvals, + y: yvals, + mode: 'lines+markers', + name: label + }; + if(barPlot){ + trace['type'] = "bar"; + } + data.push(trace); + } + + } + return data; + } + function plotGraphs(){ + var insertPlaceholder = $("#insertPlaceholder"); + var sameSpacing = $('#sameSpacing').is(":checked"); + var barPlot = $('#barPlot').is(":checked"); + var lines = $("#dataField").val(); + $('.showCheck').each(function() { + var item = $(this); + if(!item.is(":checked")){ + lines = lines.replaceAll(item.val(), "#"+item.val()) + } + }); + lines = lines.split("$"); + var nrOfGraphs = 0; + var scenarioDataMap = {}; + var scenarioNrOfVersionsMap = {}; + var scenarioList = []; + while(lines.length > 0){ + var line = lines.shift().myTrim(); + if(line == ""){ + continue; + } else if(line.startsWith("Scenario:")) { + nrOfGraphs = nrOfGraphs + 1; + var name = line; + if(scenarioDataMap[name] === undefined){ + scenarioDataMap[name] = []; + scenarioNrOfVersionsMap[name] = 0; + scenarioList.push(line); + } + scenarioNrOfVersionsMap[name] = scenarioNrOfVersionsMap[name] + 1; + var prefix = undefined; + if(scenarioNrOfVersionsMap[name] === 1){ + prefix = ""; + }else{ + prefix = "Ver: " + scenarioNrOfVersionsMap[name] + " "; + } + scenarioDataMap[name] = + scenarioDataMap[name].concat( + plotGraph(lines, sameSpacing, barPlot, prefix)); + } + } + $.each(scenarioList, + function( index, name ) { + var nrOfGraphs = index + 1; + var data = scenarioDataMap[name]; + $( "<div class='added' id='graph"+nrOfGraphs+"'>") + .insertBefore( insertPlaceholder ); + $( "<button type='button' class='added' id='fullscreenButton"+nrOfGraphs+"'>Fill screen</button>") + .insertBefore( insertPlaceholder ); + $( "<span class='added'><br><hr><br></span>") + .insertBefore( insertPlaceholder ); + var layout = { + title:name, + xaxis: { + title: '# of Processes' + }, + yaxis: { + title: 'Operations/Second' + } + + }; + + $("#fullscreenButton"+nrOfGraphs).click( + function(){ + $('#graph'+nrOfGraphs).replaceWith( + $("<div class='added' id='graph"+nrOfGraphs+"'>")); + layout = $.extend({}, layout, { + width:$(window).width()-40, + height:$(window).height()-40 + }); + Plotly.newPlot('graph'+nrOfGraphs, data, layout); + }); + Plotly.newPlot('graph'+nrOfGraphs, data, layout); + + }); + + + } + $(document).ready(function(){ + $('#renderButton').click( + function(){ + toggleLoadingScreen(); + setTimeout(function(){ + try { + $( ".added" ).remove(); + plotGraphs(); + toggleLoadingScreen(); + } catch(e){ + toggleLoadingScreen(); + console.log(e); + alert("Error happened when parsing data.\n" + + "See console for more info"); + } + }, 10); + }); + setTimeout(function(){ + $( ".added" ).remove(); + plotGraphs(); + toggleLoadingScreen(); + }, 10); + }); + </script> + </body> +</html> diff --git a/lib/stdlib/test/gen_fsm_SUITE.erl b/lib/stdlib/test/gen_fsm_SUITE.erl index a8264e5a84..62d9d0e0ae 100644 --- a/lib/stdlib/test/gen_fsm_SUITE.erl +++ b/lib/stdlib/test/gen_fsm_SUITE.erl @@ -521,14 +521,16 @@ error_format_status(Config) when is_list(Config) -> error_logger_forwarder:register(), OldFl = process_flag(trap_exit, true), StateData = "called format_status", + Parent = self(), {ok, Pid} = gen_fsm:start(gen_fsm_SUITE, {state_data, StateData}, []), %% bad return value in the gen_fsm loop {'EXIT',{{bad_return_value, badreturn},_}} = (catch gen_fsm:sync_send_event(Pid, badreturn)), receive {error,_GroupLeader,{Pid, - "** State machine"++_, - [Pid,{_,_,badreturn},idle,{formatted,StateData},_]}} -> + "** State machine "++_, + [Pid,badreturn,Parent,idle,{formatted,StateData}, + {bad_return_value,badreturn}|_]}} -> ok; Other -> io:format("Unexpected: ~p", [Other]), @@ -541,12 +543,14 @@ terminate_crash_format(Config) when is_list(Config) -> error_logger_forwarder:register(), OldFl = process_flag(trap_exit, true), StateData = crash_terminate, + Parent = self(), {ok, Pid} = gen_fsm:start(gen_fsm_SUITE, {state_data, StateData}, []), stop_it(Pid), receive {error,_GroupLeader,{Pid, - "** State machine"++_, - [Pid,{_,_,_},idle,{formatted, StateData},_]}} -> + "** State machine "++_, + [Pid,stop,Parent,idle,{formatted, StateData}, + {crash,terminate}|_]}} -> ok; Other -> io:format("Unexpected: ~p", [Other]), diff --git a/lib/stdlib/test/gen_statem_SUITE.erl b/lib/stdlib/test/gen_statem_SUITE.erl index 053233df9b..16cf8f43f9 100644 --- a/lib/stdlib/test/gen_statem_SUITE.erl +++ b/lib/stdlib/test/gen_statem_SUITE.erl @@ -121,7 +121,8 @@ end_per_testcase(_CaseName, Config) -> start1(Config) -> %%OldFl = process_flag(trap_exit, true), - {ok,Pid0} = gen_statem:start_link(?MODULE, start_arg(Config, []), []), + {ok,Pid0} = + gen_statem:start_link(?MODULE, start_arg(Config, []), [{debug,[trace]}]), ok = do_func_test(Pid0), ok = do_sync_func_test(Pid0), stop_it(Pid0), @@ -135,7 +136,8 @@ start1(Config) -> %% anonymous w. shutdown start2(Config) -> %% Dont link when shutdown - {ok,Pid0} = gen_statem:start(?MODULE, start_arg(Config, []), []), + {ok,Pid0} = + gen_statem:start(?MODULE, start_arg(Config, []), []), ok = do_func_test(Pid0), ok = do_sync_func_test(Pid0), stopped = gen_statem:call(Pid0, {stop,shutdown}), @@ -511,7 +513,9 @@ abnormal1dirty(Config) -> %% trap exit since we must link to get the real bad_return_ error abnormal2(Config) -> OldFl = process_flag(trap_exit, true), - {ok,Pid} = gen_statem:start_link(?MODULE, start_arg(Config, []), []), + {ok,Pid} = + gen_statem:start_link( + ?MODULE, start_arg(Config, []), [{debug,[log]}]), %% bad return value in the gen_statem loop {{{bad_return_from_state_function,badreturn},_},_} = @@ -529,7 +533,9 @@ abnormal2(Config) -> %% trap exit since we must link to get the real bad_return_ error abnormal3(Config) -> OldFl = process_flag(trap_exit, true), - {ok,Pid} = gen_statem:start_link(?MODULE, start_arg(Config, []), []), + {ok,Pid} = + gen_statem:start_link( + ?MODULE, start_arg(Config, []), [{debug,[log]}]), %% bad return value in the gen_statem loop {{{bad_action_from_state_function,badaction},_},_} = @@ -547,7 +553,9 @@ abnormal3(Config) -> %% trap exit since we must link to get the real bad_return_ error abnormal4(Config) -> OldFl = process_flag(trap_exit, true), - {ok,Pid} = gen_statem:start_link(?MODULE, start_arg(Config, []), []), + {ok,Pid} = + gen_statem:start_link( + ?MODULE, start_arg(Config, []), [{debug,[log]}]), %% bad return value in the gen_statem loop BadTimeout = {badtimeout,4711,ouch}, @@ -641,51 +649,80 @@ state_enter(_Config) -> end, start => fun (enter, Prev, N) -> - Self ! {enter,start,Prev,N}, + Self ! {N,enter,start,Prev}, {keep_state,N + 1}; (internal, Prev, N) -> - Self ! {internal,start,Prev,N}, + Self ! {N,internal,start,Prev}, {keep_state,N + 1}; + (timeout, M, N) -> + {keep_state, N + 1, + {reply, {Self,N}, {timeout,M}}}; ({call,From}, repeat, N) -> {repeat_state,N + 1, - [{reply,From,{repeat,start,N}}]}; + [{reply,From,{N,repeat,start}}]}; ({call,From}, echo, N) -> {next_state,wait,N + 1, - {reply,From,{echo,start,N}}}; + [{reply,From,{N,echo,start}},{timeout,0,N}]}; ({call,From}, {stop,Reason}, N) -> {stop_and_reply,Reason, - [{reply,From,{stop,N}}],N + 1} + [{reply,From,{N,stop}}],N + 1} end, wait => fun (enter, Prev, N) when N < 5 -> {repeat_state,N + 1, - {reply,{Self,N},{enter,Prev}}}; + [{reply,{Self,N},{enter,Prev}}, + {timeout,0,N}, + {state_timeout,0,N}]}; (enter, Prev, N) -> - Self ! {enter,wait,Prev,N}, - {keep_state,N + 1}; + Self ! {N,enter,wait,Prev}, + {keep_state,N + 1, + [{timeout,0,N}, + {state_timeout,0,N}]}; + (timeout, M, N) -> + {keep_state, N + 1, + {reply, {Self,N}, {timeout,M}}}; + (state_timeout, M, N) -> + {keep_state, N + 1, + {reply, {Self,N}, {state_timeout,M}}}; ({call,From}, repeat, N) -> {repeat_state_and_data, - [{reply,From,{repeat,wait,N}}]}; + [{reply,From,{N,repeat,wait}}, + {timeout,0,N}]}; ({call,From}, echo, N) -> {next_state,start,N + 1, [{next_event,internal,wait}, - {reply,From,{echo,wait,N}}]} + {reply,From,{N,echo,wait}}]} end}, {ok,STM} = gen_statem:start_link( - ?MODULE, {map_statem,Machine,[state_enter]}, []), - - [{enter,start,start,1}] = flush(), - {echo,start,2} = gen_statem:call(STM, echo), - [{3,{enter,start}},{4,{enter,start}},{enter,wait,start,5}] = flush(), - {wait,[6|_]} = sys:get_state(STM), - {repeat,wait,6} = gen_statem:call(STM, repeat), - [{enter,wait,wait,6}] = flush(), - {echo,wait,7} = gen_statem:call(STM, echo), - [{enter,start,wait,8},{internal,start,wait,9}] = flush(), - {repeat,start,10} = gen_statem:call(STM, repeat), - [{enter,start,start,11}] = flush(), - {stop,12} = gen_statem:call(STM, {stop,bye}), + ?MODULE, {map_statem,Machine,[state_enter]}, + [{debug,[trace,{log,17}]}]), + ok = sys:log(STM, false), + ok = sys:log(STM, true), + + [{1,enter,start,start}] = flush(), + {2,echo,start} = gen_statem:call(STM, echo), + [{3,{enter,start}}, + {4,{enter,start}}, + {5,enter,wait,start}, + {6,{timeout,5}}, + {7,{state_timeout,5}}] = flush(), + {wait,[8|_]} = sys:get_state(STM), + {8,repeat,wait} = gen_statem:call(STM, repeat), + [{8,enter,wait,wait}, + {9,{timeout,8}}, + {10,{state_timeout,8}}] = flush(), + {11,echo,wait} = gen_statem:call(STM, echo), + [{12,enter,start,wait}, + {13,internal,start,wait}] = flush(), + {14,repeat,start} = gen_statem:call(STM, repeat), + [{15,enter,start,start}] = flush(), + + {ok,Log} = sys:log(STM, get), + io:format("sys:log ~p~n", [Log]), + ok = sys:log(STM, print), + + {16,stop} = gen_statem:call(STM, {stop,bye}), [{'EXIT',STM,bye}] = flush(), {noproc,_} = @@ -1447,7 +1484,8 @@ enter_loop(_Config) -> %% Locally registered process + {local,Name} {ok,Pid1a} = - proc_lib:start_link(?MODULE, enter_loop, [local,local]), + proc_lib:start_link( + ?MODULE, enter_loop, [local,local,[{debug,[{log,7}]}]]), yes = gen_statem:call(Pid1a, 'alive?'), stopped = gen_statem:call(Pid1a, stop), receive @@ -1459,7 +1497,8 @@ enter_loop(_Config) -> %% Unregistered process + {local,Name} {ok,Pid1b} = - proc_lib:start_link(?MODULE, enter_loop, [anon,local]), + proc_lib:start_link( + ?MODULE, enter_loop, [anon,local,[{debug,[log]}]]), receive {'EXIT',Pid1b,process_not_registered} -> ok @@ -1568,6 +1607,9 @@ enter_loop(_Config) -> ok = verify_empty_msgq(). enter_loop(Reg1, Reg2) -> + enter_loop(Reg1, Reg2, []). +%% +enter_loop(Reg1, Reg2, Opts) -> process_flag(trap_exit, true), case Reg1 of local -> register(armitage, self()); @@ -1579,15 +1621,15 @@ enter_loop(Reg1, Reg2) -> case Reg2 of local -> gen_statem:enter_loop( - ?MODULE, [], state0, [], {local,armitage}); + ?MODULE, Opts, state0, [], {local,armitage}); global -> gen_statem:enter_loop( - ?MODULE, [], state0, [], {global,armitage}); + ?MODULE, Opts, state0, [], {global,armitage}); via -> gen_statem:enter_loop( - ?MODULE, [], state0, [], {via, dummy_via, armitage}); + ?MODULE, Opts, state0, [], {via, dummy_via, armitage}); anon -> - gen_statem:enter_loop(?MODULE, [], state0, []) + gen_statem:enter_loop(?MODULE, Opts, state0, []) end. undef_code_change(_Config) -> @@ -1619,7 +1661,9 @@ undef_terminate2(_Config) -> undef_in_terminate(_Config) -> Data = {undef_in_terminate, {?MODULE, terminate}}, - {ok, Statem} = gen_statem:start(?MODULE, {data, Data}, []), + {ok, Statem} = + gen_statem:start( + ?MODULE, {data, Data}, [{debug,[log]}]), try gen_statem:stop(Statem), ct:fail(should_crash) diff --git a/lib/stdlib/test/lists_SUITE.erl b/lib/stdlib/test/lists_SUITE.erl index 984b51e7ae..5dab6f6697 100644 --- a/lib/stdlib/test/lists_SUITE.erl +++ b/lib/stdlib/test/lists_SUITE.erl @@ -158,6 +158,20 @@ append_2(Config) when is_list(Config) -> "abcdef"=lists:append("abc", "def"), [hej, du]=lists:append([hej], [du]), [10, [elem]]=lists:append([10], [[elem]]), + + %% Trapping, both crashing and otherwise. + [append_trapping_1(N) || N <- lists:seq(0, 20)], + + ok. + +append_trapping_1(N) -> + List = lists:duplicate(N + (1 bsl N), gurka), + ImproperList = List ++ crash, + + {'EXIT',_} = (catch (ImproperList ++ [])), + + [3, 2, 1 | List] = lists:reverse(List ++ [1, 2, 3]), + ok. %% Tests the lists:reverse() implementation. The function is @@ -1679,7 +1693,7 @@ make_fun() -> receive {Pid, Fun} -> Fun end. make_fun(Pid) -> - Pid ! {self(), fun make_fun/1}. + Pid ! {self(), fun (X) -> {X, Pid} end}. fun_pid(Fun) -> erlang:fun_info(Fun, pid). diff --git a/lib/stdlib/test/qlc_SUITE.erl b/lib/stdlib/test/qlc_SUITE.erl index d7354438f9..8a43f15d2c 100644 --- a/lib/stdlib/test/qlc_SUITE.erl +++ b/lib/stdlib/test/qlc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2018. All Rights Reserved. +%% Copyright Ericsson AB 2004-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -2436,7 +2436,7 @@ info(Config) when is_list(Config) -> <<"{'EXIT', {badarg, _}} = (catch qlc:info([X || {X} <- []], {n_elements, 0})), L = lists:seq(1, 1000), - \"[1,2,3,4,5,6,7,8,9,10|'...']\" = qlc:info(L, {n_elements, 10}), + \"[1, 2, 3, 4, 5, 6, 7, 8, 9, 10 | '...']\" = qlc:info(L, {n_elements, 10}), {cons,A1,{integer,A2,1},{atom,A3,'...'}} = qlc:info(L, [{n_elements, 1},{format,abstract_code}]), 1 = erl_anno:line(A1), @@ -2447,8 +2447,8 @@ info(Config) when is_list(Config) -> {atom,_,'...'}}}}, {call,_,_,_}]} = qlc:info(Q, [{n_elements, 3},{format,abstract_code}]), - \"ets:match_spec_run([a,b,c,d,e,f],\n\" - \" ets:match_spec_compile([{'$1',[true],\" + \"ets:match_spec_run([a, b, c, d, e, f],\n\" + \" ets:match_spec_compile([{'$1', [true], \" \"[{{'$1'}}]}]))\" = qlc:info(Q, [{n_elements, infinity}])">>, @@ -3163,19 +3163,13 @@ lookup2(Config) when is_list(Config) -> [a] = lookup_keys(Q) end, [{a},{b},{c}])">>, - {cres, - <<"etsc(fun(E) -> + <<"etsc(fun(E) -> Q = qlc:q([X || {X}=Y <- ets:table(E), element(2, Y) == b, X =:= 1]), [] = qlc:e(Q), false = lookup_keys(Q) - end, [{1,b},{2,3}])">>, - %% {warnings,[{2,sys_core_fold,nomatch_guard}, - %% {3,qlc,nomatch_filter}, - %% {3,sys_core_fold,{eval_failure,badarg}}]}}, - {warnings,[{2,sys_core_fold,nomatch_guard}, - {3,sys_core_fold,{eval_failure,badarg}}]}}, + end, [{1,b},{2,3}])">>, <<"etsc(fun(E) -> Q = qlc:q([X || {X} <- ets:table(E), element(1,{X}) =:= 1]), @@ -6553,7 +6547,7 @@ otp_7114(Config) when is_list(Config) -> otp_7232(Config) when is_list(Config) -> Ts = [<<"L = [fun math:sqrt/1, list_to_pid(\"<0.4.1>\"), erlang:make_ref()], - \"[fun math:sqrt/1,<0.4.1>,#Ref<\" ++ _ = qlc:info(L), + \"[fun math:sqrt/1, <0.4.1>, #Ref<\" ++ _ = qlc:info(L), {call,_, {remote,_,{atom,_,qlc},{atom,_,sort}}, [{cons,_, @@ -6569,7 +6563,7 @@ otp_7232(Config) when is_list(Config) -> \"qlc:sort([55296,56296],[{order,fun'-function/0-fun-2-'/2}])\" = format_info(Q, true), AC = qlc:info(Q, {format, abstract_code}), - \"qlc:sort([55296,56296], [{order,fun '-function/0-fun-2-'/2}])\" = + \"qlc:sort([55296, 56296], [{order, fun '-function/0-fun-2-'/2}])\" = binary_to_list(iolist_to_binary(erl_pp:expr(AC)))">>, %% OTP-7234. erl_parse:abstract() handles bit strings @@ -7094,21 +7088,21 @@ manpage(Config) when is_list(Config) -> \" V1 =\n\" \" qlc:q([ \n\" \" SQV ||\n\" - \" SQV <- [x,y]\n\" + \" SQV <- [x, y]\n\" \" ],\n\" - \" [{unique,true}]),\n\" + \" [{unique, true}]),\n\" \" V2 =\n\" \" qlc:q([ \n\" \" SQV ||\n\" - \" SQV <- [a,b]\n\" + \" SQV <- [a, b]\n\" \" ],\n\" - \" [{unique,true}]),\n\" + \" [{unique, true}]),\n\" \" qlc:q([ \n\" - \" {X,Y} ||\n\" + \" {X, Y} ||\n\" \" X <- V1,\n\" \" Y <- V2\n\" \" ],\n\" - \" [{unique,true}])\n\" + \" [{unique, true}])\n\" \"end\", true = B =:= qlc:info(QH, unique_all)">>, @@ -7124,19 +7118,19 @@ manpage(Config) when is_list(Config) -> \" V1 =\n\" \" qlc:q([ \n\" \" P0 ||\n\" - \" P0 = {W,Y} <- ets:table(_)\n\" + \" P0 = {W, Y} <- ets:table(_)\n\" \" ]),\n\" \" V2 =\n\" \" qlc:q([ \n\" - \" [G1|G2] ||\n\" + \" [G1 | G2] ||\n\" \" G2 <- V1,\n\" \" G1 <- ets:table(_),\n\" \" element(2, G1) =:= element(1, G2)\n\" \" ],\n\" - \" [{join,lookup}]),\n\" + \" [{join, lookup}]),\n\" \" qlc:q([ \n\" - \" {X,Z,W} ||\n\" - \" [{X,Z}|{W,Y}] <- V2\n\" + \" {X, Z, W} ||\n\" + \" [{X, Z} | {W, Y}] <- V2\n\" \" ])\n\" \"end\", Info1 = @@ -7161,25 +7155,28 @@ manpage(Config) when is_list(Config) -> \" V1 =\n\" \" qlc:q([ \n\" \" P0 ||\n\" - \" P0 = {X,Z} <- qlc:keysort(1, [{a,1},{b,4},{c,6}], [])\n\" + \" P0 = {X, Z} <-\n\" + \" qlc:keysort(1, [{a, 1}, {b, 4}, {c, 6}], [])\n\" \" ]),\n\" \" V2 =\n\" \" qlc:q([ \n\" \" P0 ||\n\" - \" P0 = {W,Y} <- qlc:keysort(2, [{2,a},{3,b},{4,c}], [])\n\" + \" P0 = {W, Y} <-\n\" + \" qlc:keysort(2, [{2, a}, {3, b}, {4, c}], [])\n\" + \" ]),\n\" \" V3 =\n\" \" qlc:q([ \n\" - \" [G1|G2] ||\n\" + \" [G1 | G2] ||\n\" \" G1 <- V1,\n\" \" G2 <- V2,\n\" \" element(1, G1) == element(2, G2)\n\" \" ],\n\" - \" [{join,merge},{cache,list}]),\n\" + \" [{join, merge}, {cache, list}]),\n\" \" qlc:q([ \n\" - \" {A,X,Z,W} ||\n\" - \" A <- [a,b,c],\n\" - \" [{X,Z}|{W,Y}] <- V3,\n\" + \" {A, X, Z, W} ||\n\" + \" A <- [a, b, c],\n\" + \" [{X, Z} | {W, Y}] <- V3,\n\" \" X =:= Y\n\" \" ])\n\" \"end\", @@ -7221,14 +7218,21 @@ manpage(Config) when is_list(Config) -> gb_trees:lookup(K, gb_trees:from_orddict([])) of - {value,V} -> - [{K,V}]; + {value, V} -> + [{K, V}]; none -> [] end end, - [{1,a},{1,b},{1,c},{2,a},{2,b},{2,c}]), - ets:match_spec_compile([{{{'$1','$2'},'_'},[],['$1']}]))\", + [{1, a}, + {1, b}, + {1, c}, + {2, a}, + {2, b}, + {2, c}]), + ets:match_spec_compile([{{{'$1', '$2'}, '_'}, + [], + ['$1']}]))\", L = qlc:info(QH)">> ], run(Config, Ts), diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index b76c9f5341..7685c17967 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -21,24 +21,7 @@ -compile({nowarn_deprecated_function,[{random,seed,1}, {random,uniform_s,1}, {random,uniform_s,2}]}). - --export([all/0, suite/0, groups/0, group/1]). - --export([interval_int/1, interval_float/1, seed/1, - api_eq/1, reference/1, - basic_stats_uniform_1/1, basic_stats_uniform_2/1, - basic_stats_standard_normal/1, - basic_stats_normal/1, - stats_standard_normal_box_muller/1, - stats_standard_normal_box_muller_2/1, - stats_standard_normal/1, - uniform_real_conv/1, - plugin/1, measure/1, - reference_jump_state/1, reference_jump_procdict/1]). - --export([test/0, gen/1]). - --export([uniform_real_gen/1, uniform_gen/2]). +-compile([export_all, nowarn_export_all]). -include_lib("common_test/include/ct.hrl"). @@ -56,7 +39,8 @@ all() -> {group, distr_stats}, uniform_real_conv, plugin, measure, - {group, reference_jump} + {group, reference_jump}, + short_jump ]. groups() -> @@ -95,7 +79,7 @@ test() -> end, Tests). algs() -> - [exrop, exsp, exs1024s, exs64, exsplus, exs1024]. + [exsss, exrop, exsp, exs1024s, exs64, exsplus, exs1024, exro928ss]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -125,7 +109,7 @@ seed_1(Alg) -> S0 = get(rand_seed), S0 = rand:seed_s(Alg, {0, 0, 0}), %% Check that process_dict should not be used for seed_s functionality - _ = rand:seed_s(Alg, {1, 0, 0}), + _ = rand:seed_s(Alg, 4711), S0 = get(rand_seed), %% Test export ES0 = rand:export_seed(), @@ -262,31 +246,43 @@ reference(Config) when is_list(Config) -> ok. reference_1(Alg) -> - Refval = reference_val(Alg), - Testval = gen(Alg), - case Refval =:= Testval of - true -> ok; - false when Refval =:= not_implemented -> - exit({not_implemented,Alg}); - false -> - io:format("Failed: ~p~n",[Alg]), - io:format("Length ~p ~p~n",[length(Refval), length(Testval)]), - io:format("Head ~p ~p~n",[hd(Refval), hd(Testval)]), - exit(wrong_value) + Refval = reference_val(Alg), + if + Refval =:= not_implemented -> Refval; + true -> + case gen(Alg) of + Refval -> + io:format("Ok: ~p~n",[Alg]), + ok; + Testval -> + io:format("Failed: ~p~n",[Alg]), + io:format("Length ~p ~p~n",[length(Refval), length(Testval)]), + io:format("Head ~p ~p~n",[hd(Refval), hd(Testval)]), + show_wrong(Refval, Testval), + exit(wrong_value) + end end. +show_wrong([], []) -> + ok; +show_wrong([H|T1], [H|T2]) -> + show_wrong(T1, T2); +show_wrong([H1|_], [H2|_]) -> + io:format("Wrong ~p ~p~n",[H1,H2]). + + gen(Algo) -> State = - case Algo of - exs64 -> %% Printed with orig 'C' code and this seed - rand:seed_s({exs64, 12345678}); - _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop -> + if + Algo =:= exs64 -> %% Printed with orig 'C' code and this seed + rand:seed_s(exs64, [12345678]); + Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop; Algo =:= exsss -> %% Printed with orig 'C' code and this seed - rand:seed_s({Algo, [12345678|12345678]}); - _ when Algo =:= exs1024; Algo =:= exs1024s -> + rand:seed_s(Algo, [12345678,12345678]); + Algo =:= exs1024; Algo =:= exs1024s; Algo =:= exro928ss -> %% Printed with orig 'C' code and this seed - rand:seed_s({Algo, {lists:duplicate(16, 12345678), []}}); - _ -> + rand:seed_s(Algo, lists:duplicate(16, 12345678)); + true -> rand:seed(Algo, {100, 200, 300}) end, Max = range(State), @@ -442,7 +438,7 @@ stats_standard_normal_box_muller(Config) when is_list(Config) -> ([S|Z]) -> {Z, [S]} end, - State = [rand:seed(exrop)], + State = [rand:seed(exsss)], stats_standard_normal(NormalS, State, 3) catch error:_ -> {skip, "math:erfc/1 not supported"} @@ -467,7 +463,7 @@ stats_standard_normal_box_muller_2(Config) when is_list(Config) -> ([S|Z]) -> {Z, [S]} end, - State = [rand:seed(exrop)], + State = [rand:seed(exsss)], stats_standard_normal(NormalS, State, 3) catch error:_ -> {skip, "math:erfc/1 not supported"} @@ -479,7 +475,7 @@ stats_standard_normal(Config) when is_list(Config) -> try math:erfc(1.0) of _ -> stats_standard_normal( - fun rand:normal_s/1, rand:seed_s(exrop), Retries) + fun rand:normal_s/1, rand:seed_s(exsss), Retries) catch error:_ -> {skip, "math:erfc/1 not supported"} end. @@ -853,7 +849,8 @@ do_measure(_Config) -> Algs = algs() ++ try crypto:strong_rand_bytes(1) of - <<_>> -> [crypto64, crypto_cache, crypto] + <<_>> -> + [crypto64, crypto_cache, crypto_aes, crypto] catch error:low_entropy -> []; error:undef -> [] @@ -1074,7 +1071,7 @@ do_measure(_Config) -> end, State) end, - exrop, TMarkNormalFloat), + exsss, TMarkNormalFloat), ok. -define(LOOP_MEASURE, (?LOOP div 5)). @@ -1102,6 +1099,10 @@ measure_1(RangeFun, Fun, Alg, TMark) -> {rand, crypto:rand_seed_alg(crypto_cache)}; crypto -> {rand, crypto:rand_seed_s()}; + crypto_aes -> + {rand, + crypto:rand_seed_alg( + crypto_aes, crypto:strong_rand_bytes(256))}; random -> {random, random:seed(os:timestamp()), get(random_seed)}; _ -> @@ -1117,7 +1118,7 @@ measure_1(RangeFun, Fun, Alg, TMark) -> _ -> (Time * 100 + 50) div TMark end, io:format( - "~.12w: ~p ns ~p% [16#~.16b]~n", + "~.20w: ~p ns ~p% [16#~.16b]~n", [Alg, (Time * 1000 + 500) div ?LOOP_MEASURE, Percent, Range]), Parent ! {self(), Time}, @@ -1142,104 +1143,156 @@ reference_jump_state(Config) when is_list(Config) -> ok. reference_jump_1(Alg) -> - Refval = reference_jump_val(Alg), - Testval = gen_jump_1(Alg), - case Refval =:= Testval of - true -> ok; - false -> - io:format("Failed: ~p~n",[Alg]), - io:format("Length ~p ~p~n",[length(Refval), length(Testval)]), - io:format("Head ~p ~p~n",[hd(Refval), hd(Testval)]), - io:format("Vals ~p ~p~n",[Refval, Testval]), - exit(wrong_value) + Refval = reference_jump_val(Alg), + if + Refval =:= not_implemented -> Refval; + true -> + case gen_jump_1(Alg) of + Refval -> ok; + Testval -> + io:format( + "Failed: ~p~n",[Alg]), + io:format( + "Length ~p ~p~n", + [length(Refval), length(Testval)]), + io:format( + "Head ~p ~p~n",[hd(Refval), hd(Testval)]), + io:format( + "Vals ~p ~p~n",[Refval, Testval]), + exit(wrong_value) + end end. gen_jump_1(Algo) -> - State = - case Algo of - exs64 -> %% Test exception of not_implemented notice - try rand:jump(rand:seed_s(exs64)) - catch - error:not_implemented -> not_implemented - end; - _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop -> - %% Printed with orig 'C' code and this seed - rand:seed_s({Algo, [12345678|12345678]}); - _ when Algo =:= exs1024; Algo =:= exs1024s -> - %% Printed with orig 'C' code and this seed - rand:seed_s({Algo, {lists:duplicate(16, 12345678), []}}); - _ -> % unimplemented - not_implemented - end, - case State of - not_implemented -> [not_implemented]; - _ -> - Max = range(State), - gen_jump_1(?LOOP_JUMP, State, Max, []) + case Algo of + exs64 -> %% Test exception of not_implemented notice + try rand:jump(rand:seed_s(exs64)) + catch + error:not_implemented -> [error_not_implemented] + end; + _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop; Algo =:= exsss -> + %% Printed with orig 'C' code and this seed + gen_jump_2( + rand:seed_s(Algo, [12345678,12345678])); + _ when Algo =:= exs1024; Algo =:= exs1024s; Algo =:= exro928ss -> + %% Printed with orig 'C' code and this seed + gen_jump_2( + rand:seed_s(Algo, lists:duplicate(16, 12345678))) end. -gen_jump_1(N, State0, Max, Acc) when N > 0 -> +gen_jump_2(State) -> + Max = range(State), + gen_jump_3(?LOOP_JUMP, State, Max, []). + +gen_jump_3(N, State0, Max, Acc) when N > 0 -> {_, State1} = rand:uniform_s(Max, State0), {Random, State2} = rand:uniform_s(Max, rand:jump(State1)), case N rem (?LOOP_JUMP div 100) of - 0 -> gen_jump_1(N-1, State2, Max, [Random|Acc]); - _ -> gen_jump_1(N-1, State2, Max, Acc) + 0 -> gen_jump_3(N-1, State2, Max, [Random|Acc]); + _ -> gen_jump_3(N-1, State2, Max, Acc) end; -gen_jump_1(_, _, _, Acc) -> lists:reverse(Acc). +gen_jump_3(_, _, _, Acc) -> lists:reverse(Acc). %% Check if each algorithm generates the proper jump sequence %% with the internal state in the process dictionary. reference_jump_procdict(Config) when is_list(Config) -> - [reference_jump_0(Alg) || Alg <- algs()], + [reference_jump_p1(Alg) || Alg <- algs()], ok. -reference_jump_0(Alg) -> +reference_jump_p1(Alg) -> Refval = reference_jump_val(Alg), - Testval = gen_jump_0(Alg), - case Refval =:= Testval of - true -> ok; - false -> - io:format("Failed: ~p~n",[Alg]), - io:format("Length ~p ~p~n",[length(Refval), length(Testval)]), - io:format("Head ~p ~p~n",[hd(Refval), hd(Testval)]), - exit(wrong_value) + if + Refval =:= not_implemented -> Refval; + true -> + case gen_jump_p1(Alg) of + Refval -> ok; + Testval -> + io:format("Failed: ~p~n",[Alg]), + io:format("Length ~p ~p~n",[length(Refval), length(Testval)]), + io:format("Head ~p ~p~n",[hd(Refval), hd(Testval)]), + exit(wrong_value) + end end. -gen_jump_0(Algo) -> - Seed = case Algo of - exs64 -> %% Test exception of not_implemented notice - try - _ = rand:seed(exs64), - rand:jump() - catch - error:not_implemented -> not_implemented - end; - _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop -> - %% Printed with orig 'C' code and this seed - rand:seed({Algo, [12345678|12345678]}); - _ when Algo =:= exs1024; Algo =:= exs1024s -> - %% Printed with orig 'C' code and this seed - rand:seed({Algo, {lists:duplicate(16, 12345678), []}}); - _ -> % unimplemented - not_implemented - end, - case Seed of - not_implemented -> [not_implemented]; - _ -> - Max = range(Seed), - gen_jump_0(?LOOP_JUMP, Max, []) +gen_jump_p1(Algo) -> + case Algo of + exs64 -> %% Test exception of not_implemented notice + try + _ = rand:seed(exs64), + rand:jump() + catch + error:not_implemented -> [error_not_implemented] + end; + _ when Algo =:= exsplus; Algo =:= exsp; Algo =:= exrop; Algo =:= exsss -> + %% Printed with orig 'C' code and this seed + gen_jump_p2( + rand:seed(Algo, [12345678,12345678])); + _ when Algo =:= exs1024; Algo =:= exs1024s; Algo =:= exro928ss -> + %% Printed with orig 'C' code and this seed + gen_jump_p2( + rand:seed(Algo, lists:duplicate(16, 12345678))) end. -gen_jump_0(N, Max, Acc) when N > 0 -> +gen_jump_p2(Seed) -> + Max = range(Seed), + gen_jump_p3(?LOOP_JUMP, Max, []). + +gen_jump_p3(N, Max, Acc) when N > 0 -> _ = rand:uniform(Max), _ = rand:jump(), Random = rand:uniform(Max), case N rem (?LOOP_JUMP div 100) of - 0 -> gen_jump_0(N-1, Max, [Random|Acc]); - _ -> gen_jump_0(N-1, Max, Acc) + 0 -> gen_jump_p3(N-1, Max, [Random|Acc]); + _ -> gen_jump_p3(N-1, Max, Acc) end; -gen_jump_0(_, _, Acc) -> lists:reverse(Acc). +gen_jump_p3(_, _, Acc) -> lists:reverse(Acc). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +short_jump(Config) when is_list(Config) -> + Seed = erlang:system_time(), + short_jump( + rand:seed_s(exro928ss, Seed), + fun ({Alg,AlgState}) -> + {Alg,rand:exro928_jump_2pow20(AlgState)} + end), + short_jump( + crypto:rand_seed_alg_s(crypto_aes, integer_to_list(Seed)), + fun ({Alg,AlgState}) -> + {Alg,crypto:rand_plugin_aes_jump_2pow20(AlgState)} + end), + ok. + +short_jump({#{bits := Bits},_} = State_0, Jump2Pow20) -> + Range = 1 bsl Bits, + State_1 = repeat(7, Range, State_0), + %% + State_2a = repeat(1 bsl 20, Range, State_1), + State_2b = Jump2Pow20(State_1), + check(17, Range, State_2a, State_2b), + %% + {_,State_3a} = rand:uniform_s(Range, State_2a), + State_4a = Jump2Pow20(State_3a), + State_4b = repeat((1 bsl 20) + 1, Range, State_2b), + check(17, Range, State_4a, State_4b). + +repeat(0, _Range, State) -> + State; +repeat(N, Range, State) -> + {_, NewState} = rand:uniform_s(Range, State), + repeat(N - 1, Range, NewState). + +check(0, _Range, _StateA, _StateB) -> + ok; +check(N, Range, StateA, StateB) -> + {V,NewStateA} = rand:uniform_s(Range, StateA), + case rand:uniform_s(Range, StateB) of + {V,NewStateB} -> + check(N - 1, Range, NewStateA, NewStateB); + {Wrong,_} -> + ct:fail({Wrong,neq,V,for,N}) + end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Data @@ -1324,6 +1377,34 @@ reference_val(exsplus) -> 16#3dd493b8012970f,16#be13bed1e00e5c,16#ceef033b74ae10,16#3da38c6a50abe03, 16#15cbd1a421c7a8c,16#22794e3ec6ef3b1,16#26154d26e7ea99f,16#3a66681359a6ab6]; +reference_val(exsss) -> + [16#108e8d5b01,16#33b72092117209a,16#224d4d2961a2d0a,16#2c4c81aac3da48d, + 16#2f4bc39bfc36f3a,16#41826d4c4d243a,16#19871b8bb4e23ee,16#3e2112cdf9384b1, + 16#69801943bf91ab,16#2de1a603c31ec45,16#a90ca1991b831e,16#51ca29571a69a7, + 16#93ce3e511906cf,16#93ebc5768aef75,16#2412f284b902ae7,16#1ac10e758410c52, + 16#3f32494560368f6,16#39a5e82dcf0de95,16#3f4b14d59cc6a21,16#3174668db0b36ae, + 16#1449812fb8bd54e,16#eaca1f8ece51e1,16#2564b2545fd23c1,16#3cf3a2d2217e0d7, + 16#226f4164ba1d054,16#10dac9ae207ceef,16#17f2c4b2d40fcb9,16#1c1b282d386fdcb, + 16#a264f450ba2912,16#2a0a1dd67e52666,16#2be84eb835cb1e1,16#2a1cd9aa16ccc37, + 16#7dd5e8c2b3f490,16#254a3db4976c05b,16#2a0a67971ec1e63,16#13a0cbf7c0eed8a, + 16#3192d7edc0a20bc,16#2705ad756292e84,16#3ec429a18119c81,16#25944b38baa975b, + 16#291dcc43e3256f4,16#30d10b759237db,16#c1522a652058a,16#8ef1e9378381e6, + 16#1f442f33c2439f4,16#186087710a73818,16#12887f94b2b8387,16#3e42e8b1f3c9b4b, + 16#e462859d55f9d8,16#2356ae85be908de,16#15e96a927b3bc52,16#35c6dc52511ce46, + 16#7bc0624ce66e01,16#33ab7d95b738322,16#26f01effc182aa0,16#1b66ae7eaafea88, + 16#278f3dc14943b90,16#22178bc8d8faf28,16#396c37d53c11985,16#5e0d79d0b10f18, + 16#1be3de3b5675ec,16#d4db298f1f4b50,16#2da6cb99bb5c7b1,16#130b2dc17d03be8, + 16#f1847e7e059e9f,16#2da6591788326e7,16#222e4a18c24211c,16#949213ca49baab, + 16#b5129fec56f6a2,16#30f25f1e926f43e,16#1ddd8d04445fb4d,16#15995b542514150, + 16#1595fe879296296,16#e2f237a488453b,16#23e5cd2d6047890,16#3a5dc88fc954666, + 16#89bca9969b103,16#5e6893cd35dc63,16#1fed534feeeef5a,16#26f40e2147ee558, + 16#30c131a00625837,16#2618a7e617422e9,16#23630b297e45e7,16#1143b17502f3219, + 16#15607dac41168da,16#2886bdc314b3fb8,16#465d1cc1536546,16#30b09123e3a02e4, + 16#245a375f810be52,16#6a1b0792376a03,16#221425f59f2470f,16#867ce16dfac81c, + 16#9c62d95fae9b58,16#380381db1394426,16#34908dedc01c324,16#1f0ff517089b561, + 16#1571366dd873d32,16#3ee353dc56e192,16#15a1dee8d889b11,16#41036ad76d9888 + ]; + reference_val(exsp) -> reference_val(exsplus); reference_val(exs1024s) -> @@ -1390,7 +1471,50 @@ reference_val(exrop) -> 250789092615679985,78848633178610658,72059442721196128, 98223942961505519,191144652663779840, 102425686803727694,89058927716079076,80721467542933080, - 8462479817391645,2774921106204163]. + 8462479817391645,2774921106204163]; + +reference_val(exro928ss) -> +%% Same as for exrop, but this state init: +%% for (n = 0; n < 16; n++) { +%% s[n] = 12345678; + [16#000000108e8d5b01,16#03604028f2769dff,16#007f92f60bc7170c, + 16#035ea81a9898a5e2,16#0104c90c5a0c8178,16#0313514025cca717, + 16#03c5506b2a2e98cf,16#0098a5405961552e,16#004ad29eabb785a0, + 16#033ea8ec4efb8058,16#00b21545e62bef1c,16#0333fc5320703482, + 16#02c3c650e51a8d47,16#03a3b7fc848c9cda,16#03775adea6cddff5, + 16#01ae5499c9049973,16#03d3c90e5504e16b,16#0383cd6b6cb852e6, + 16#009c8d0996ef543a,16#0059cf671371af60,16#03dfd68ed980b719, + 16#0290f2a0acf2c5b0,16#029061df18d63b55,16#02e702ea4b45137b, + 16#029a0ccca604d848,16#01664c7cd31f0fa6,16#00dced83e60ccddc, + 16#008764d2c9a05f3e,16#02b9ca5f6a80c4ba,16#02daf93d2c566750, + 16#0147d326ead18ace,16#014b452efc19297f,16#0242d3f7a7237eca, + 16#0141bb68c2abce39,16#02d798e1230baf45,16#0216bf8f25c1ec2d, + 16#003a43ea733f1e1f,16#036c75390db736f3,16#028cca5f5f48c6f9, + 16#0186e4a17174d6cf,16#02152679dfa4c25c,16#01429b9f15e3b9d6, + 16#0134a61411d22bb0,16#01593f7d970d1c94,16#0205a7d8a305490f, + 16#01dd092272595a9c,16#0028c95208aad2d4,16#016347c25cc24162, + 16#025306acfb891309,16#0207a07e2bebef2f,16#024ee78d86ff5288, + 16#030b53192db97613,16#03f765cb9e98e611,16#025ec35a1e237377, + 16#03d81fd73102ef6f,16#0242dc8fea9a68b2,16#00abb876c1d4ea1b, + 16#00871ffd2b7e45fb,16#03593ff73c9be08d,16#00b96b2b8aca3688, + 16#0174aba957b7cf7b,16#012b7a5d4cf4a5b7,16#032a5260f2123db8, + 16#00f9374d88ee0080,16#030df39bec2ad657,16#00dce0cb81d006c4, + 16#038213b806303c76,16#03940aafdbfabf84,16#0398dbb26aeba037, + 16#01eb28d61951587f,16#00fed3d2aacfeef4,16#03499587547d6e40, + 16#01b192fe6e979e3c,16#00e974bf5f0a26d0,16#012ed94f76459c83, + 16#02d76859e7a82587,16#00d1d2c7b791f51b,16#03988058017a031b, + 16#00bbcf4b59d8e86d,16#015ed8b73a1b767c,16#0277283ea6a5ee74, + 16#002211460dd6d422,16#001ad62761ee9fbd,16#037311b44518b067, + 16#02b5ed61bf70904e,16#011862a05c1929fa,16#014be68683c3bab4, + 16#025c29aa5c508b07,16#00895c6106f97378,16#026ce91a3d671c7f, + 16#02591f4c74784293,16#02f0ed2a70bc1853,16#00a2762ff614bfbc, + 16#008f4e354f0c20d4,16#038b66fb587ed430,16#00636296e188de89, + 16#0278fadd143e74f5,16#029697ccf1b3a4c2,16#011eccb273404458, + 16#03f204064a9fe0c0]; + +reference_val(_) -> + not_implemented. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1421,6 +1545,33 @@ reference_jump_val(exsplus) -> 12504080415362731, 45083100453836317, 270968267812126657, 93505647407734103, 252852934678537969, 258758309277167202, 74250882143432077, 141629095984552833]; +reference_jump_val(exsss) -> + [16#304ae783d40db2b,16#1dfb196b3a5600a,16#2a24116effc6a0d,16#1f138d68c56725, + 16#9360a445e2f989,16#32ed8080390e242,16#294ca85a270cff6,16#1418e6296a88bf, + 16#114fae3dc578ba7,16#479c42c760eb72,16#334a40655df22d6,16#e7a85dd4d37d72, + 16#181db16c8925c77,16#1b8a5a8afd16cbd,16#329107bf9777a39,16#2fc915c08535e42, + 16#16696d142c6078,16#2e2a2601c919448,16#2246150d1000568,16#26109007cb3dd44, + 16#3761360723e3175,16#169abd352db74de,16#1c97d520983684f,16#12455f0adee8c66, + 16#46719cff00622d,16#1fc92792ed4e437,16#18e2edae21affb5,16#3a67fa9e3e7d46e, + 16#1313fdc2728aa74,16#1c1a2b577581db8,16#db49357ea196b1,16#10e219a21d93fc7, + 16#3c43abede083666,16#3eef5055a58bbf9,16#1975056f95d90e3,16#3916c133ab16d87, + 16#2bc0bea891c26f1,16#391e4b369fc6b36,16#183f83155a359f6,16#1d9f137e9d2e488, + 16#ef084de5f4cd3c,16#36a9cf7e29e55d3,16#19eca704e0409a7,16#1bdb99902896c69, + 16#21777e2ad128203,16#5d0369ec0563e4,16#36db40b863bd74a,16#33feb71b7515159, + 16#208d923ce26f257,16#3841b32891c082d,16#2748f224c2ba226,16#2fcd93b2daf79bb, + 16#2c8e6cacad58ec4,16#39850131a1a85f,16#134648d6eea624d,16#2e102e197d5725c, + 16#12ac280fa744758,16#1c18266c7442d16,16#22b5f91b15fe17e,16#316740ca870f7c8, + 16#720ed4836c426,16#1aac0f738d04f8c,16#34fcd2a647b462c,16#3d430ac755114a3, + 16#3692e3670fdf2a,16#265279ab0fc0a15,16#10bd883dee80945,16#10e7843413175e4, + 16#b291deba08cee2,16#3915a8234caf11,16#34b911b96707dbd,16#ae63fcda15fde6, + 16#b13b9091e82e41,16#29de1b6d70dc04f,16#23fbcbc409617e8,16#1389a0738061066, + 16#360f39af790f5d1,16#f436da2a7d12f5,16#2d06ba8da21e08,16#3601a6492b887d, + 16#2b2590b8c6cc186,16#f8d613b6904464,16#e5456786e46b78,16#201b8b1f96ed80c, + 16#1b75b86d9b843f2,16#2e8bfaa7243a630,16#125ff068a78c3b4,16#3875a28c48bd26e, + 16#f09a06941fc9d7,16#107c4de8ca77744,16#357c34144bb9ed6,16#3ccc55d3ebb3378, + 16#28db7cea7d3fdee,16#3197fd0b49f6370,16#11af6fedb708ea6,16#2bde0382e37469e, + 16#10666171abddb3f,16#1a8876c1f4e78a8,16#169c0efd4422043,16#1501c49abf0440f]; + reference_jump_val(exs1024) -> [2655961906500790629, 17003395417078685063, 10466831598958356428, 7603399148503548021, 1650550950190587188, 12294992315080723704, 15743995773860389219, 5492181000145247327, @@ -1452,7 +1603,7 @@ reference_jump_val(exsp) -> reference_jump_val(exsplus); reference_jump_val(exs1024s) -> reference_jump_val(exs1024); -reference_jump_val(exs64) -> [not_implemented]; +reference_jump_val(exs64) -> [error_not_implemented]; reference_jump_val(exrop) -> %% #include <stdint.h> %% #include <stdio.h> @@ -1517,7 +1668,50 @@ reference_jump_val(exrop) -> 250227633882474729,171181147785250210,55437891969696407, 241227318715885854,77323084015890802, 1663590009695191,234064400749487599,222983191707424780, - 254956809144783896,203898972156838252]. + 254956809144783896,203898972156838252]; + +reference_jump_val(exro928ss) -> +%% Same as for exrop, but this state init: +%% for (n = 0; n < 16; n++) { +%% s[n] = 12345678; + [16#031ee449e53b6689,16#001afeee12813137,16#005e2172711df36b, + 16#02850aea3a595d36,16#0029705187e891c7,16#001794badd489667, + 16#00ab621be15be56c,16#024b663a6924786b,16#03cab70b8ab854bf, + 16#01daa37601285320,16#02db955a53c40e89,16#01fbef51d5c65891, + 16#02fecf4116ed5f77,16#0349c2057246ac5d,16#01217f257c4fa148, + 16#0367ee84d020697d,16#01d5cf647fe23335,16#020941838adfb750, + 16#02c2da26b1d7b3e5,16#00d1583d34cea6c0,16#038be9cb5b527f50, + 16#00bfa93c1d7f4864,16#03778912a4f56b14,16#037fcabc483fa5c5, + 16#00a3c9de6aaf5fc7,16#03600b883b2f2b42,16#03797a99ffddfdfb, + 16#0189fead429945b7,16#0103ac90cd912508,16#03e3d872fd950d64, + 16#0214fc3e77dc2f02,16#02a084f4f0e580ca,16#035d2fe72266a7f3, + 16#02887c49ae7e41a4,16#0011dc026af83c51,16#02d28bfd32c2c517, + 16#022e4165c33ad4f3,16#01f053cf0687b052,16#035315e6e53c8918, + 16#01255312da07b572,16#0237f1da11ec9221,16#02faf2e282fb1fb1, + 16#0227423ec1787ebc,16#011fa5eb1505571c,16#0275ff9eaaa1abdd, + 16#03e2d032c3981cb4,16#0181bb32d51d3072,16#01b1d3939b9f16ec, + 16#0259f09f55d1112f,16#0396464a2767e428,16#039777c0368bdb9e, + 16#0320925f35f36c5f,16#02a35289e0af1248,16#02e80bd4bc72254b, + 16#00a8b11af1674d68,16#027735036100a69e,16#03c8c268ded7f254, + 16#03de80aa57c65217,16#00f2247754d24000,16#005582a42b467f89, + 16#0031906569729477,16#00fd523f2ca4fefe,16#00ad223113d1e336, + 16#0238ddf026cbfca9,16#028b98211cfed876,16#0354353ebcc0de9a, + 16#009ee370c1e154f4,16#033131af3b8a7f88,16#032291baa45801e3, + 16#00941fc2b45eb217,16#035d6a61fa101647,16#03fdb51f736f1bbc, + 16#0232f7b98539faa0,16#0311b35319e3a61e,16#0048356b17860eb5, + 16#01a205b2554ce71e,16#03f873ea136e29d6,16#003c67d5c3df5ffd, + 16#00cd19e7a8641648,16#0149a8c54e4ba45e,16#0329498d134d2f6a, + 16#03b69421ae65ee2b,16#01a8d20b59447429,16#006b2292571032a2, + 16#00c193b17da22ba5,16#01faa7ab62181249,16#00acd401cd596a00, + 16#005b5086c3531402,16#0259113d5d3d058d,16#00bef3f3ce4a43b2, + 16#014837a4070b893c,16#00460a26ac2eeec1,16#026219a8b8c63d7e, + 16#03c7b8ed032cf5a6,16#004da912a1fff131,16#0297de3716215741, + 16#0079fb9b4c715466,16#00a73bad4ae5a356,16#0072e606c0d4ab86, + 16#02374382d5f9bd2e]; + +reference_jump_val(_) -> + not_implemented. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/stdlib/test/rand_Xoroshiro928ss_dev.txt b/lib/stdlib/test/rand_Xoroshiro928ss_dev.txt new file mode 100644 index 0000000000..150f37fcfa --- /dev/null +++ b/lib/stdlib/test/rand_Xoroshiro928ss_dev.txt @@ -0,0 +1,343 @@ +%CopyrightBegin% + +Copyright Ericsson AB 2015-2017. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +%CopyrightEnd% + + +Memorable facts from designing the Xoroshiro928** generator +=========================================================== +AKA: exro928ss in the rand module + +Author: Raimo Niskanen, for the Erlang/OTP team @ Ericsson. + +Reference URL: http://vigna.di.unimi.it/ftp/papers/ScrambledLinear.pdf +i.e the Xoroshiro1024 generator with ** scrambler: + + int p; + uint64_t s[16]; + + const int q = p; + const uint64_t s0 = s[p = (p + 1) & 15]; + + uint64_t s15 = s[q]; + + const uint64_t result_starstar = rotl(s0 * S, R) * T; + + s15 ^= s0; + s[q] = rotl(s0, A) ^ s15 ^ (s15 << B); + s[p] = rotl(s15, C); + +Where {S, R, T} = {5, 7, 9} as recommended in the paper. + +We want to scale down to 58 bit words (16 of them) +so we get a generator with period 2^928 - 1. + +{A, B, C} were deduced as follows +--------------------------------- + +First, to find which triplets that give a full period generator +one have to factor 2^928 - 1. + +https://www.alpertron.com.ar/ECM.HTM actually could do that +and gave the result: + + Value + 2^928 - 1 + + 2269 007733 883335 972287 082669 296112 915239 349672 942191 252221 331572 + 442536 403137 824056 312817 862695 551072 066953 619064 625508 194663 + 368599 769448 406663 254670 871573 830845 597595 897613 333042 429214 + 224697 474472 410882 236254 024057 110212 260250 671521 235807 709272 + 244389 361641 091086 035023 229622 419455 (280 digits) = 3 × 5 × 17 × 59 × + 233 × 257 × 929 × 1103 × 2089 × 5569 × 8353 × 59393 × 65537 × 3 033169 × + 39 594977 × 107 367629 × 536 903681 × 748 264961 × 2245 984577 × 239 + 686663 718401 × 15 929619 591127 520827 829953 × 82280 195167 144119 + 832390 568177 × 6033 312171 721035 031651 315652 130497 (34 digits) × 18 + 774318 450142 955120 650303 957350 521748 903233 (44 digits) × 15 694604 + 006012 505869 851221 169365 594050 637743 819041 (50 digits) + +Sebastiano Vigna from that calculated all full period triplets, at the +end of this document, and the ones with the highest degree were: + + 23-25-12 411 + 36-2-35 411 + 55-5-54 411 + 14-19-11 415 + 34-37-5 415 + 37-3-56 415 + 55-11-54 417 + 30-3-41 419 + 11-45-50 423 + 50-19-47 423 + 52-27-13 427 + 54-9-25 433 + 56-43-35 433 + 44-9-45 441 + +All these candidates were tested with TestU01-1.2.3: + + http://simul.iro.umontreal.ca/testu01/tu01.html + +A plugin was created with parameters for {A, B, C} and, since +TestU01 is a 32-bit test tool, with parameters to reverse +the generated bits or not, and to take the 32 highest or lowest +bits from the reversed or non-reversed 58 bit output. + +The generators were seeded with a SplitMix64 generator like the +one used for seeding this generator in the rand module, +taking the 58 lowest bits and wasting all zero values. + +3 runs were made with all candidates and all four bit selection variants. +For these runs the seeder was initialized with 12345678, 876543212345678 +and 1234567890. + +After all these runs the candidate with the highest degree: 44-9-45 +had not gotten any suspicious p-value at all. All the other +got p-values around 5.0e-4 worst 9.8e-6 suggesting only random +failures, so they would probably have worked about as well. + +Finally 44-9-45 was run through PractRand-0.93: + + http://pracrand.sourceforge.net/ + +Again, all 4 bit selection variants of 32 bits were run. +Random failures with p-values around e-3..e-4 for the "smaller" +tests, but for 8, 16 and 32 TB tests no anomalies were found +(with the internal seeder masked to 58 bits): + + Xoroshiro928High seed: 0x3178ec5d + Xoroshiro928Low seed: 0xa9a04fb9 + Xoroshiro928ReverseHigh seed: 0xfa0bdbab + Xoroshiro928ReverseLow seed: 0xada51705 + + +Then, S. Vigna calculated the 2^512 jump coefficient as well +as a 2^20 jump coefficient (for testing purposes) for 44-9-45. + +2^512: + { 0x44085302f77130ca, 0xba05381fdfd14902, 0x10a1de1d7d6813d2, + 0xb83fe51a1eb3be19, 0xa81b0090567fd9f0, 0x5ac26d5d20f9b49f, + 0x4ddd98ee4be41e01, 0x0657e19f00d4b358, 0xf02f778573cf0f0a, + 0xb45a3a8a3cef3cc0, 0x6e62a33cc2323831, 0xbcb3b7c4cc049c53, + 0x83f240c6007e76ce, 0xe19f5fc1a1504acd, 0x00000000b10773cb } + +2^20: + { 0xbdb966a3daf905e6, 0x644807a56270cf78, 0xda90f4a806c17e9e, + 0x4a426866bfad3c77, 0xaf699c306d8e7566, 0x8ebc73c700b8b091, + 0xc081a7bf148531fb, 0xdc4d3af15f8a4dfd, 0x90627c014098f4b6, + 0x06df2eb1feaf0fb6, 0x5bdeb1a5a90f2e6b, 0xa480c5878c3549bd, + 0xff45ef33c82f3d48, 0xa30bebc15fefcc78, 0x00000000cb3d181c } + +Standard jump function pseudocode: + + Jump constant j = 0xb10773cb...44085302f77130ca + Generator state: s + New generator state: t = 0 + foreach bit in j, low to high: + if the bit is one: + t ^= s + next s + s = t + + +The complete list of full period constants +------------------------------------------ + +29-48-54 27 x^928 + x^874 + x^870 + x^840 + x^814 + x^784 + x^759 + x^750 + x^724 + x^720 + x^634 + x^630 + x^600 + x^574 + x^544 + x^519 + x^510 + x^484 + x^480 + x^390 + x^360 + x^270 + x^240 + x^150 + x^120 + x^30 + 1 + +29-18-56 65 x^928 + x^870 + x^850 + x^840 + x^832 + x^821 + x^802 + x^791 + x^761 + x^750 + x^734 + x^731 + x^720 + x^712 + x^701 + x^686 + x^674 + x^671 + x^663 + x^641 + x^630 + x^611 + x^610 + x^603 + x^600 + x^596 + x^577 + x^566 + x^547 + x^524 + x^517 + x^510 + x^487 + x^480 + x^476 + x^464 + x^457 + x^446 + x^427 + x^423 + x^397 + x^390 + x^367 + x^363 + x^360 + x^356 + x^352 + x^341 + x^326 + x^322 + x^311 + x^281 + x^270 + x^251 + x^240 + x^236 + x^232 + x^221 + x^191 + x^161 + x^150 + x^131 + x^120 + x^30 + 1 + +29-32-36 81 x^928 + x^874 + x^870 + x^840 + x^820 + x^819 + x^794 + x^790 + x^770 + x^765 + x^764 + x^760 + x^754 + x^750 + x^740 + x^739 + x^735 + x^734 + x^730 + x^709 + x^704 + x^674 + x^670 + x^665 + x^660 + x^650 + x^649 + x^635 + x^634 + x^619 + x^614 + x^605 + x^595 + x^585 + x^584 + x^579 + x^564 + x^555 + x^545 + x^540 + x^524 + x^515 + x^514 + x^495 + x^490 + x^485 + x^460 + x^455 + x^435 + x^425 + x^400 + x^395 + x^375 + x^370 + x^365 + x^340 + x^315 + x^310 + x^305 + x^300 + x^290 + x^285 + x^275 + x^260 + x^250 + x^245 + x^240 + x^215 + x^190 + x^180 + x^170 + x^150 + x^130 + x^120 + x^115 + x^105 + x^100 + x^75 + x^40 + x^30 + 1 + +39-10-20 105 x^928 + x^898 + x^870 + x^841 + x^840 + x^808 + x^783 + x^782 + x^781 + x^780 + x^778 + x^754 + x^752 + x^724 + x^721 + x^720 + x^696 + x^692 + x^688 + x^667 + x^666 + x^661 + x^660 + x^658 + x^638 + x^636 + x^632 + x^609 + x^607 + x^606 + x^605 + x^604 + x^601 + x^600 + x^580 + x^578 + x^576 + x^572 + x^568 + x^549 + x^548 + x^541 + x^540 + x^538 + x^516 + x^512 + x^489 + x^485 + x^484 + x^481 + x^480 + x^456 + x^452 + x^448 + x^429 + x^428 + x^421 + x^420 + x^418 + x^400 + x^398 + x^396 + x^392 + x^368 + x^365 + x^364 + x^336 + x^328 + x^319 + x^318 + x^315 + x^314 + x^298 + x^286 + x^278 + x^276 + x^274 + x^255 + x^254 + x^249 + x^245 + x^228 + x^226 + x^220 + x^208 + x^199 + x^198 + x^189 + x^188 + x^178 + x^168 + x^166 + x^160 + x^129 + x^128 + x^108 + x^100 + x^88 + x^79 + x^78 + x^69 + x^68 + x^58 + x^40 + 1 + +4-4-9 149 x^928 + x^898 + x^886 + x^870 + x^856 + x^826 + x^823 + x^808 + x^796 + x^793 + x^784 + x^778 + x^772 + x^766 + x^760 + x^754 + x^751 + x^742 + x^736 + x^734 + x^724 + x^713 + x^709 + x^706 + x^703 + x^694 + x^691 + x^688 + x^676 + x^674 + x^673 + x^664 + x^662 + x^658 + x^653 + x^641 + x^640 + x^634 + x^632 + x^631 + x^620 + x^614 + x^611 + x^608 + x^604 + x^602 + x^599 + x^590 + x^583 + x^581 + x^578 + x^574 + x^572 + x^571 + x^569 + x^568 + x^561 + x^554 + x^553 + x^551 + x^544 + x^542 + x^540 + x^538 + x^532 + x^531 + x^518 + x^514 + x^512 + x^502 + x^498 + x^489 + x^488 + x^484 + x^482 + x^480 + x^479 + x^468 + x^463 + x^456 + x^454 + x^452 + x^449 + x^448 + x^438 + x^433 + x^429 + x^424 + x^422 + x^420 + x^418 + x^408 + x^406 + x^401 + x^396 + x^394 + x^392 + x^380 + x^378 + x^376 + x^371 + x^367 + x^364 + x^362 + x^360 + x^350 + x^348 + x^346 + x^341 + x^334 + x^332 + x^328 + x^318 + x^311 + x^307 + x^304 + x^302 + x^300 + x^298 + x^288 + x^274 + x^272 + x^254 + x^249 + x^242 + x^240 + x^233 + x^212 + x^208 + x^189 + x^180 + x^178 + x^166 + x^152 + x^143 + x^136 + x^127 + x^122 + x^120 + x^113 + x^106 + x^88 + x^83 + x^81 + x^67 + x^58 + x^51 + x^14 + 1 + +44-44-45 157 x^928 + x^898 + x^870 + x^850 + x^830 + x^820 + x^808 + x^800 + x^790 + x^782 + x^778 + x^777 + x^760 + x^757 + x^752 + x^747 + x^742 + x^727 + x^702 + x^694 + x^689 + x^688 + x^679 + x^674 + x^659 + x^658 + x^657 + x^644 + x^642 + x^637 + x^634 + x^627 + x^619 + x^612 + x^607 + x^606 + x^601 + x^596 + x^594 + x^591 + x^590 + x^584 + x^576 + x^568 + x^566 + x^561 + x^560 + x^556 + x^554 + x^546 + x^541 + x^539 + x^538 + x^518 + x^516 + x^511 + x^509 + x^503 + x^502 + x^499 + x^496 + x^488 + x^476 + x^471 + x^468 + x^466 + x^464 + x^448 + x^446 + x^441 + x^436 + x^421 + x^420 + x^415 + x^411 + x^408 + x^404 + x^400 + x^388 + x^383 + x^381 + x^379 + x^376 + x^363 + x^358 + x^356 + x^354 + x^346 + x^344 + x^340 + x^333 + x^330 + x^328 + x^325 + x^316 + x^312 + x^307 + x^305 + x^302 + x^284 + x^282 + x^277 + x^270 + x^268 + x^267 + x^256 + x^252 + x^249 + x^245 + x^244 + x^239 + x^238 + x^236 + x^231 + x^228 + x^226 + x^224 + x^222 + x^215 + x^208 + x^207 + x^205 + x^203 + x^201 + x^199 + x^196 + x^194 + x^192 + x^189 + x^185 + x^184 + x^182 + x^180 + x^175 + x^171 + x^168 + x^166 + x^162 + x^161 + x^155 + x^152 + x^150 + x^141 + x^134 + x^132 + x^131 + x^127 + x^122 + x^111 + x^104 + x^92 + x^90 + x^83 + x^81 + x^58 + x^37 + 1 + +40-40-7 167 x^928 + x^898 + x^870 + x^866 + x^838 + x^836 + x^807 + x^806 + x^804 + x^778 + x^777 + x^774 + x^745 + x^744 + x^742 + x^718 + x^714 + x^713 + x^712 + x^687 + x^658 + x^657 + x^656 + x^655 + x^654 + x^626 + x^620 + x^619 + x^598 + x^596 + x^592 + x^591 + x^589 + x^566 + x^564 + x^563 + x^562 + x^560 + x^558 + x^538 + x^534 + x^530 + x^529 + x^504 + x^503 + x^502 + x^500 + x^499 + x^498 + x^496 + x^478 + x^474 + x^470 + x^469 + x^443 + x^442 + x^441 + x^439 + x^438 + x^436 + x^434 + x^432 + x^418 + x^416 + x^414 + x^410 + x^409 + x^407 + x^406 + x^405 + x^402 + x^386 + x^383 + x^380 + x^374 + x^373 + x^372 + x^370 + x^356 + x^350 + x^349 + x^348 + x^347 + x^346 + x^345 + x^340 + x^328 + x^326 + x^324 + x^323 + x^321 + x^316 + x^314 + x^308 + x^298 + x^296 + x^294 + x^289 + x^285 + x^283 + x^282 + x^281 + x^280 + x^278 + x^263 + x^262 + x^259 + x^257 + x^255 + x^251 + x^249 + x^248 + x^246 + x^233 + x^232 + x^229 + x^220 + x^218 + x^217 + x^216 + x^208 + x^202 + x^201 + x^199 + x^195 + x^190 + x^188 + x^187 + x^186 + x^178 + x^172 + x^169 + x^167 + x^162 + x^161 + x^159 + x^158 + x^157 + x^146 + x^142 + x^139 + x^135 + x^125 + x^123 + x^116 + x^112 + x^107 + x^105 + x^94 + x^93 + x^88 + x^86 + x^84 + x^82 + x^75 + x^73 + x^69 + x^64 + x^58 + x^56 + x^54 + x^52 + x^43 + x^41 + x^39 + x^32 + 1 + +47-47-18 217 x^928 + x^898 + x^871 + x^870 + x^843 + x^842 + x^841 + x^813 + x^812 + x^811 + x^808 + x^787 + x^786 + x^785 + x^783 + x^781 + x^778 + x^759 + x^757 + x^754 + x^753 + x^751 + x^729 + x^727 + x^726 + x^723 + x^721 + x^701 + x^700 + x^699 + x^697 + x^693 + x^691 + x^688 + x^673 + x^672 + x^670 + x^667 + x^663 + x^661 + x^658 + x^647 + x^644 + x^641 + x^637 + x^633 + x^631 + x^618 + x^617 + x^614 + x^613 + x^612 + x^611 + x^610 + x^607 + x^603 + x^601 + x^586 + x^585 + x^582 + x^581 + x^580 + x^577 + x^573 + x^571 + x^568 + x^562 + x^560 + x^556 + x^555 + x^554 + x^553 + x^551 + x^547 + x^543 + x^541 + x^538 + x^532 + x^530 + x^528 + x^524 + x^522 + x^521 + x^517 + x^513 + x^511 + x^506 + x^504 + x^502 + x^500 + x^497 + x^495 + x^491 + x^487 + x^483 + x^481 + x^478 + x^474 + x^472 + x^467 + x^466 + x^465 + x^461 + x^457 + x^453 + x^451 + x^444 + x^438 + x^437 + x^436 + x^435 + x^431 + x^427 + x^423 + x^412 + x^408 + x^407 + x^405 + x^401 + x^397 + x^390 + x^384 + x^377 + x^375 + x^371 + x^367 + x^366 + x^365 + x^363 + x^362 + x^360 + x^358 + x^356 + x^352 + x^350 + x^347 + x^345 + x^341 + x^337 + x^336 + x^334 + x^332 + x^324 + x^317 + x^315 + x^311 + x^307 + x^306 + x^305 + x^304 + x^302 + x^300 + x^298 + x^296 + x^287 + x^285 + x^281 + x^279 + x^277 + x^276 + x^274 + x^268 + x^266 + x^257 + x^255 + x^247 + x^244 + x^242 + x^240 + x^238 + x^234 + x^227 + x^221 + x^219 + x^216 + x^214 + x^206 + x^197 + x^195 + x^193 + x^182 + x^176 + x^161 + x^156 + x^152 + x^146 + x^139 + x^133 + x^116 + x^111 + x^109 + x^107 + x^96 + x^94 + x^92 + x^90 + x^88 + x^85 + x^81 + x^79 + x^77 + x^73 + x^66 + x^64 + x^62 + x^60 + x^53 + x^51 + x^47 + x^45 + x^36 + x^34 + x^32 + x^19 + x^17 + x^15 + 1 + +31-52-54 249 x^928 + x^898 + x^872 + x^870 + x^842 + x^841 + x^838 + x^836 + x^812 + x^810 + x^782 + x^781 + x^779 + x^778 + x^774 + x^753 + x^752 + x^750 + x^748 + x^723 + x^722 + x^721 + x^719 + x^717 + x^714 + x^712 + x^694 + x^690 + x^689 + x^688 + x^686 + x^684 + x^664 + x^663 + x^662 + x^661 + x^660 + x^658 + x^657 + x^656 + x^655 + x^654 + x^653 + x^652 + x^650 + x^634 + x^633 + x^631 + x^630 + x^629 + x^628 + x^622 + x^604 + x^599 + x^597 + x^596 + x^593 + x^592 + x^590 + x^588 + x^570 + x^567 + x^566 + x^562 + x^560 + x^536 + x^534 + x^533 + x^531 + x^530 + x^526 + x^513 + x^512 + x^510 + x^509 + x^508 + x^507 + x^506 + x^505 + x^504 + x^502 + x^498 + x^483 + x^482 + x^481 + x^479 + x^478 + x^476 + x^466 + x^464 + x^454 + x^445 + x^438 + x^424 + x^423 + x^422 + x^421 + x^419 + x^418 + x^411 + x^404 + x^402 + x^394 + x^393 + x^392 + x^391 + x^390 + x^387 + x^386 + x^383 + x^380 + x^379 + x^378 + x^377 + x^376 + x^374 + x^364 + x^362 + x^361 + x^359 + x^357 + x^356 + x^355 + x^354 + x^351 + x^349 + x^348 + x^347 + x^345 + x^342 + x^340 + x^332 + x^330 + x^329 + x^328 + x^327 + x^325 + x^324 + x^323 + x^321 + x^319 + x^318 + x^316 + x^314 + x^312 + x^302 + x^301 + x^300 + x^296 + x^295 + x^294 + x^291 + x^289 + x^287 + x^286 + x^283 + x^278 + x^266 + x^265 + x^264 + x^263 + x^262 + x^260 + x^259 + x^255 + x^250 + x^234 + x^232 + x^229 + x^227 + x^226 + x^225 + x^222 + x^218 + x^216 + x^208 + x^204 + x^200 + x^198 + x^197 + x^195 + x^194 + x^193 + x^190 + x^188 + x^178 + x^174 + x^173 + x^172 + x^171 + x^170 + x^169 + x^168 + x^166 + x^165 + x^164 + x^162 + x^161 + x^160 + x^159 + x^158 + x^157 + x^156 + x^154 + x^142 + x^141 + x^139 + x^138 + x^135 + x^131 + x^128 + x^126 + x^116 + x^112 + x^110 + x^107 + x^106 + x^105 + x^104 + x^102 + x^100 + x^98 + x^96 + x^95 + x^94 + x^92 + x^88 + x^80 + x^79 + x^76 + x^74 + x^73 + x^71 + x^70 + x^69 + x^66 + x^65 + x^64 + x^56 + x^54 + x^48 + x^46 + x^45 + x^43 + x^38 + x^35 + x^33 + x^30 + 1 + +57-54-32 249 x^928 + x^898 + x^870 + x^855 + x^848 + x^834 + x^826 + x^825 + x^808 + x^806 + x^805 + x^804 + x^798 + x^796 + x^788 + x^778 + x^776 + x^775 + x^766 + x^763 + x^756 + x^752 + x^745 + x^742 + x^736 + x^735 + x^734 + x^731 + x^715 + x^714 + x^713 + x^705 + x^703 + x^694 + x^688 + x^684 + x^682 + x^671 + x^660 + x^658 + x^652 + x^642 + x^641 + x^639 + x^634 + x^630 + x^621 + x^620 + x^616 + x^612 + x^609 + x^604 + x^602 + x^593 + x^591 + x^584 + x^579 + x^570 + x^568 + x^566 + x^565 + x^562 + x^558 + x^554 + x^550 + x^544 + x^536 + x^535 + x^533 + x^530 + x^529 + x^528 + x^524 + x^522 + x^517 + x^512 + x^505 + x^496 + x^494 + x^492 + x^491 + x^489 + x^480 + x^478 + x^475 + x^471 + x^464 + x^461 + x^454 + x^452 + x^450 + x^446 + x^445 + x^441 + x^438 + x^437 + x^436 + x^434 + x^430 + x^429 + x^427 + x^425 + x^420 + x^418 + x^416 + x^415 + x^413 + x^411 + x^409 + x^402 + x^397 + x^394 + x^392 + x^388 + x^386 + x^384 + x^377 + x^375 + x^371 + x^368 + x^367 + x^365 + x^364 + x^363 + x^362 + x^360 + x^358 + x^356 + x^353 + x^351 + x^346 + x^342 + x^341 + x^338 + x^335 + x^333 + x^330 + x^328 + x^326 + x^325 + x^317 + x^307 + x^304 + x^300 + x^296 + x^295 + x^293 + x^292 + x^288 + x^286 + x^285 + x^284 + x^283 + x^278 + x^277 + x^276 + x^273 + x^271 + x^270 + x^262 + x^261 + x^255 + x^253 + x^249 + x^248 + x^246 + x^245 + x^240 + x^236 + x^232 + x^231 + x^227 + x^224 + x^223 + x^221 + x^220 + x^217 + x^216 + x^212 + x^208 + x^205 + x^204 + x^198 + x^196 + x^192 + x^191 + x^190 + x^189 + x^187 + x^186 + x^178 + x^176 + x^175 + x^171 + x^169 + x^165 + x^160 + x^159 + x^158 + x^154 + x^152 + x^150 + x^148 + x^146 + x^144 + x^141 + x^134 + x^132 + x^131 + x^129 + x^127 + x^125 + x^123 + x^120 + x^118 + x^114 + x^113 + x^111 + x^106 + x^104 + x^103 + x^102 + x^101 + x^100 + x^99 + x^97 + x^95 + x^93 + x^88 + x^85 + x^77 + x^67 + x^62 + x^60 + x^55 + x^54 + x^53 + x^52 + x^51 + x^50 + x^48 + x^46 + x^40 + x^37 + x^33 + x^28 + x^27 + x^24 + 1 + +1-38-28 251 x^928 + x^898 + x^870 + x^866 + x^836 + x^834 + x^832 + x^818 + x^808 + x^806 + x^804 + x^797 + x^788 + x^778 + x^776 + x^770 + x^767 + x^765 + x^738 + x^736 + x^733 + x^724 + x^717 + x^708 + x^706 + x^703 + x^701 + x^690 + x^688 + x^685 + x^678 + x^677 + x^676 + x^674 + x^672 + x^664 + x^658 + x^656 + x^652 + x^647 + x^645 + x^640 + x^626 + x^624 + x^621 + x^620 + x^616 + x^610 + x^608 + x^604 + x^600 + x^594 + x^593 + x^592 + x^590 + x^580 + x^578 + x^576 + x^575 + x^574 + x^572 + x^568 + x^566 + x^565 + x^564 + x^562 + x^560 + x^558 + x^552 + x^550 + x^549 + x^548 + x^544 + x^541 + x^538 + x^536 + x^533 + x^532 + x^529 + x^526 + x^522 + x^520 + x^519 + x^517 + x^514 + x^513 + x^509 + x^508 + x^504 + x^501 + x^494 + x^493 + x^484 + x^482 + x^469 + x^468 + x^466 + x^461 + x^453 + x^452 + x^448 + x^447 + x^436 + x^428 + x^424 + x^422 + x^420 + x^416 + x^414 + x^413 + x^412 + x^409 + x^408 + x^406 + x^405 + x^404 + x^400 + x^399 + x^397 + x^393 + x^391 + x^388 + x^385 + x^382 + x^381 + x^376 + x^374 + x^372 + x^370 + x^368 + x^366 + x^365 + x^360 + x^350 + x^349 + x^344 + x^341 + x^340 + x^336 + x^335 + x^334 + x^333 + x^332 + x^328 + x^326 + x^322 + x^319 + x^318 + x^308 + x^304 + x^300 + x^298 + x^293 + x^292 + x^290 + x^289 + x^287 + x^284 + x^281 + x^279 + x^278 + x^272 + x^271 + x^269 + x^264 + x^263 + x^261 + x^256 + x^254 + x^253 + x^250 + x^240 + x^238 + x^234 + x^229 + x^228 + x^226 + x^223 + x^221 + x^218 + x^213 + x^210 + x^208 + x^204 + x^198 + x^197 + x^196 + x^194 + x^191 + x^188 + x^186 + x^180 + x^176 + x^174 + x^172 + x^170 + x^169 + x^167 + x^165 + x^161 + x^159 + x^157 + x^153 + x^150 + x^149 + x^148 + x^145 + x^136 + x^135 + x^134 + x^133 + x^132 + x^130 + x^128 + x^127 + x^125 + x^122 + x^120 + x^118 + x^113 + x^110 + x^109 + x^108 + x^100 + x^96 + x^94 + x^92 + x^90 + x^88 + x^86 + x^85 + x^84 + x^81 + x^80 + x^72 + x^70 + x^66 + x^58 + x^53 + x^45 + x^36 + x^34 + x^32 + x^30 + x^24 + x^20 + x^16 + x^12 + x^8 + x^2 + 1 + +47-57-20 279 x^928 + x^898 + x^880 + x^870 + x^864 + x^851 + x^850 + x^846 + x^830 + x^821 + x^820 + x^817 + x^812 + x^808 + x^807 + x^804 + x^796 + x^790 + x^783 + x^782 + x^777 + x^773 + x^762 + x^757 + x^753 + x^744 + x^743 + x^739 + x^731 + x^728 + x^714 + x^713 + x^710 + x^709 + x^705 + x^701 + x^694 + x^692 + x^688 + x^684 + x^683 + x^675 + x^672 + x^671 + x^668 + x^662 + x^660 + x^654 + x^653 + x^646 + x^642 + x^641 + x^637 + x^626 + x^624 + x^623 + x^608 + x^607 + x^603 + x^598 + x^593 + x^592 + x^590 + x^585 + x^578 + x^577 + x^574 + x^573 + x^572 + x^569 + x^568 + x^567 + x^563 + x^559 + x^558 + x^555 + x^551 + x^548 + x^547 + x^542 + x^539 + x^538 + x^537 + x^536 + x^535 + x^532 + x^530 + x^529 + x^525 + x^524 + x^521 + x^518 + x^517 + x^506 + x^505 + x^501 + x^496 + x^491 + x^490 + x^487 + x^471 + x^467 + x^462 + x^456 + x^452 + x^448 + x^446 + x^441 + x^440 + x^437 + x^433 + x^428 + x^420 + x^412 + x^411 + x^404 + x^403 + x^402 + x^401 + x^399 + x^398 + x^396 + x^394 + x^381 + x^378 + x^376 + x^371 + x^370 + x^366 + x^365 + x^360 + x^355 + x^354 + x^351 + x^348 + x^346 + x^344 + x^342 + x^340 + x^339 + x^337 + x^335 + x^333 + x^331 + x^330 + x^329 + x^328 + x^327 + x^320 + x^318 + x^313 + x^307 + x^299 + x^298 + x^297 + x^296 + x^295 + x^294 + x^293 + x^291 + x^283 + x^282 + x^281 + x^280 + x^279 + x^277 + x^276 + x^266 + x^265 + x^264 + x^263 + x^262 + x^261 + x^259 + x^258 + x^257 + x^256 + x^251 + x^248 + x^247 + x^242 + x^240 + x^235 + x^234 + x^233 + x^231 + x^230 + x^229 + x^227 + x^225 + x^224 + x^219 + x^214 + x^212 + x^210 + x^206 + x^204 + x^203 + x^200 + x^196 + x^195 + x^194 + x^192 + x^191 + x^190 + x^188 + x^185 + x^182 + x^178 + x^176 + x^173 + x^159 + x^158 + x^157 + x^156 + x^151 + x^146 + x^144 + x^143 + x^142 + x^140 + x^138 + x^136 + x^131 + x^130 + x^128 + x^127 + x^126 + x^123 + x^122 + x^118 + x^117 + x^115 + x^114 + x^113 + x^108 + x^105 + x^99 + x^98 + x^94 + x^90 + x^88 + x^87 + x^86 + x^84 + x^82 + x^81 + x^80 + x^79 + x^78 + x^75 + x^74 + x^72 + x^71 + x^70 + x^65 + x^64 + x^58 + x^57 + x^56 + x^55 + x^52 + x^50 + x^48 + x^47 + x^46 + x^45 + x^42 + x^39 + x^26 + x^24 + x^23 + x^22 + x^20 + x^18 + x^15 + x^8 + x^4 + 1 + +3-16-56 281 x^928 + x^898 + x^882 + x^870 + x^836 + x^820 + x^816 + x^808 + x^803 + x^802 + x^799 + x^778 + x^773 + x^770 + x^762 + x^757 + x^754 + x^753 + x^750 + x^737 + x^733 + x^720 + x^717 + x^713 + x^700 + x^688 + x^683 + x^680 + x^679 + x^674 + x^660 + x^658 + x^656 + x^654 + x^653 + x^650 + x^646 + x^644 + x^641 + x^640 + x^637 + x^634 + x^633 + x^623 + x^618 + x^617 + x^616 + x^613 + x^597 + x^590 + x^585 + x^584 + x^581 + x^577 + x^574 + x^572 + x^563 + x^562 + x^560 + x^555 + x^554 + x^553 + x^551 + x^547 + x^544 + x^537 + x^536 + x^533 + x^525 + x^521 + x^520 + x^519 + x^517 + x^515 + x^513 + x^510 + x^508 + x^505 + x^502 + x^501 + x^498 + x^493 + x^490 + x^485 + x^480 + x^477 + x^476 + x^475 + x^472 + x^469 + x^461 + x^458 + x^453 + x^449 + x^448 + x^445 + x^444 + x^442 + x^435 + x^433 + x^432 + x^431 + x^426 + x^425 + x^422 + x^420 + x^418 + x^412 + x^411 + x^409 + x^405 + x^403 + x^401 + x^400 + x^397 + x^393 + x^385 + x^381 + x^368 + x^366 + x^365 + x^362 + x^361 + x^359 + x^356 + x^353 + x^350 + x^346 + x^341 + x^338 + x^335 + x^334 + x^333 + x^332 + x^328 + x^326 + x^325 + x^323 + x^322 + x^320 + x^319 + x^317 + x^316 + x^315 + x^313 + x^312 + x^310 + x^309 + x^307 + x^305 + x^300 + x^295 + x^293 + x^291 + x^290 + x^289 + x^288 + x^286 + x^285 + x^282 + x^277 + x^274 + x^273 + x^266 + x^265 + x^263 + x^261 + x^259 + x^255 + x^253 + x^251 + x^248 + x^247 + x^246 + x^242 + x^241 + x^239 + x^234 + x^233 + x^232 + x^230 + x^229 + x^228 + x^226 + x^225 + x^224 + x^222 + x^218 + x^217 + x^215 + x^212 + x^210 + x^208 + x^206 + x^205 + x^203 + x^201 + x^200 + x^194 + x^193 + x^191 + x^189 + x^183 + x^181 + x^180 + x^178 + x^177 + x^176 + x^174 + x^172 + x^171 + x^169 + x^167 + x^166 + x^163 + x^162 + x^159 + x^157 + x^155 + x^151 + x^150 + x^149 + x^147 + x^140 + x^139 + x^138 + x^137 + x^136 + x^134 + x^131 + x^128 + x^127 + x^126 + x^121 + x^119 + x^118 + x^117 + x^111 + x^106 + x^104 + x^102 + x^101 + x^99 + x^98 + x^91 + x^90 + x^88 + x^86 + x^85 + x^81 + x^79 + x^75 + x^74 + x^73 + x^72 + x^71 + x^69 + x^68 + x^65 + x^64 + x^63 + x^62 + x^61 + x^58 + x^57 + x^56 + x^55 + x^51 + x^50 + x^49 + x^41 + x^38 + x^36 + x^34 + x^33 + x^32 + x^24 + x^21 + x^9 + x^6 + 1 + +10-38-9 285 x^928 + x^898 + x^870 + x^841 + x^840 + x^838 + x^836 + x^811 + x^810 + x^781 + x^777 + x^774 + x^751 + x^748 + x^723 + x^721 + x^720 + x^719 + x^718 + x^717 + x^716 + x^714 + x^712 + x^694 + x^692 + x^691 + x^688 + x^686 + x^664 + x^663 + x^661 + x^660 + x^659 + x^657 + x^652 + x^650 + x^632 + x^631 + x^630 + x^628 + x^627 + x^625 + x^624 + x^623 + x^605 + x^604 + x^603 + x^599 + x^597 + x^594 + x^593 + x^591 + x^590 + x^588 + x^576 + x^575 + x^574 + x^572 + x^571 + x^570 + x^569 + x^567 + x^565 + x^560 + x^544 + x^543 + x^542 + x^540 + x^539 + x^537 + x^535 + x^531 + x^526 + x^514 + x^510 + x^509 + x^507 + x^506 + x^505 + x^504 + x^503 + x^502 + x^498 + x^485 + x^484 + x^478 + x^474 + x^471 + x^470 + x^468 + x^466 + x^464 + x^458 + x^455 + x^452 + x^450 + x^449 + x^448 + x^445 + x^441 + x^439 + x^428 + x^419 + x^416 + x^415 + x^411 + x^407 + x^406 + x^405 + x^404 + x^402 + x^398 + x^392 + x^388 + x^385 + x^384 + x^383 + x^382 + x^381 + x^380 + x^379 + x^376 + x^374 + x^364 + x^359 + x^357 + x^356 + x^355 + x^354 + x^353 + x^351 + x^349 + x^346 + x^345 + x^343 + x^340 + x^338 + x^334 + x^328 + x^326 + x^325 + x^324 + x^321 + x^320 + x^319 + x^318 + x^315 + x^314 + x^312 + x^306 + x^298 + x^297 + x^295 + x^294 + x^293 + x^291 + x^288 + x^287 + x^286 + x^285 + x^283 + x^282 + x^281 + x^278 + x^276 + x^274 + x^270 + x^269 + x^268 + x^263 + x^261 + x^256 + x^255 + x^254 + x^244 + x^242 + x^237 + x^236 + x^235 + x^234 + x^233 + x^232 + x^229 + x^225 + x^222 + x^219 + x^216 + x^214 + x^212 + x^210 + x^209 + x^208 + x^207 + x^206 + x^202 + x^198 + x^194 + x^193 + x^192 + x^191 + x^190 + x^184 + x^182 + x^178 + x^176 + x^175 + x^174 + x^173 + x^168 + x^166 + x^164 + x^160 + x^159 + x^157 + x^154 + x^152 + x^151 + x^150 + x^145 + x^144 + x^142 + x^136 + x^135 + x^134 + x^129 + x^128 + x^122 + x^121 + x^119 + x^118 + x^116 + x^114 + x^113 + x^112 + x^111 + x^108 + x^105 + x^103 + x^100 + x^98 + x^97 + x^95 + x^94 + x^92 + x^89 + x^88 + x^87 + x^85 + x^84 + x^83 + x^78 + x^76 + x^74 + x^73 + x^71 + x^70 + x^69 + x^67 + x^66 + x^64 + x^61 + x^59 + x^57 + x^56 + x^54 + x^53 + x^51 + x^49 + x^48 + x^46 + x^45 + x^43 + x^42 + x^41 + x^40 + x^38 + x^37 + x^36 + x^35 + x^34 + x^33 + x^32 + x^30 + x^4 + 1 + +15-22-28 289 x^928 + x^898 + x^870 + x^838 + x^821 + x^806 + x^805 + x^792 + x^790 + x^778 + x^774 + x^762 + x^745 + x^744 + x^718 + x^697 + x^682 + x^672 + x^670 + x^667 + x^666 + x^658 + x^656 + x^653 + x^652 + x^651 + x^650 + x^642 + x^641 + x^638 + x^637 + x^623 + x^608 + x^607 + x^598 + x^594 + x^588 + x^575 + x^574 + x^573 + x^566 + x^565 + x^564 + x^563 + x^562 + x^557 + x^548 + x^547 + x^542 + x^538 + x^533 + x^531 + x^529 + x^526 + x^517 + x^516 + x^514 + x^513 + x^505 + x^504 + x^503 + x^502 + x^501 + x^499 + x^498 + x^497 + x^488 + x^487 + x^484 + x^483 + x^482 + x^474 + x^470 + x^468 + x^467 + x^466 + x^464 + x^457 + x^454 + x^453 + x^449 + x^447 + x^444 + x^443 + x^442 + x^441 + x^440 + x^439 + x^437 + x^436 + x^435 + x^428 + x^427 + x^426 + x^425 + x^422 + x^418 + x^417 + x^414 + x^409 + x^406 + x^405 + x^392 + x^391 + x^389 + x^388 + x^387 + x^386 + x^379 + x^378 + x^374 + x^372 + x^371 + x^368 + x^367 + x^362 + x^361 + x^358 + x^357 + x^354 + x^350 + x^349 + x^347 + x^342 + x^341 + x^340 + x^334 + x^333 + x^332 + x^331 + x^330 + x^329 + x^328 + x^327 + x^324 + x^323 + x^320 + x^319 + x^313 + x^308 + x^307 + x^305 + x^303 + x^302 + x^301 + x^300 + x^299 + x^298 + x^297 + x^296 + x^295 + x^294 + x^293 + x^292 + x^288 + x^287 + x^286 + x^283 + x^282 + x^280 + x^279 + x^278 + x^277 + x^276 + x^271 + x^269 + x^268 + x^267 + x^266 + x^265 + x^255 + x^254 + x^253 + x^249 + x^247 + x^241 + x^238 + x^237 + x^236 + x^235 + x^226 + x^225 + x^224 + x^223 + x^220 + x^219 + x^215 + x^214 + x^212 + x^210 + x^209 + x^205 + x^202 + x^201 + x^200 + x^199 + x^197 + x^190 + x^186 + x^182 + x^181 + x^180 + x^177 + x^174 + x^173 + x^172 + x^171 + x^170 + x^169 + x^168 + x^165 + x^163 + x^161 + x^160 + x^158 + x^155 + x^153 + x^151 + x^150 + x^146 + x^143 + x^142 + x^137 + x^136 + x^135 + x^131 + x^130 + x^128 + x^127 + x^126 + x^125 + x^124 + x^123 + x^122 + x^120 + x^119 + x^117 + x^114 + x^113 + x^111 + x^105 + x^104 + x^103 + x^102 + x^101 + x^99 + x^97 + x^93 + x^90 + x^89 + x^88 + x^87 + x^86 + x^84 + x^83 + x^82 + x^81 + x^80 + x^79 + x^77 + x^75 + x^72 + x^71 + x^70 + x^69 + x^68 + x^65 + x^60 + x^59 + x^58 + x^55 + x^54 + x^52 + x^51 + x^50 + x^48 + x^43 + x^42 + x^40 + x^39 + x^30 + x^28 + x^27 + x^24 + x^23 + x^19 + x^16 + x^15 + 1 + +12-41-35 291 x^928 + x^898 + x^886 + x^873 + x^870 + x^860 + x^856 + x^847 + x^844 + x^834 + x^830 + x^826 + x^818 + x^817 + x^805 + x^804 + x^796 + x^795 + x^791 + x^784 + x^782 + x^779 + x^769 + x^765 + x^762 + x^758 + x^756 + x^753 + x^752 + x^743 + x^739 + x^736 + x^735 + x^732 + x^731 + x^730 + x^727 + x^726 + x^723 + x^722 + x^717 + x^713 + x^710 + x^705 + x^701 + x^700 + x^692 + x^685 + x^683 + x^680 + x^676 + x^671 + x^663 + x^659 + x^657 + x^654 + x^653 + x^645 + x^642 + x^637 + x^632 + x^628 + x^624 + x^623 + x^619 + x^616 + x^615 + x^612 + x^610 + x^607 + x^606 + x^603 + x^598 + x^597 + x^594 + x^593 + x^590 + x^589 + x^585 + x^580 + x^577 + x^576 + x^572 + x^567 + x^564 + x^560 + x^559 + x^556 + x^550 + x^545 + x^543 + x^538 + x^537 + x^534 + x^533 + x^532 + x^528 + x^520 + x^516 + x^513 + x^508 + x^507 + x^506 + x^504 + x^499 + x^496 + x^491 + x^490 + x^487 + x^486 + x^485 + x^481 + x^477 + x^474 + x^466 + x^465 + x^457 + x^452 + x^448 + x^447 + x^446 + x^438 + x^436 + x^435 + x^434 + x^430 + x^425 + x^422 + x^421 + x^417 + x^413 + x^409 + x^404 + x^395 + x^393 + x^392 + x^388 + x^387 + x^386 + x^384 + x^382 + x^380 + x^379 + x^374 + x^373 + x^370 + x^367 + x^366 + x^362 + x^361 + x^358 + x^357 + x^354 + x^353 + x^352 + x^350 + x^349 + x^345 + x^344 + x^343 + x^340 + x^337 + x^336 + x^332 + x^331 + x^328 + x^324 + x^323 + x^317 + x^315 + x^314 + x^313 + x^311 + x^305 + x^302 + x^298 + x^297 + x^296 + x^295 + x^292 + x^291 + x^288 + x^283 + x^280 + x^276 + x^275 + x^272 + x^268 + x^266 + x^262 + x^259 + x^258 + x^255 + x^251 + x^250 + x^242 + x^241 + x^240 + x^232 + x^227 + x^225 + x^224 + x^223 + x^216 + x^214 + x^212 + x^211 + x^210 + x^206 + x^203 + x^198 + x^189 + x^188 + x^177 + x^176 + x^175 + x^173 + x^168 + x^164 + x^162 + x^160 + x^159 + x^156 + x^155 + x^154 + x^152 + x^149 + x^147 + x^146 + x^145 + x^142 + x^139 + x^138 + x^137 + x^134 + x^132 + x^130 + x^124 + x^123 + x^122 + x^115 + x^113 + x^112 + x^111 + x^110 + x^108 + x^106 + x^103 + x^102 + x^98 + x^97 + x^96 + x^95 + x^94 + x^92 + x^91 + x^90 + x^88 + x^85 + x^80 + x^78 + x^77 + x^76 + x^73 + x^71 + x^70 + x^69 + x^68 + x^67 + x^65 + x^64 + x^63 + x^54 + x^53 + x^52 + x^51 + x^50 + x^49 + x^48 + x^47 + x^44 + x^43 + x^42 + x^40 + x^39 + x^31 + x^30 + x^29 + x^21 + x^13 + 1 + +35-54-56 293 x^928 + x^898 + x^870 + x^862 + x^832 + x^826 + x^808 + x^796 + x^794 + x^790 + x^778 + x^765 + x^758 + x^754 + x^734 + x^729 + x^728 + x^722 + x^718 + x^715 + x^706 + x^700 + x^699 + x^698 + x^693 + x^686 + x^682 + x^676 + x^673 + x^672 + x^669 + x^668 + x^662 + x^658 + x^656 + x^655 + x^652 + x^650 + x^649 + x^646 + x^645 + x^642 + x^639 + x^638 + x^636 + x^630 + x^622 + x^614 + x^613 + x^612 + x^610 + x^608 + x^607 + x^600 + x^596 + x^595 + x^592 + x^590 + x^589 + x^587 + x^582 + x^574 + x^571 + x^568 + x^566 + x^560 + x^558 + x^554 + x^553 + x^552 + x^548 + x^544 + x^542 + x^538 + x^536 + x^534 + x^530 + x^528 + x^527 + x^524 + x^523 + x^522 + x^520 + x^518 + x^516 + x^515 + x^511 + x^506 + x^499 + x^497 + x^493 + x^492 + x^490 + x^487 + x^482 + x^478 + x^474 + x^467 + x^462 + x^461 + x^452 + x^445 + x^442 + x^439 + x^437 + x^436 + x^433 + x^428 + x^427 + x^422 + x^418 + x^413 + x^412 + x^410 + x^409 + x^407 + x^406 + x^403 + x^401 + x^398 + x^396 + x^392 + x^390 + x^386 + x^384 + x^383 + x^382 + x^380 + x^378 + x^377 + x^373 + x^372 + x^371 + x^366 + x^365 + x^362 + x^353 + x^352 + x^350 + x^348 + x^346 + x^344 + x^342 + x^340 + x^338 + x^336 + x^335 + x^334 + x^332 + x^331 + x^328 + x^326 + x^320 + x^318 + x^317 + x^316 + x^313 + x^312 + x^311 + x^310 + x^308 + x^304 + x^300 + x^299 + x^298 + x^296 + x^295 + x^293 + x^290 + x^288 + x^284 + x^282 + x^275 + x^271 + x^269 + x^268 + x^266 + x^265 + x^264 + x^263 + x^262 + x^260 + x^259 + x^253 + x^250 + x^247 + x^246 + x^244 + x^242 + x^240 + x^239 + x^238 + x^235 + x^233 + x^232 + x^228 + x^227 + x^226 + x^224 + x^221 + x^220 + x^218 + x^216 + x^214 + x^212 + x^208 + x^203 + x^199 + x^198 + x^196 + x^195 + x^194 + x^193 + x^192 + x^189 + x^187 + x^182 + x^178 + x^176 + x^175 + x^173 + x^168 + x^167 + x^166 + x^163 + x^161 + x^160 + x^156 + x^152 + x^150 + x^148 + x^146 + x^140 + x^138 + x^135 + x^133 + x^131 + x^130 + x^129 + x^128 + x^127 + x^126 + x^125 + x^123 + x^122 + x^120 + x^118 + x^116 + x^115 + x^113 + x^112 + x^109 + x^106 + x^105 + x^104 + x^103 + x^101 + x^100 + x^95 + x^94 + x^92 + x^84 + x^82 + x^80 + x^79 + x^77 + x^76 + x^74 + x^71 + x^69 + x^68 + x^66 + x^65 + x^64 + x^63 + x^55 + x^54 + x^53 + x^49 + x^48 + x^46 + x^45 + x^43 + x^42 + x^41 + x^40 + x^38 + x^36 + x^34 + x^24 + x^18 + x^6 + 1 + +44-42-5 307 x^928 + x^898 + x^880 + x^870 + x^856 + x^850 + x^839 + x^832 + x^826 + x^820 + x^816 + x^815 + x^808 + x^792 + x^791 + x^790 + x^784 + x^779 + x^778 + x^768 + x^762 + x^760 + x^756 + x^754 + x^753 + x^747 + x^744 + x^738 + x^737 + x^736 + x^731 + x^726 + x^723 + x^720 + x^719 + x^717 + x^713 + x^707 + x^696 + x^693 + x^689 + x^688 + x^682 + x^678 + x^677 + x^670 + x^666 + x^665 + x^663 + x^660 + x^658 + x^657 + x^653 + x^646 + x^633 + x^630 + x^628 + x^624 + x^617 + x^616 + x^612 + x^611 + x^606 + x^605 + x^604 + x^603 + x^600 + x^599 + x^597 + x^594 + x^592 + x^588 + x^587 + x^586 + x^582 + x^580 + x^576 + x^575 + x^568 + x^563 + x^556 + x^552 + x^546 + x^544 + x^542 + x^540 + x^539 + x^537 + x^534 + x^533 + x^526 + x^522 + x^521 + x^520 + x^518 + x^517 + x^516 + x^514 + x^511 + x^510 + x^508 + x^505 + x^503 + x^494 + x^492 + x^488 + x^485 + x^481 + x^475 + x^474 + x^470 + x^467 + x^464 + x^463 + x^462 + x^454 + x^452 + x^449 + x^447 + x^446 + x^444 + x^443 + x^442 + x^439 + x^438 + x^437 + x^432 + x^431 + x^430 + x^427 + x^426 + x^424 + x^422 + x^419 + x^418 + x^413 + x^410 + x^409 + x^406 + x^404 + x^401 + x^398 + x^396 + x^393 + x^392 + x^391 + x^390 + x^389 + x^385 + x^383 + x^380 + x^379 + x^376 + x^374 + x^372 + x^370 + x^368 + x^367 + x^366 + x^365 + x^364 + x^363 + x^361 + x^358 + x^355 + x^354 + x^353 + x^349 + x^348 + x^347 + x^344 + x^343 + x^342 + x^341 + x^338 + x^337 + x^336 + x^334 + x^332 + x^329 + x^328 + x^318 + x^317 + x^312 + x^310 + x^308 + x^307 + x^306 + x^305 + x^304 + x^299 + x^296 + x^294 + x^293 + x^291 + x^289 + x^281 + x^280 + x^278 + x^277 + x^276 + x^274 + x^272 + x^270 + x^267 + x^265 + x^264 + x^263 + x^260 + x^253 + x^252 + x^251 + x^250 + x^247 + x^242 + x^241 + x^239 + x^234 + x^233 + x^231 + x^230 + x^229 + x^224 + x^223 + x^222 + x^217 + x^214 + x^212 + x^210 + x^209 + x^208 + x^203 + x^200 + x^199 + x^198 + x^196 + x^195 + x^192 + x^191 + x^186 + x^185 + x^184 + x^181 + x^180 + x^179 + x^178 + x^177 + x^176 + x^175 + x^172 + x^171 + x^170 + x^169 + x^168 + x^166 + x^165 + x^161 + x^160 + x^159 + x^151 + x^146 + x^144 + x^143 + x^142 + x^141 + x^139 + x^137 + x^136 + x^135 + x^133 + x^131 + x^130 + x^120 + x^116 + x^114 + x^113 + x^108 + x^107 + x^106 + x^104 + x^101 + x^98 + x^97 + x^96 + x^91 + x^90 + x^89 + x^87 + x^82 + x^80 + x^78 + x^73 + x^72 + x^67 + x^62 + x^58 + x^56 + x^55 + x^53 + x^49 + x^46 + x^44 + x^39 + x^32 + x^26 + x^17 + 1 + +18-7-29 311 x^928 + x^926 + x^920 + x^918 + x^912 + x^910 + x^904 + x^902 + x^898 + x^896 + x^894 + x^890 + x^888 + x^886 + x^882 + x^880 + x^878 + x^874 + x^872 + x^870 + x^866 + x^858 + x^850 + x^842 + x^808 + x^806 + x^792 + x^790 + x^778 + x^776 + x^774 + x^762 + x^760 + x^758 + x^746 + x^730 + x^688 + x^686 + x^680 + x^678 + x^669 + x^667 + x^661 + x^659 + x^658 + x^656 + x^655 + x^654 + x^651 + x^650 + x^648 + x^647 + x^646 + x^643 + x^626 + x^618 + x^611 + x^605 + x^595 + x^589 + x^587 + x^575 + x^569 + x^568 + x^566 + x^561 + x^553 + x^541 + x^539 + x^538 + x^537 + x^536 + x^534 + x^533 + x^515 + x^511 + x^509 + x^507 + x^506 + x^503 + x^501 + x^497 + x^495 + x^493 + x^489 + x^485 + x^483 + x^481 + x^479 + x^475 + x^473 + x^471 + x^469 + x^457 + x^455 + x^453 + x^451 + x^449 + x^448 + x^447 + x^446 + x^443 + x^440 + x^439 + x^438 + x^437 + x^435 + x^433 + x^432 + x^431 + x^429 + x^426 + x^425 + x^424 + x^421 + x^418 + x^417 + x^416 + x^415 + x^414 + x^412 + x^409 + x^408 + x^406 + x^405 + x^404 + x^401 + x^398 + x^396 + x^395 + x^393 + x^392 + x^390 + x^388 + x^387 + x^380 + x^379 + x^377 + x^376 + x^372 + x^371 + x^370 + x^369 + x^367 + x^364 + x^363 + x^362 + x^361 + x^359 + x^358 + x^354 + x^352 + x^350 + x^349 + x^348 + x^341 + x^340 + x^337 + x^336 + x^335 + x^331 + x^330 + x^328 + x^326 + x^320 + x^319 + x^318 + x^317 + x^315 + x^314 + x^313 + x^312 + x^310 + x^308 + x^307 + x^302 + x^301 + x^299 + x^297 + x^295 + x^293 + x^291 + x^287 + x^284 + x^283 + x^282 + x^281 + x^280 + x^279 + x^278 + x^274 + x^271 + x^267 + x^266 + x^264 + x^263 + x^257 + x^256 + x^255 + x^252 + x^251 + x^250 + x^246 + x^241 + x^240 + x^239 + x^234 + x^233 + x^231 + x^230 + x^229 + x^226 + x^225 + x^224 + x^223 + x^222 + x^218 + x^216 + x^214 + x^212 + x^211 + x^206 + x^203 + x^202 + x^199 + x^198 + x^197 + x^193 + x^191 + x^190 + x^188 + x^185 + x^183 + x^181 + x^180 + x^179 + x^176 + x^171 + x^169 + x^167 + x^158 + x^156 + x^155 + x^154 + x^153 + x^151 + x^150 + x^148 + x^145 + x^143 + x^142 + x^141 + x^138 + x^133 + x^132 + x^130 + x^127 + x^125 + x^123 + x^121 + x^118 + x^109 + x^108 + x^105 + x^104 + x^103 + x^102 + x^100 + x^97 + x^95 + x^94 + x^92 + x^90 + x^85 + x^84 + x^82 + x^80 + x^76 + x^74 + x^73 + x^71 + x^70 + x^68 + x^67 + x^66 + x^65 + x^64 + x^61 + x^59 + x^57 + x^55 + x^53 + x^52 + x^48 + x^47 + x^45 + x^41 + x^38 + x^36 + x^34 + x^31 + x^28 + x^26 + x^24 + x^22 + x^18 + x^16 + x^12 + x^10 + x^2 + 1 + +48-55-49 313 x^928 + x^898 + x^894 + x^893 + x^870 + x^864 + x^860 + x^858 + x^838 + x^836 + x^832 + x^831 + x^830 + x^828 + x^804 + x^802 + x^796 + x^794 + x^778 + x^774 + x^773 + x^770 + x^769 + x^768 + x^741 + x^740 + x^737 + x^736 + x^734 + x^732 + x^731 + x^718 + x^712 + x^711 + x^706 + x^704 + x^702 + x^701 + x^684 + x^681 + x^679 + x^678 + x^676 + x^675 + x^672 + x^671 + x^669 + x^658 + x^656 + x^654 + x^653 + x^652 + x^651 + x^647 + x^646 + x^645 + x^621 + x^618 + x^617 + x^616 + x^615 + x^614 + x^611 + x^610 + x^609 + x^598 + x^596 + x^592 + x^590 + x^589 + x^583 + x^582 + x^580 + x^579 + x^576 + x^561 + x^559 + x^558 + x^557 + x^554 + x^548 + x^547 + x^545 + x^538 + x^534 + x^533 + x^531 + x^529 + x^527 + x^526 + x^524 + x^522 + x^521 + x^515 + x^512 + x^500 + x^497 + x^486 + x^484 + x^478 + x^474 + x^470 + x^469 + x^468 + x^467 + x^466 + x^465 + x^464 + x^462 + x^460 + x^458 + x^456 + x^454 + x^453 + x^450 + x^444 + x^438 + x^436 + x^435 + x^434 + x^433 + x^432 + x^429 + x^424 + x^422 + x^421 + x^418 + x^416 + x^413 + x^412 + x^407 + x^406 + x^405 + x^404 + x^402 + x^399 + x^398 + x^397 + x^396 + x^394 + x^392 + x^391 + x^388 + x^384 + x^380 + x^378 + x^377 + x^375 + x^373 + x^372 + x^371 + x^364 + x^359 + x^356 + x^354 + x^353 + x^352 + x^351 + x^350 + x^347 + x^343 + x^342 + x^340 + x^339 + x^334 + x^331 + x^328 + x^327 + x^325 + x^324 + x^322 + x^316 + x^312 + x^311 + x^310 + x^308 + x^306 + x^305 + x^304 + x^302 + x^301 + x^300 + x^298 + x^297 + x^295 + x^294 + x^292 + x^291 + x^290 + x^289 + x^288 + x^282 + x^280 + x^279 + x^278 + x^277 + x^276 + x^275 + x^273 + x^271 + x^268 + x^264 + x^261 + x^258 + x^256 + x^254 + x^253 + x^251 + x^249 + x^248 + x^246 + x^245 + x^244 + x^242 + x^241 + x^240 + x^239 + x^237 + x^236 + x^232 + x^231 + x^229 + x^228 + x^227 + x^226 + x^224 + x^223 + x^219 + x^216 + x^215 + x^211 + x^210 + x^207 + x^206 + x^204 + x^203 + x^199 + x^198 + x^194 + x^188 + x^187 + x^186 + x^182 + x^181 + x^180 + x^174 + x^171 + x^170 + x^168 + x^165 + x^164 + x^160 + x^152 + x^151 + x^150 + x^149 + x^148 + x^147 + x^144 + x^143 + x^142 + x^141 + x^139 + x^134 + x^129 + x^127 + x^124 + x^121 + x^119 + x^115 + x^108 + x^107 + x^105 + x^104 + x^103 + x^102 + x^95 + x^94 + x^90 + x^89 + x^88 + x^86 + x^84 + x^79 + x^77 + x^73 + x^72 + x^69 + x^66 + x^64 + x^63 + x^62 + x^60 + x^57 + x^56 + x^55 + x^54 + x^53 + x^52 + x^50 + x^49 + x^48 + x^46 + x^45 + x^41 + x^36 + x^34 + x^31 + x^29 + x^28 + x^26 + x^25 + 1 + +40-44-57 319 x^928 + x^898 + x^870 + x^846 + x^822 + x^819 + x^808 + x^800 + x^794 + x^792 + x^786 + x^778 + x^770 + x^764 + x^759 + x^748 + x^746 + x^740 + x^732 + x^716 + x^713 + x^697 + x^696 + x^691 + x^686 + x^674 + x^664 + x^661 + x^658 + x^650 + x^648 + x^645 + x^644 + x^636 + x^629 + x^626 + x^624 + x^623 + x^621 + x^620 + x^618 + x^615 + x^594 + x^593 + x^591 + x^590 + x^586 + x^583 + x^580 + x^579 + x^577 + x^575 + x^574 + x^572 + x^571 + x^568 + x^564 + x^563 + x^561 + x^558 + x^556 + x^555 + x^553 + x^550 + x^547 + x^545 + x^544 + x^542 + x^541 + x^538 + x^536 + x^533 + x^528 + x^526 + x^525 + x^522 + x^519 + x^513 + x^512 + x^510 + x^509 + x^508 + x^507 + x^504 + x^503 + x^501 + x^495 + x^488 + x^487 + x^485 + x^484 + x^483 + x^482 + x^480 + x^479 + x^474 + x^473 + x^471 + x^470 + x^466 + x^463 + x^460 + x^458 + x^457 + x^454 + x^453 + x^450 + x^449 + x^446 + x^444 + x^443 + x^441 + x^440 + x^439 + x^436 + x^434 + x^433 + x^432 + x^431 + x^430 + x^427 + x^425 + x^424 + x^422 + x^420 + x^417 + x^416 + x^415 + x^413 + x^412 + x^411 + x^410 + x^409 + x^408 + x^407 + x^406 + x^405 + x^403 + x^398 + x^397 + x^395 + x^392 + x^390 + x^387 + x^386 + x^385 + x^383 + x^375 + x^370 + x^369 + x^368 + x^367 + x^365 + x^364 + x^363 + x^360 + x^359 + x^357 + x^356 + x^353 + x^351 + x^349 + x^342 + x^340 + x^339 + x^336 + x^335 + x^329 + x^327 + x^326 + x^324 + x^323 + x^320 + x^319 + x^318 + x^317 + x^316 + x^315 + x^312 + x^311 + x^307 + x^306 + x^302 + x^299 + x^298 + x^297 + x^295 + x^294 + x^293 + x^291 + x^290 + x^287 + x^285 + x^284 + x^283 + x^282 + x^281 + x^279 + x^277 + x^276 + x^275 + x^272 + x^271 + x^268 + x^266 + x^263 + x^262 + x^261 + x^255 + x^251 + x^249 + x^248 + x^245 + x^244 + x^243 + x^240 + x^239 + x^238 + x^236 + x^235 + x^234 + x^230 + x^229 + x^228 + x^227 + x^226 + x^224 + x^221 + x^217 + x^214 + x^212 + x^209 + x^208 + x^204 + x^202 + x^201 + x^197 + x^194 + x^193 + x^192 + x^191 + x^189 + x^188 + x^185 + x^184 + x^182 + x^181 + x^178 + x^176 + x^173 + x^172 + x^171 + x^169 + x^166 + x^163 + x^161 + x^160 + x^158 + x^156 + x^155 + x^154 + x^152 + x^151 + x^149 + x^148 + x^146 + x^145 + x^144 + x^142 + x^141 + x^134 + x^132 + x^129 + x^122 + x^121 + x^119 + x^117 + x^116 + x^113 + x^112 + x^110 + x^109 + x^103 + x^102 + x^101 + x^100 + x^97 + x^95 + x^94 + x^93 + x^92 + x^91 + x^87 + x^86 + x^85 + x^84 + x^81 + x^80 + x^75 + x^74 + x^71 + x^70 + x^68 + x^66 + x^63 + x^52 + x^51 + x^50 + x^47 + x^45 + x^44 + x^43 + x^39 + x^38 + x^34 + x^24 + x^16 + x^8 + 1 + +21-52-12 321 x^928 + x^898 + x^870 + x^868 + x^856 + x^838 + x^828 + x^826 + x^816 + x^815 + x^808 + x^804 + x^786 + x^784 + x^778 + x^748 + x^744 + x^736 + x^734 + x^733 + x^732 + x^731 + x^726 + x^722 + x^718 + x^710 + x^706 + x^704 + x^703 + x^694 + x^692 + x^691 + x^688 + x^683 + x^680 + x^679 + x^678 + x^674 + x^672 + x^668 + x^664 + x^660 + x^658 + x^656 + x^653 + x^649 + x^648 + x^644 + x^643 + x^638 + x^632 + x^630 + x^625 + x^623 + x^619 + x^618 + x^616 + x^615 + x^613 + x^610 + x^607 + x^606 + x^602 + x^601 + x^600 + x^595 + x^593 + x^589 + x^588 + x^586 + x^585 + x^580 + x^579 + x^578 + x^571 + x^570 + x^566 + x^565 + x^564 + x^563 + x^560 + x^559 + x^555 + x^554 + x^550 + x^548 + x^547 + x^546 + x^544 + x^543 + x^541 + x^539 + x^538 + x^537 + x^535 + x^531 + x^530 + x^529 + x^522 + x^520 + x^514 + x^513 + x^509 + x^507 + x^504 + x^501 + x^500 + x^499 + x^498 + x^496 + x^495 + x^494 + x^492 + x^491 + x^490 + x^486 + x^483 + x^480 + x^479 + x^476 + x^473 + x^469 + x^466 + x^463 + x^460 + x^459 + x^457 + x^451 + x^449 + x^443 + x^441 + x^440 + x^436 + x^433 + x^432 + x^431 + x^424 + x^420 + x^418 + x^416 + x^414 + x^411 + x^410 + x^409 + x^408 + x^406 + x^404 + x^403 + x^401 + x^400 + x^398 + x^393 + x^390 + x^388 + x^385 + x^384 + x^381 + x^375 + x^372 + x^366 + x^365 + x^361 + x^358 + x^356 + x^354 + x^352 + x^347 + x^346 + x^341 + x^340 + x^338 + x^335 + x^334 + x^333 + x^331 + x^330 + x^326 + x^323 + x^320 + x^319 + x^318 + x^317 + x^315 + x^312 + x^311 + x^309 + x^305 + x^303 + x^301 + x^299 + x^297 + x^296 + x^295 + x^290 + x^289 + x^283 + x^282 + x^277 + x^276 + x^273 + x^268 + x^267 + x^266 + x^264 + x^262 + x^261 + x^260 + x^256 + x^254 + x^251 + x^250 + x^247 + x^245 + x^244 + x^243 + x^240 + x^238 + x^235 + x^234 + x^233 + x^231 + x^229 + x^225 + x^224 + x^217 + x^212 + x^211 + x^209 + x^208 + x^205 + x^204 + x^202 + x^200 + x^198 + x^196 + x^195 + x^191 + x^190 + x^188 + x^187 + x^186 + x^182 + x^181 + x^180 + x^175 + x^174 + x^172 + x^171 + x^170 + x^169 + x^166 + x^164 + x^158 + x^154 + x^153 + x^152 + x^149 + x^148 + x^147 + x^146 + x^144 + x^142 + x^141 + x^140 + x^138 + x^137 + x^136 + x^135 + x^133 + x^131 + x^127 + x^125 + x^123 + x^122 + x^121 + x^120 + x^119 + x^115 + x^113 + x^112 + x^111 + x^106 + x^102 + x^101 + x^100 + x^98 + x^96 + x^95 + x^93 + x^92 + x^91 + x^90 + x^89 + x^88 + x^87 + x^81 + x^79 + x^76 + x^72 + x^71 + x^70 + x^64 + x^60 + x^57 + x^56 + x^55 + x^54 + x^50 + x^49 + x^48 + x^47 + x^43 + x^39 + x^38 + x^30 + x^29 + x^26 + x^25 + x^14 + x^10 + 1 + +25-44-4 331 x^928 + x^898 + x^870 + x^866 + x^862 + x^859 + x^836 + x^834 + x^829 + x^822 + x^808 + x^806 + x^804 + x^800 + x^799 + x^798 + x^797 + x^795 + x^778 + x^776 + x^770 + x^769 + x^767 + x^762 + x^753 + x^739 + x^735 + x^734 + x^731 + x^721 + x^713 + x^709 + x^708 + x^706 + x^705 + x^703 + x^701 + x^698 + x^688 + x^679 + x^677 + x^676 + x^674 + x^670 + x^669 + x^667 + x^666 + x^661 + x^659 + x^658 + x^649 + x^647 + x^645 + x^643 + x^642 + x^638 + x^635 + x^633 + x^629 + x^626 + x^624 + x^620 + x^618 + x^617 + x^614 + x^611 + x^605 + x^603 + x^598 + x^596 + x^594 + x^593 + x^590 + x^587 + x^581 + x^580 + x^579 + x^571 + x^570 + x^568 + x^566 + x^562 + x^558 + x^557 + x^551 + x^550 + x^548 + x^547 + x^546 + x^544 + x^542 + x^541 + x^539 + x^536 + x^533 + x^532 + x^530 + x^527 + x^526 + x^525 + x^524 + x^523 + x^521 + x^520 + x^519 + x^518 + x^517 + x^515 + x^514 + x^513 + x^507 + x^506 + x^504 + x^501 + x^497 + x^493 + x^492 + x^490 + x^488 + x^483 + x^482 + x^481 + x^478 + x^477 + x^476 + x^475 + x^473 + x^467 + x^466 + x^465 + x^463 + x^462 + x^461 + x^460 + x^456 + x^454 + x^453 + x^452 + x^450 + x^448 + x^446 + x^444 + x^443 + x^442 + x^433 + x^426 + x^423 + x^422 + x^420 + x^419 + x^416 + x^413 + x^412 + x^411 + x^410 + x^406 + x^404 + x^403 + x^402 + x^401 + x^400 + x^398 + x^396 + x^395 + x^392 + x^391 + x^389 + x^387 + x^386 + x^385 + x^380 + x^372 + x^366 + x^365 + x^363 + x^362 + x^359 + x^357 + x^354 + x^353 + x^352 + x^348 + x^346 + x^345 + x^342 + x^340 + x^338 + x^335 + x^334 + x^333 + x^332 + x^325 + x^324 + x^323 + x^320 + x^319 + x^318 + x^314 + x^312 + x^309 + x^308 + x^303 + x^302 + x^301 + x^299 + x^295 + x^294 + x^292 + x^291 + x^290 + x^289 + x^286 + x^284 + x^279 + x^278 + x^274 + x^273 + x^268 + x^266 + x^265 + x^263 + x^261 + x^257 + x^256 + x^255 + x^254 + x^253 + x^248 + x^245 + x^244 + x^241 + x^240 + x^237 + x^235 + x^234 + x^233 + x^232 + x^230 + x^229 + x^228 + x^226 + x^223 + x^222 + x^220 + x^218 + x^214 + x^212 + x^211 + x^209 + x^207 + x^206 + x^203 + x^202 + x^201 + x^200 + x^199 + x^198 + x^196 + x^191 + x^189 + x^184 + x^178 + x^177 + x^176 + x^175 + x^172 + x^171 + x^169 + x^166 + x^165 + x^162 + x^161 + x^159 + x^157 + x^156 + x^155 + x^150 + x^149 + x^148 + x^147 + x^143 + x^142 + x^141 + x^140 + x^137 + x^136 + x^135 + x^133 + x^131 + x^130 + x^129 + x^127 + x^126 + x^125 + x^124 + x^123 + x^120 + x^113 + x^112 + x^108 + x^105 + x^104 + x^98 + x^96 + x^92 + x^90 + x^82 + x^81 + x^80 + x^78 + x^76 + x^74 + x^73 + x^68 + x^66 + x^64 + x^62 + x^60 + x^58 + x^54 + x^42 + x^38 + x^36 + x^32 + x^26 + x^18 + x^16 + x^10 + x^8 + x^2 + 1 + +16-8-37 341 x^928 + x^898 + x^870 + x^842 + x^814 + x^808 + x^788 + x^786 + x^784 + x^778 + x^777 + x^775 + x^773 + x^762 + x^756 + x^751 + x^745 + x^736 + x^723 + x^721 + x^717 + x^715 + x^713 + x^710 + x^700 + x^697 + x^693 + x^691 + x^688 + x^685 + x^678 + x^674 + x^673 + x^672 + x^667 + x^654 + x^650 + x^647 + x^645 + x^644 + x^643 + x^642 + x^641 + x^635 + x^634 + x^632 + x^631 + x^630 + x^622 + x^621 + x^618 + x^615 + x^614 + x^612 + x^608 + x^605 + x^604 + x^603 + x^601 + x^596 + x^595 + x^592 + x^590 + x^589 + x^588 + x^587 + x^586 + x^585 + x^584 + x^583 + x^581 + x^579 + x^578 + x^576 + x^575 + x^573 + x^571 + x^569 + x^565 + x^564 + x^561 + x^560 + x^557 + x^556 + x^555 + x^554 + x^552 + x^544 + x^543 + x^540 + x^535 + x^533 + x^530 + x^528 + x^527 + x^524 + x^522 + x^520 + x^519 + x^517 + x^516 + x^515 + x^514 + x^513 + x^510 + x^509 + x^508 + x^507 + x^503 + x^502 + x^500 + x^497 + x^496 + x^493 + x^491 + x^490 + x^486 + x^480 + x^477 + x^476 + x^475 + x^474 + x^473 + x^472 + x^471 + x^467 + x^464 + x^463 + x^461 + x^458 + x^457 + x^453 + x^446 + x^444 + x^441 + x^439 + x^436 + x^434 + x^433 + x^432 + x^430 + x^427 + x^426 + x^422 + x^418 + x^417 + x^416 + x^414 + x^412 + x^411 + x^410 + x^405 + x^404 + x^401 + x^399 + x^396 + x^395 + x^392 + x^391 + x^390 + x^388 + x^387 + x^386 + x^385 + x^384 + x^383 + x^378 + x^377 + x^375 + x^373 + x^372 + x^371 + x^366 + x^364 + x^358 + x^357 + x^356 + x^352 + x^348 + x^345 + x^342 + x^340 + x^339 + x^338 + x^336 + x^335 + x^331 + x^330 + x^329 + x^327 + x^324 + x^323 + x^322 + x^321 + x^320 + x^317 + x^316 + x^315 + x^314 + x^310 + x^307 + x^306 + x^303 + x^297 + x^295 + x^294 + x^293 + x^292 + x^291 + x^290 + x^288 + x^286 + x^284 + x^283 + x^282 + x^278 + x^276 + x^275 + x^274 + x^269 + x^268 + x^267 + x^263 + x^262 + x^261 + x^260 + x^259 + x^258 + x^256 + x^251 + x^249 + x^248 + x^247 + x^245 + x^242 + x^241 + x^239 + x^238 + x^236 + x^235 + x^234 + x^233 + x^232 + x^230 + x^229 + x^225 + x^224 + x^223 + x^222 + x^220 + x^218 + x^216 + x^214 + x^213 + x^212 + x^211 + x^209 + x^206 + x^201 + x^200 + x^199 + x^197 + x^196 + x^194 + x^193 + x^190 + x^186 + x^184 + x^181 + x^180 + x^179 + x^178 + x^177 + x^176 + x^175 + x^169 + x^168 + x^166 + x^164 + x^160 + x^158 + x^157 + x^156 + x^153 + x^152 + x^147 + x^146 + x^136 + x^134 + x^132 + x^129 + x^126 + x^125 + x^124 + x^123 + x^122 + x^120 + x^119 + x^116 + x^114 + x^112 + x^108 + x^107 + x^106 + x^104 + x^102 + x^98 + x^95 + x^91 + x^85 + x^84 + x^82 + x^81 + x^80 + x^79 + x^77 + x^76 + x^73 + x^72 + x^70 + x^62 + x^60 + x^59 + x^58 + x^56 + x^55 + x^53 + x^49 + x^44 + x^42 + x^41 + x^39 + x^38 + x^35 + x^34 + x^32 + x^28 + x^26 + x^24 + x^22 + 1 + +17-12-36 341 x^928 + x^898 + x^870 + x^868 + x^860 + x^848 + x^838 + x^830 + x^829 + x^806 + x^798 + x^797 + x^789 + x^788 + x^772 + x^770 + x^767 + x^762 + x^761 + x^756 + x^751 + x^749 + x^748 + x^746 + x^742 + x^740 + x^738 + x^730 + x^726 + x^721 + x^718 + x^716 + x^714 + x^710 + x^709 + x^707 + x^706 + x^705 + x^696 + x^688 + x^684 + x^681 + x^677 + x^676 + x^674 + x^669 + x^666 + x^664 + x^659 + x^658 + x^657 + x^656 + x^655 + x^654 + x^651 + x^650 + x^647 + x^645 + x^639 + x^637 + x^636 + x^634 + x^631 + x^627 + x^625 + x^624 + x^622 + x^621 + x^619 + x^615 + x^614 + x^608 + x^605 + x^601 + x^599 + x^597 + x^596 + x^594 + x^593 + x^592 + x^591 + x^588 + x^587 + x^586 + x^585 + x^583 + x^582 + x^580 + x^578 + x^577 + x^576 + x^575 + x^574 + x^573 + x^572 + x^569 + x^568 + x^567 + x^565 + x^564 + x^561 + x^560 + x^559 + x^558 + x^557 + x^554 + x^553 + x^546 + x^545 + x^543 + x^542 + x^541 + x^539 + x^538 + x^536 + x^533 + x^530 + x^529 + x^527 + x^526 + x^525 + x^521 + x^520 + x^513 + x^509 + x^508 + x^504 + x^501 + x^500 + x^499 + x^498 + x^497 + x^496 + x^494 + x^488 + x^486 + x^484 + x^480 + x^479 + x^478 + x^475 + x^471 + x^469 + x^466 + x^463 + x^458 + x^456 + x^455 + x^452 + x^450 + x^449 + x^445 + x^437 + x^436 + x^433 + x^432 + x^430 + x^428 + x^426 + x^425 + x^423 + x^417 + x^416 + x^415 + x^413 + x^401 + x^400 + x^395 + x^394 + x^392 + x^391 + x^387 + x^386 + x^385 + x^384 + x^383 + x^379 + x^375 + x^373 + x^372 + x^371 + x^368 + x^366 + x^361 + x^360 + x^359 + x^358 + x^357 + x^354 + x^352 + x^350 + x^349 + x^348 + x^347 + x^345 + x^344 + x^343 + x^340 + x^334 + x^332 + x^330 + x^328 + x^327 + x^325 + x^323 + x^321 + x^318 + x^317 + x^316 + x^315 + x^314 + x^311 + x^310 + x^309 + x^308 + x^307 + x^306 + x^304 + x^299 + x^298 + x^296 + x^295 + x^294 + x^292 + x^291 + x^288 + x^287 + x^286 + x^282 + x^281 + x^280 + x^272 + x^271 + x^268 + x^267 + x^262 + x^256 + x^253 + x^252 + x^250 + x^244 + x^242 + x^241 + x^240 + x^238 + x^234 + x^233 + x^232 + x^231 + x^229 + x^228 + x^227 + x^226 + x^223 + x^222 + x^221 + x^220 + x^219 + x^218 + x^217 + x^215 + x^214 + x^212 + x^211 + x^208 + x^206 + x^203 + x^201 + x^199 + x^198 + x^195 + x^194 + x^193 + x^192 + x^189 + x^185 + x^182 + x^180 + x^179 + x^177 + x^176 + x^173 + x^172 + x^170 + x^164 + x^163 + x^160 + x^157 + x^155 + x^153 + x^152 + x^150 + x^147 + x^146 + x^139 + x^137 + x^132 + x^131 + x^127 + x^126 + x^125 + x^123 + x^122 + x^120 + x^115 + x^113 + x^111 + x^109 + x^107 + x^106 + x^104 + x^103 + x^101 + x^100 + x^99 + x^98 + x^97 + x^94 + x^88 + x^85 + x^78 + x^77 + x^76 + x^71 + x^68 + x^66 + x^64 + x^51 + x^47 + x^45 + x^39 + x^38 + x^36 + x^31 + x^30 + x^26 + x^24 + x^22 + x^20 + x^15 + 1 + +2-30-27 343 x^928 + x^898 + x^881 + x^870 + x^868 + x^866 + x^851 + x^838 + x^819 + x^817 + x^808 + x^806 + x^804 + x^778 + x^776 + x^774 + x^772 + x^761 + x^759 + x^755 + x^753 + x^748 + x^742 + x^740 + x^731 + x^725 + x^718 + x^716 + x^712 + x^710 + x^706 + x^695 + x^693 + x^691 + x^689 + x^688 + x^682 + x^678 + x^674 + x^665 + x^663 + x^661 + x^658 + x^657 + x^652 + x^646 + x^644 + x^642 + x^641 + x^637 + x^629 + x^628 + x^627 + x^626 + x^625 + x^618 + x^616 + x^614 + x^612 + x^611 + x^610 + x^607 + x^601 + x^598 + x^597 + x^593 + x^592 + x^588 + x^580 + x^579 + x^577 + x^575 + x^573 + x^571 + x^569 + x^568 + x^566 + x^564 + x^563 + x^562 + x^561 + x^560 + x^554 + x^552 + x^550 + x^548 + x^543 + x^539 + x^538 + x^536 + x^534 + x^531 + x^530 + x^529 + x^528 + x^524 + x^521 + x^519 + x^517 + x^515 + x^513 + x^509 + x^508 + x^507 + x^505 + x^502 + x^500 + x^499 + x^498 + x^497 + x^492 + x^491 + x^487 + x^486 + x^484 + x^479 + x^478 + x^477 + x^476 + x^473 + x^470 + x^468 + x^466 + x^465 + x^464 + x^458 + x^454 + x^450 + x^448 + x^447 + x^446 + x^445 + x^439 + x^436 + x^435 + x^434 + x^433 + x^432 + x^428 + x^426 + x^422 + x^420 + x^416 + x^415 + x^413 + x^411 + x^409 + x^407 + x^406 + x^404 + x^399 + x^396 + x^392 + x^389 + x^384 + x^383 + x^382 + x^381 + x^379 + x^377 + x^376 + x^374 + x^372 + x^370 + x^367 + x^359 + x^355 + x^352 + x^347 + x^346 + x^344 + x^342 + x^341 + x^339 + x^338 + x^336 + x^335 + x^334 + x^328 + x^327 + x^324 + x^320 + x^318 + x^316 + x^314 + x^313 + x^309 + x^308 + x^304 + x^303 + x^300 + x^298 + x^297 + x^296 + x^294 + x^292 + x^287 + x^286 + x^284 + x^283 + x^282 + x^278 + x^276 + x^272 + x^269 + x^266 + x^265 + x^264 + x^261 + x^256 + x^252 + x^250 + x^248 + x^247 + x^245 + x^244 + x^243 + x^242 + x^239 + x^238 + x^237 + x^236 + x^235 + x^234 + x^233 + x^231 + x^229 + x^226 + x^225 + x^224 + x^220 + x^219 + x^216 + x^213 + x^212 + x^210 + x^205 + x^204 + x^203 + x^201 + x^199 + x^197 + x^194 + x^192 + x^191 + x^190 + x^189 + x^188 + x^186 + x^184 + x^183 + x^182 + x^178 + x^177 + x^176 + x^175 + x^173 + x^171 + x^170 + x^169 + x^166 + x^165 + x^163 + x^162 + x^161 + x^157 + x^156 + x^155 + x^153 + x^152 + x^150 + x^148 + x^147 + x^146 + x^144 + x^143 + x^141 + x^139 + x^137 + x^134 + x^133 + x^129 + x^128 + x^127 + x^126 + x^125 + x^123 + x^119 + x^117 + x^116 + x^115 + x^114 + x^113 + x^111 + x^110 + x^102 + x^101 + x^100 + x^99 + x^97 + x^96 + x^94 + x^93 + x^89 + x^88 + x^84 + x^83 + x^81 + x^78 + x^74 + x^72 + x^71 + x^69 + x^67 + x^62 + x^61 + x^59 + x^57 + x^56 + x^55 + x^53 + x^51 + x^47 + x^44 + x^42 + x^39 + x^37 + x^36 + x^35 + x^33 + x^30 + x^26 + x^24 + x^23 + x^21 + x^19 + x^18 + x^17 + x^16 + x^10 + x^8 + x^2 + 1 + +26-55-39 345 x^928 + x^904 + x^898 + x^870 + x^861 + x^856 + x^851 + x^850 + x^831 + x^821 + x^820 + x^818 + x^813 + x^802 + x^796 + x^794 + x^790 + x^788 + x^783 + x^778 + x^775 + x^772 + x^770 + x^765 + x^760 + x^754 + x^746 + x^745 + x^741 + x^740 + x^736 + x^735 + x^734 + x^732 + x^731 + x^729 + x^727 + x^723 + x^722 + x^721 + x^716 + x^715 + x^710 + x^708 + x^705 + x^701 + x^699 + x^686 + x^685 + x^684 + x^681 + x^679 + x^678 + x^676 + x^674 + x^673 + x^672 + x^668 + x^664 + x^663 + x^661 + x^660 + x^658 + x^656 + x^650 + x^648 + x^645 + x^643 + x^641 + x^638 + x^637 + x^636 + x^632 + x^622 + x^621 + x^618 + x^616 + x^615 + x^614 + x^612 + x^608 + x^603 + x^600 + x^598 + x^595 + x^594 + x^593 + x^591 + x^590 + x^589 + x^588 + x^585 + x^582 + x^581 + x^576 + x^574 + x^571 + x^569 + x^568 + x^564 + x^562 + x^557 + x^556 + x^553 + x^550 + x^549 + x^548 + x^547 + x^546 + x^545 + x^544 + x^543 + x^538 + x^535 + x^530 + x^525 + x^520 + x^517 + x^515 + x^514 + x^510 + x^509 + x^506 + x^501 + x^497 + x^495 + x^489 + x^486 + x^484 + x^483 + x^482 + x^480 + x^479 + x^476 + x^474 + x^473 + x^471 + x^470 + x^469 + x^467 + x^464 + x^462 + x^460 + x^453 + x^452 + x^450 + x^449 + x^448 + x^446 + x^444 + x^443 + x^441 + x^436 + x^435 + x^434 + x^433 + x^432 + x^429 + x^428 + x^427 + x^424 + x^419 + x^418 + x^417 + x^416 + x^413 + x^412 + x^410 + x^407 + x^404 + x^403 + x^401 + x^399 + x^398 + x^397 + x^396 + x^394 + x^393 + x^392 + x^390 + x^386 + x^385 + x^383 + x^379 + x^378 + x^377 + x^376 + x^375 + x^373 + x^372 + x^366 + x^362 + x^359 + x^357 + x^355 + x^354 + x^351 + x^350 + x^347 + x^346 + x^344 + x^340 + x^339 + x^338 + x^337 + x^333 + x^332 + x^331 + x^330 + x^328 + x^326 + x^325 + x^323 + x^319 + x^315 + x^314 + x^313 + x^311 + x^310 + x^308 + x^307 + x^306 + x^304 + x^301 + x^300 + x^298 + x^296 + x^294 + x^293 + x^292 + x^291 + x^290 + x^286 + x^283 + x^280 + x^277 + x^274 + x^272 + x^265 + x^264 + x^263 + x^261 + x^260 + x^258 + x^255 + x^252 + x^251 + x^250 + x^249 + x^247 + x^245 + x^244 + x^241 + x^238 + x^235 + x^234 + x^230 + x^228 + x^225 + x^221 + x^218 + x^212 + x^211 + x^210 + x^208 + x^206 + x^204 + x^203 + x^202 + x^199 + x^198 + x^196 + x^195 + x^193 + x^188 + x^186 + x^184 + x^183 + x^177 + x^176 + x^167 + x^166 + x^165 + x^163 + x^161 + x^160 + x^158 + x^156 + x^153 + x^151 + x^146 + x^144 + x^141 + x^140 + x^134 + x^133 + x^129 + x^128 + x^127 + x^124 + x^123 + x^121 + x^119 + x^117 + x^115 + x^114 + x^113 + x^112 + x^111 + x^110 + x^109 + x^105 + x^104 + x^103 + x^100 + x^99 + x^93 + x^92 + x^91 + x^86 + x^84 + x^81 + x^80 + x^77 + x^75 + x^72 + x^71 + x^70 + x^68 + x^66 + x^65 + x^64 + x^61 + x^54 + x^53 + x^47 + x^45 + x^36 + x^35 + x^34 + x^32 + x^29 + x^18 + x^7 + 1 + +40-36-23 345 x^928 + x^898 + x^886 + x^870 + x^862 + x^844 + x^838 + x^826 + x^820 + x^816 + x^814 + x^808 + x^797 + x^792 + x^791 + x^790 + x^784 + x^778 + x^777 + x^772 + x^768 + x^767 + x^766 + x^762 + x^761 + x^756 + x^753 + x^749 + x^747 + x^744 + x^742 + x^738 + x^735 + x^734 + x^726 + x^724 + x^723 + x^720 + x^712 + x^702 + x^701 + x^695 + x^689 + x^684 + x^683 + x^681 + x^677 + x^675 + x^670 + x^669 + x^664 + x^663 + x^660 + x^659 + x^658 + x^653 + x^648 + x^646 + x^644 + x^634 + x^629 + x^628 + x^627 + x^626 + x^625 + x^622 + x^621 + x^617 + x^616 + x^611 + x^609 + x^605 + x^604 + x^603 + x^602 + x^597 + x^595 + x^594 + x^593 + x^592 + x^586 + x^585 + x^584 + x^578 + x^575 + x^574 + x^573 + x^572 + x^571 + x^569 + x^565 + x^563 + x^561 + x^559 + x^557 + x^555 + x^554 + x^547 + x^545 + x^543 + x^539 + x^537 + x^536 + x^535 + x^534 + x^529 + x^528 + x^527 + x^525 + x^524 + x^521 + x^520 + x^519 + x^518 + x^517 + x^515 + x^514 + x^512 + x^510 + x^509 + x^508 + x^507 + x^506 + x^504 + x^503 + x^501 + x^499 + x^496 + x^494 + x^490 + x^489 + x^487 + x^485 + x^484 + x^482 + x^481 + x^480 + x^479 + x^478 + x^477 + x^475 + x^473 + x^470 + x^466 + x^465 + x^464 + x^463 + x^460 + x^457 + x^456 + x^454 + x^453 + x^452 + x^447 + x^446 + x^442 + x^441 + x^439 + x^436 + x^434 + x^432 + x^430 + x^428 + x^427 + x^423 + x^421 + x^416 + x^415 + x^414 + x^413 + x^411 + x^408 + x^405 + x^402 + x^401 + x^399 + x^396 + x^394 + x^393 + x^391 + x^390 + x^389 + x^388 + x^387 + x^386 + x^384 + x^381 + x^379 + x^378 + x^377 + x^375 + x^374 + x^372 + x^369 + x^367 + x^366 + x^364 + x^363 + x^361 + x^360 + x^359 + x^358 + x^356 + x^355 + x^354 + x^353 + x^352 + x^349 + x^345 + x^343 + x^342 + x^338 + x^336 + x^335 + x^334 + x^333 + x^331 + x^329 + x^328 + x^327 + x^320 + x^318 + x^317 + x^316 + x^312 + x^307 + x^304 + x^303 + x^300 + x^298 + x^296 + x^295 + x^294 + x^293 + x^291 + x^290 + x^288 + x^287 + x^286 + x^285 + x^284 + x^276 + x^275 + x^274 + x^273 + x^272 + x^271 + x^269 + x^268 + x^266 + x^263 + x^262 + x^259 + x^257 + x^255 + x^254 + x^251 + x^248 + x^246 + x^244 + x^240 + x^238 + x^237 + x^235 + x^234 + x^232 + x^228 + x^226 + x^225 + x^224 + x^222 + x^220 + x^217 + x^213 + x^212 + x^211 + x^209 + x^208 + x^207 + x^202 + x^200 + x^199 + x^198 + x^192 + x^190 + x^187 + x^183 + x^182 + x^180 + x^175 + x^172 + x^171 + x^170 + x^169 + x^167 + x^166 + x^165 + x^164 + x^163 + x^160 + x^157 + x^154 + x^153 + x^152 + x^150 + x^149 + x^148 + x^147 + x^146 + x^144 + x^142 + x^140 + x^137 + x^136 + x^134 + x^132 + x^129 + x^128 + x^124 + x^119 + x^117 + x^116 + x^115 + x^110 + x^108 + x^106 + x^103 + x^98 + x^88 + x^82 + x^76 + x^75 + x^74 + x^69 + x^68 + x^64 + x^62 + x^51 + x^47 + x^46 + x^39 + x^32 + x^23 + x^22 + 1 + +24-36-37 347 x^928 + x^898 + x^878 + x^870 + x^854 + x^830 + x^824 + x^821 + x^818 + x^816 + x^813 + x^808 + x^806 + x^800 + x^792 + x^791 + x^789 + x^783 + x^780 + x^778 + x^776 + x^768 + x^765 + x^759 + x^753 + x^750 + x^747 + x^745 + x^744 + x^741 + x^740 + x^736 + x^734 + x^729 + x^721 + x^717 + x^715 + x^714 + x^708 + x^705 + x^704 + x^702 + x^699 + x^694 + x^690 + x^688 + x^684 + x^682 + x^681 + x^680 + x^679 + x^670 + x^669 + x^666 + x^664 + x^663 + x^660 + x^652 + x^649 + x^646 + x^645 + x^643 + x^639 + x^636 + x^634 + x^633 + x^631 + x^629 + x^627 + x^622 + x^621 + x^620 + x^614 + x^613 + x^612 + x^611 + x^610 + x^609 + x^605 + x^603 + x^601 + x^600 + x^599 + x^596 + x^593 + x^590 + x^587 + x^586 + x^585 + x^584 + x^583 + x^582 + x^581 + x^579 + x^577 + x^573 + x^565 + x^564 + x^563 + x^562 + x^561 + x^560 + x^559 + x^558 + x^557 + x^554 + x^550 + x^549 + x^546 + x^545 + x^544 + x^543 + x^542 + x^539 + x^538 + x^536 + x^535 + x^534 + x^529 + x^527 + x^525 + x^524 + x^523 + x^521 + x^519 + x^518 + x^516 + x^515 + x^512 + x^510 + x^505 + x^504 + x^503 + x^501 + x^500 + x^499 + x^493 + x^492 + x^488 + x^484 + x^483 + x^479 + x^474 + x^472 + x^470 + x^465 + x^464 + x^459 + x^457 + x^455 + x^454 + x^450 + x^440 + x^435 + x^434 + x^433 + x^427 + x^426 + x^424 + x^421 + x^420 + x^419 + x^414 + x^413 + x^412 + x^409 + x^407 + x^406 + x^405 + x^403 + x^401 + x^400 + x^399 + x^398 + x^395 + x^393 + x^391 + x^390 + x^388 + x^383 + x^381 + x^376 + x^374 + x^370 + x^367 + x^365 + x^363 + x^359 + x^356 + x^355 + x^353 + x^352 + x^348 + x^347 + x^346 + x^345 + x^344 + x^343 + x^342 + x^339 + x^336 + x^330 + x^327 + x^326 + x^325 + x^324 + x^323 + x^321 + x^318 + x^316 + x^315 + x^314 + x^312 + x^311 + x^310 + x^302 + x^300 + x^298 + x^295 + x^294 + x^292 + x^284 + x^282 + x^279 + x^278 + x^277 + x^276 + x^271 + x^268 + x^267 + x^266 + x^265 + x^264 + x^262 + x^260 + x^256 + x^253 + x^250 + x^249 + x^248 + x^244 + x^243 + x^242 + x^239 + x^236 + x^235 + x^232 + x^231 + x^229 + x^227 + x^226 + x^222 + x^220 + x^219 + x^216 + x^215 + x^214 + x^213 + x^211 + x^209 + x^208 + x^204 + x^202 + x^200 + x^199 + x^198 + x^196 + x^193 + x^191 + x^190 + x^189 + x^188 + x^187 + x^186 + x^179 + x^178 + x^177 + x^176 + x^175 + x^174 + x^170 + x^169 + x^167 + x^163 + x^162 + x^161 + x^160 + x^153 + x^150 + x^149 + x^147 + x^145 + x^140 + x^138 + x^136 + x^135 + x^133 + x^131 + x^130 + x^128 + x^127 + x^119 + x^117 + x^113 + x^111 + x^110 + x^108 + x^104 + x^103 + x^102 + x^101 + x^100 + x^99 + x^97 + x^96 + x^93 + x^88 + x^87 + x^86 + x^85 + x^84 + x^83 + x^82 + x^80 + x^78 + x^77 + x^72 + x^71 + x^70 + x^67 + x^66 + x^64 + x^63 + x^62 + x^61 + x^59 + x^58 + x^56 + x^54 + x^53 + x^50 + x^47 + x^46 + x^38 + x^24 + x^16 + 1 + +26-38-9 353 x^928 + x^898 + x^875 + x^870 + x^848 + x^822 + x^808 + x^806 + x^800 + x^798 + x^797 + x^795 + x^792 + x^788 + x^778 + x^776 + x^769 + x^767 + x^765 + x^762 + x^758 + x^756 + x^747 + x^741 + x^732 + x^730 + x^728 + x^727 + x^723 + x^719 + x^714 + x^712 + x^711 + x^709 + x^706 + x^703 + x^700 + x^697 + x^695 + x^689 + x^688 + x^686 + x^684 + x^682 + x^680 + x^678 + x^676 + x^674 + x^673 + x^667 + x^666 + x^664 + x^661 + x^658 + x^653 + x^652 + x^649 + x^648 + x^646 + x^645 + x^635 + x^634 + x^633 + x^629 + x^626 + x^625 + x^620 + x^616 + x^614 + x^612 + x^611 + x^610 + x^607 + x^606 + x^604 + x^603 + x^599 + x^595 + x^594 + x^591 + x^583 + x^580 + x^578 + x^577 + x^572 + x^569 + x^568 + x^567 + x^566 + x^564 + x^559 + x^558 + x^555 + x^554 + x^550 + x^549 + x^548 + x^543 + x^542 + x^541 + x^539 + x^533 + x^532 + x^531 + x^528 + x^527 + x^526 + x^523 + x^519 + x^518 + x^515 + x^514 + x^513 + x^512 + x^511 + x^510 + x^509 + x^508 + x^507 + x^506 + x^502 + x^497 + x^495 + x^491 + x^488 + x^487 + x^485 + x^483 + x^482 + x^481 + x^478 + x^477 + x^476 + x^475 + x^473 + x^469 + x^466 + x^463 + x^462 + x^460 + x^458 + x^457 + x^456 + x^451 + x^447 + x^444 + x^443 + x^442 + x^440 + x^438 + x^437 + x^435 + x^433 + x^432 + x^430 + x^427 + x^424 + x^418 + x^417 + x^416 + x^414 + x^410 + x^409 + x^403 + x^401 + x^398 + x^397 + x^396 + x^394 + x^393 + x^391 + x^389 + x^388 + x^386 + x^385 + x^384 + x^382 + x^381 + x^380 + x^375 + x^374 + x^368 + x^364 + x^363 + x^359 + x^358 + x^357 + x^356 + x^354 + x^350 + x^349 + x^347 + x^346 + x^339 + x^337 + x^336 + x^334 + x^332 + x^330 + x^329 + x^328 + x^327 + x^326 + x^325 + x^324 + x^320 + x^318 + x^317 + x^315 + x^313 + x^309 + x^307 + x^305 + x^304 + x^303 + x^300 + x^299 + x^295 + x^293 + x^290 + x^288 + x^286 + x^285 + x^281 + x^279 + x^277 + x^274 + x^273 + x^269 + x^267 + x^266 + x^265 + x^263 + x^262 + x^257 + x^256 + x^255 + x^251 + x^249 + x^247 + x^246 + x^243 + x^241 + x^237 + x^236 + x^233 + x^230 + x^228 + x^227 + x^226 + x^225 + x^224 + x^223 + x^222 + x^221 + x^220 + x^212 + x^211 + x^210 + x^209 + x^208 + x^205 + x^204 + x^202 + x^201 + x^200 + x^197 + x^192 + x^191 + x^186 + x^184 + x^183 + x^181 + x^177 + x^175 + x^174 + x^170 + x^167 + x^165 + x^164 + x^162 + x^156 + x^154 + x^149 + x^148 + x^147 + x^144 + x^143 + x^142 + x^141 + x^139 + x^138 + x^137 + x^133 + x^131 + x^130 + x^128 + x^127 + x^126 + x^120 + x^116 + x^115 + x^113 + x^111 + x^108 + x^107 + x^105 + x^101 + x^99 + x^97 + x^95 + x^91 + x^90 + x^88 + x^87 + x^85 + x^81 + x^79 + x^78 + x^75 + x^72 + x^71 + x^70 + x^69 + x^66 + x^65 + x^63 + x^59 + x^57 + x^55 + x^54 + x^52 + x^51 + x^50 + x^49 + x^46 + x^44 + x^43 + x^39 + x^36 + x^34 + x^33 + x^32 + x^31 + x^30 + x^26 + x^24 + x^22 + x^16 + x^14 + x^8 + x^6 + 1 + +31-6-30 353 x^928 + x^898 + x^870 + x^834 + x^814 + x^796 + x^790 + x^788 + x^784 + x^774 + x^772 + x^770 + x^762 + x^760 + x^756 + x^748 + x^740 + x^734 + x^732 + x^728 + x^727 + x^726 + x^718 + x^716 + x^712 + x^707 + x^701 + x^700 + x^697 + x^692 + x^688 + x^686 + x^684 + x^682 + x^681 + x^678 + x^677 + x^676 + x^674 + x^671 + x^670 + x^668 + x^665 + x^660 + x^658 + x^655 + x^652 + x^651 + x^650 + x^649 + x^646 + x^642 + x^641 + x^639 + x^638 + x^637 + x^636 + x^635 + x^630 + x^625 + x^624 + x^623 + x^622 + x^621 + x^620 + x^619 + x^617 + x^612 + x^611 + x^610 + x^609 + x^608 + x^607 + x^606 + x^604 + x^601 + x^598 + x^597 + x^593 + x^592 + x^591 + x^590 + x^582 + x^579 + x^578 + x^572 + x^570 + x^567 + x^566 + x^565 + x^563 + x^561 + x^559 + x^558 + x^555 + x^554 + x^544 + x^543 + x^542 + x^540 + x^539 + x^536 + x^533 + x^529 + x^528 + x^525 + x^522 + x^517 + x^516 + x^512 + x^511 + x^510 + x^507 + x^505 + x^502 + x^501 + x^499 + x^494 + x^492 + x^491 + x^489 + x^488 + x^487 + x^486 + x^484 + x^483 + x^481 + x^480 + x^476 + x^473 + x^469 + x^468 + x^466 + x^465 + x^464 + x^457 + x^451 + x^449 + x^448 + x^445 + x^444 + x^441 + x^440 + x^438 + x^436 + x^435 + x^433 + x^432 + x^431 + x^428 + x^425 + x^422 + x^420 + x^418 + x^414 + x^413 + x^412 + x^411 + x^410 + x^406 + x^405 + x^402 + x^401 + x^400 + x^394 + x^390 + x^389 + x^387 + x^384 + x^382 + x^380 + x^376 + x^375 + x^372 + x^371 + x^370 + x^369 + x^366 + x^363 + x^362 + x^361 + x^359 + x^357 + x^356 + x^354 + x^352 + x^351 + x^349 + x^347 + x^346 + x^344 + x^343 + x^342 + x^341 + x^340 + x^338 + x^337 + x^336 + x^334 + x^333 + x^331 + x^328 + x^323 + x^322 + x^318 + x^315 + x^313 + x^312 + x^310 + x^309 + x^308 + x^307 + x^301 + x^300 + x^298 + x^296 + x^295 + x^294 + x^292 + x^287 + x^285 + x^284 + x^283 + x^282 + x^279 + x^276 + x^275 + x^272 + x^270 + x^269 + x^268 + x^267 + x^263 + x^261 + x^259 + x^258 + x^256 + x^250 + x^248 + x^247 + x^246 + x^245 + x^244 + x^240 + x^239 + x^238 + x^237 + x^236 + x^235 + x^232 + x^231 + x^230 + x^229 + x^226 + x^223 + x^220 + x^217 + x^216 + x^213 + x^212 + x^210 + x^208 + x^205 + x^204 + x^201 + x^195 + x^194 + x^192 + x^191 + x^189 + x^188 + x^182 + x^181 + x^177 + x^175 + x^174 + x^173 + x^171 + x^169 + x^167 + x^162 + x^161 + x^159 + x^158 + x^154 + x^153 + x^148 + x^146 + x^145 + x^144 + x^143 + x^142 + x^141 + x^140 + x^139 + x^138 + x^135 + x^134 + x^133 + x^122 + x^119 + x^118 + x^115 + x^111 + x^110 + x^108 + x^107 + x^106 + x^105 + x^101 + x^99 + x^97 + x^96 + x^95 + x^94 + x^93 + x^92 + x^90 + x^87 + x^84 + x^83 + x^80 + x^74 + x^73 + x^71 + x^70 + x^67 + x^66 + x^63 + x^55 + x^53 + x^52 + x^51 + x^49 + x^45 + x^44 + x^43 + x^40 + x^35 + x^34 + x^33 + x^32 + x^27 + x^25 + x^24 + x^22 + x^21 + x^20 + x^17 + x^16 + x^11 + x^4 + 1 + +54-21-7 353 x^928 + x^898 + x^870 + x^861 + x^860 + x^859 + x^858 + x^846 + x^842 + x^837 + x^836 + x^835 + x^831 + x^830 + x^829 + x^828 + x^822 + x^818 + x^811 + x^810 + x^808 + x^807 + x^804 + x^798 + x^792 + x^788 + x^781 + x^778 + x^758 + x^756 + x^751 + x^750 + x^746 + x^745 + x^740 + x^739 + x^738 + x^722 + x^720 + x^716 + x^703 + x^701 + x^696 + x^695 + x^688 + x^686 + x^674 + x^666 + x^659 + x^658 + x^656 + x^655 + x^654 + x^649 + x^644 + x^643 + x^641 + x^637 + x^632 + x^630 + x^629 + x^626 + x^623 + x^621 + x^620 + x^618 + x^614 + x^613 + x^606 + x^601 + x^599 + x^597 + x^595 + x^592 + x^591 + x^589 + x^588 + x^586 + x^584 + x^582 + x^577 + x^576 + x^575 + x^568 + x^567 + x^565 + x^564 + x^562 + x^558 + x^556 + x^553 + x^551 + x^550 + x^548 + x^547 + x^541 + x^540 + x^535 + x^534 + x^528 + x^527 + x^526 + x^522 + x^521 + x^520 + x^517 + x^514 + x^512 + x^510 + x^504 + x^503 + x^502 + x^498 + x^497 + x^496 + x^487 + x^482 + x^474 + x^473 + x^469 + x^468 + x^466 + x^463 + x^462 + x^459 + x^454 + x^453 + x^452 + x^451 + x^447 + x^442 + x^441 + x^433 + x^431 + x^430 + x^429 + x^426 + x^424 + x^423 + x^422 + x^421 + x^419 + x^417 + x^415 + x^412 + x^411 + x^409 + x^405 + x^402 + x^397 + x^396 + x^393 + x^392 + x^391 + x^385 + x^384 + x^382 + x^381 + x^380 + x^370 + x^369 + x^367 + x^365 + x^364 + x^361 + x^359 + x^356 + x^355 + x^351 + x^350 + x^349 + x^343 + x^342 + x^341 + x^340 + x^339 + x^338 + x^336 + x^335 + x^333 + x^332 + x^331 + x^330 + x^326 + x^325 + x^324 + x^323 + x^318 + x^317 + x^315 + x^313 + x^310 + x^309 + x^306 + x^301 + x^300 + x^298 + x^295 + x^294 + x^293 + x^292 + x^291 + x^290 + x^289 + x^288 + x^282 + x^281 + x^278 + x^277 + x^276 + x^275 + x^273 + x^271 + x^268 + x^266 + x^265 + x^263 + x^261 + x^260 + x^259 + x^257 + x^252 + x^249 + x^247 + x^244 + x^243 + x^238 + x^236 + x^232 + x^231 + x^229 + x^228 + x^227 + x^226 + x^225 + x^224 + x^221 + x^220 + x^219 + x^215 + x^214 + x^211 + x^209 + x^207 + x^204 + x^201 + x^198 + x^196 + x^195 + x^194 + x^193 + x^191 + x^189 + x^186 + x^185 + x^182 + x^181 + x^179 + x^178 + x^174 + x^173 + x^172 + x^170 + x^168 + x^166 + x^164 + x^162 + x^160 + x^159 + x^156 + x^155 + x^154 + x^152 + x^151 + x^149 + x^148 + x^147 + x^146 + x^145 + x^143 + x^140 + x^134 + x^133 + x^130 + x^129 + x^128 + x^126 + x^124 + x^123 + x^122 + x^121 + x^119 + x^118 + x^117 + x^116 + x^115 + x^113 + x^110 + x^107 + x^106 + x^105 + x^104 + x^103 + x^102 + x^101 + x^99 + x^97 + x^96 + x^95 + x^94 + x^93 + x^92 + x^91 + x^90 + x^89 + x^86 + x^85 + x^82 + x^81 + x^80 + x^79 + x^78 + x^76 + x^73 + x^71 + x^69 + x^65 + x^63 + x^60 + x^58 + x^57 + x^56 + x^55 + x^54 + x^52 + x^49 + x^47 + x^46 + x^45 + x^43 + x^42 + x^40 + x^39 + x^38 + x^36 + x^33 + x^32 + x^28 + x^25 + x^19 + x^18 + x^7 + 1 + +8-36-47 355 x^928 + x^898 + x^870 + x^860 + x^856 + x^830 + x^818 + x^814 + x^813 + x^808 + x^800 + x^797 + x^796 + x^792 + x^788 + x^783 + x^778 + x^776 + x^772 + x^771 + x^770 + x^767 + x^766 + x^762 + x^755 + x^750 + x^740 + x^737 + x^736 + x^734 + x^730 + x^729 + x^724 + x^719 + x^711 + x^710 + x^707 + x^706 + x^698 + x^693 + x^692 + x^690 + x^688 + x^687 + x^680 + x^676 + x^674 + x^673 + x^672 + x^669 + x^665 + x^663 + x^662 + x^658 + x^657 + x^656 + x^653 + x^651 + x^650 + x^648 + x^647 + x^642 + x^635 + x^632 + x^631 + x^630 + x^627 + x^625 + x^624 + x^623 + x^622 + x^620 + x^619 + x^618 + x^609 + x^600 + x^595 + x^593 + x^592 + x^591 + x^589 + x^587 + x^583 + x^582 + x^579 + x^578 + x^577 + x^574 + x^570 + x^566 + x^565 + x^561 + x^559 + x^558 + x^557 + x^556 + x^555 + x^552 + x^550 + x^548 + x^544 + x^542 + x^541 + x^540 + x^537 + x^536 + x^534 + x^533 + x^532 + x^531 + x^528 + x^527 + x^525 + x^524 + x^521 + x^519 + x^518 + x^517 + x^516 + x^515 + x^514 + x^513 + x^512 + x^511 + x^509 + x^507 + x^505 + x^504 + x^502 + x^501 + x^497 + x^496 + x^493 + x^492 + x^491 + x^490 + x^488 + x^486 + x^485 + x^480 + x^478 + x^476 + x^475 + x^473 + x^469 + x^468 + x^467 + x^466 + x^463 + x^462 + x^453 + x^452 + x^451 + x^449 + x^447 + x^446 + x^445 + x^444 + x^443 + x^440 + x^438 + x^437 + x^436 + x^435 + x^427 + x^426 + x^423 + x^421 + x^419 + x^416 + x^411 + x^410 + x^409 + x^408 + x^406 + x^405 + x^404 + x^403 + x^402 + x^400 + x^398 + x^397 + x^395 + x^391 + x^390 + x^389 + x^385 + x^384 + x^380 + x^378 + x^377 + x^374 + x^373 + x^372 + x^370 + x^367 + x^366 + x^365 + x^363 + x^361 + x^360 + x^354 + x^352 + x^351 + x^350 + x^347 + x^342 + x^339 + x^337 + x^334 + x^333 + x^330 + x^328 + x^326 + x^324 + x^323 + x^315 + x^314 + x^309 + x^306 + x^304 + x^302 + x^298 + x^297 + x^296 + x^295 + x^293 + x^291 + x^289 + x^284 + x^282 + x^281 + x^279 + x^275 + x^274 + x^273 + x^269 + x^267 + x^265 + x^263 + x^261 + x^260 + x^259 + x^257 + x^256 + x^254 + x^252 + x^248 + x^247 + x^246 + x^244 + x^243 + x^242 + x^241 + x^238 + x^235 + x^234 + x^232 + x^229 + x^227 + x^225 + x^224 + x^223 + x^219 + x^217 + x^215 + x^211 + x^208 + x^206 + x^205 + x^203 + x^200 + x^197 + x^195 + x^190 + x^189 + x^187 + x^185 + x^176 + x^174 + x^172 + x^169 + x^165 + x^160 + x^159 + x^158 + x^156 + x^155 + x^154 + x^153 + x^152 + x^150 + x^146 + x^141 + x^139 + x^137 + x^135 + x^132 + x^131 + x^130 + x^127 + x^124 + x^120 + x^119 + x^116 + x^114 + x^111 + x^110 + x^109 + x^108 + x^105 + x^102 + x^99 + x^98 + x^95 + x^94 + x^93 + x^92 + x^90 + x^87 + x^86 + x^76 + x^75 + x^74 + x^72 + x^70 + x^68 + x^67 + x^65 + x^64 + x^63 + x^60 + x^59 + x^58 + x^57 + x^54 + x^51 + x^50 + x^48 + x^46 + x^45 + x^44 + x^43 + x^41 + x^39 + x^38 + x^36 + x^30 + x^27 + x^26 + x^24 + x^18 + x^14 + x^12 + 1 + +44-14-41 357 x^928 + x^898 + x^870 + x^856 + x^852 + x^847 + x^831 + x^822 + x^817 + x^814 + x^808 + x^796 + x^795 + x^792 + x^784 + x^778 + x^777 + x^775 + x^772 + x^771 + x^766 + x^763 + x^756 + x^754 + x^747 + x^746 + x^745 + x^744 + x^742 + x^736 + x^733 + x^732 + x^730 + x^727 + x^726 + x^724 + x^721 + x^717 + x^716 + x^715 + x^714 + x^711 + x^705 + x^704 + x^694 + x^693 + x^691 + x^684 + x^682 + x^678 + x^674 + x^670 + x^664 + x^662 + x^655 + x^653 + x^647 + x^645 + x^644 + x^643 + x^637 + x^636 + x^635 + x^633 + x^631 + x^627 + x^626 + x^625 + x^624 + x^620 + x^618 + x^605 + x^604 + x^603 + x^601 + x^597 + x^588 + x^585 + x^583 + x^578 + x^575 + x^574 + x^569 + x^568 + x^567 + x^566 + x^565 + x^563 + x^558 + x^556 + x^554 + x^553 + x^552 + x^551 + x^550 + x^548 + x^544 + x^543 + x^542 + x^539 + x^538 + x^537 + x^535 + x^533 + x^532 + x^530 + x^528 + x^525 + x^523 + x^521 + x^520 + x^516 + x^515 + x^514 + x^513 + x^510 + x^509 + x^506 + x^504 + x^502 + x^497 + x^496 + x^491 + x^487 + x^485 + x^484 + x^482 + x^481 + x^480 + x^473 + x^471 + x^470 + x^468 + x^466 + x^464 + x^463 + x^462 + x^461 + x^459 + x^458 + x^455 + x^452 + x^450 + x^449 + x^447 + x^446 + x^443 + x^442 + x^440 + x^438 + x^436 + x^435 + x^432 + x^429 + x^428 + x^423 + x^420 + x^418 + x^417 + x^416 + x^413 + x^412 + x^411 + x^410 + x^407 + x^406 + x^405 + x^403 + x^402 + x^401 + x^400 + x^397 + x^396 + x^393 + x^392 + x^391 + x^390 + x^389 + x^386 + x^385 + x^384 + x^382 + x^380 + x^378 + x^376 + x^371 + x^370 + x^369 + x^368 + x^365 + x^364 + x^363 + x^359 + x^358 + x^357 + x^356 + x^353 + x^351 + x^350 + x^349 + x^348 + x^346 + x^345 + x^343 + x^342 + x^340 + x^339 + x^338 + x^335 + x^333 + x^332 + x^327 + x^324 + x^322 + x^317 + x^316 + x^315 + x^313 + x^311 + x^308 + x^307 + x^306 + x^304 + x^303 + x^301 + x^300 + x^299 + x^296 + x^294 + x^293 + x^292 + x^291 + x^290 + x^289 + x^284 + x^281 + x^279 + x^278 + x^276 + x^275 + x^273 + x^272 + x^271 + x^269 + x^266 + x^264 + x^260 + x^258 + x^256 + x^255 + x^251 + x^250 + x^248 + x^247 + x^244 + x^243 + x^241 + x^240 + x^239 + x^236 + x^235 + x^233 + x^232 + x^230 + x^226 + x^224 + x^223 + x^221 + x^220 + x^218 + x^217 + x^216 + x^215 + x^211 + x^210 + x^209 + x^208 + x^206 + x^204 + x^201 + x^200 + x^197 + x^196 + x^192 + x^187 + x^185 + x^183 + x^182 + x^181 + x^179 + x^178 + x^177 + x^175 + x^173 + x^170 + x^169 + x^168 + x^166 + x^159 + x^158 + x^157 + x^155 + x^154 + x^150 + x^149 + x^146 + x^144 + x^139 + x^138 + x^134 + x^132 + x^131 + x^128 + x^125 + x^123 + x^122 + x^120 + x^119 + x^116 + x^115 + x^111 + x^110 + x^109 + x^108 + x^101 + x^100 + x^99 + x^97 + x^96 + x^93 + x^92 + x^90 + x^87 + x^77 + x^74 + x^71 + x^70 + x^63 + x^58 + x^56 + x^54 + x^53 + x^52 + x^51 + x^50 + x^46 + x^42 + x^40 + x^38 + x^37 + x^36 + x^30 + x^28 + x^26 + x^14 + x^12 + 1 + +22-40-11 359 x^928 + x^898 + x^870 + x^829 + x^816 + x^814 + x^808 + x^803 + x^799 + x^786 + x^784 + x^778 + x^777 + x^773 + x^764 + x^762 + x^743 + x^738 + x^736 + x^734 + x^725 + x^723 + x^719 + x^713 + x^710 + x^708 + x^699 + x^691 + x^688 + x^686 + x^684 + x^682 + x^673 + x^672 + x^665 + x^661 + x^657 + x^656 + x^654 + x^652 + x^650 + x^648 + x^645 + x^644 + x^642 + x^639 + x^637 + x^633 + x^631 + x^630 + x^626 + x^624 + x^622 + x^621 + x^620 + x^619 + x^618 + x^615 + x^614 + x^612 + x^611 + x^609 + x^606 + x^605 + x^604 + x^603 + x^601 + x^600 + x^599 + x^598 + x^594 + x^591 + x^587 + x^586 + x^585 + x^582 + x^580 + x^578 + x^577 + x^576 + x^572 + x^562 + x^561 + x^559 + x^558 + x^557 + x^556 + x^554 + x^553 + x^552 + x^550 + x^549 + x^548 + x^545 + x^543 + x^541 + x^540 + x^537 + x^535 + x^534 + x^533 + x^531 + x^530 + x^528 + x^527 + x^526 + x^522 + x^521 + x^518 + x^517 + x^516 + x^515 + x^514 + x^512 + x^511 + x^510 + x^509 + x^508 + x^507 + x^501 + x^499 + x^494 + x^491 + x^490 + x^489 + x^488 + x^486 + x^484 + x^483 + x^481 + x^480 + x^479 + x^478 + x^476 + x^475 + x^474 + x^472 + x^471 + x^469 + x^468 + x^466 + x^465 + x^464 + x^463 + x^461 + x^459 + x^457 + x^450 + x^449 + x^448 + x^446 + x^445 + x^439 + x^438 + x^437 + x^436 + x^435 + x^433 + x^431 + x^430 + x^429 + x^427 + x^426 + x^423 + x^418 + x^417 + x^415 + x^413 + x^403 + x^402 + x^401 + x^399 + x^398 + x^397 + x^395 + x^394 + x^393 + x^386 + x^382 + x^379 + x^378 + x^377 + x^368 + x^367 + x^366 + x^364 + x^363 + x^362 + x^360 + x^359 + x^356 + x^354 + x^352 + x^344 + x^343 + x^342 + x^341 + x^340 + x^337 + x^336 + x^331 + x^330 + x^325 + x^323 + x^321 + x^320 + x^319 + x^318 + x^315 + x^313 + x^311 + x^310 + x^308 + x^306 + x^303 + x^302 + x^300 + x^298 + x^297 + x^296 + x^294 + x^293 + x^292 + x^291 + x^290 + x^288 + x^286 + x^285 + x^283 + x^278 + x^277 + x^275 + x^274 + x^273 + x^271 + x^269 + x^268 + x^267 + x^265 + x^261 + x^260 + x^259 + x^257 + x^256 + x^255 + x^254 + x^251 + x^249 + x^248 + x^245 + x^244 + x^242 + x^240 + x^238 + x^237 + x^236 + x^231 + x^229 + x^227 + x^225 + x^224 + x^223 + x^222 + x^221 + x^216 + x^215 + x^213 + x^211 + x^205 + x^202 + x^196 + x^194 + x^193 + x^192 + x^191 + x^190 + x^184 + x^180 + x^179 + x^178 + x^177 + x^176 + x^175 + x^174 + x^171 + x^169 + x^164 + x^162 + x^161 + x^160 + x^155 + x^153 + x^149 + x^143 + x^142 + x^139 + x^138 + x^135 + x^134 + x^133 + x^132 + x^130 + x^128 + x^127 + x^126 + x^124 + x^123 + x^120 + x^118 + x^117 + x^115 + x^114 + x^113 + x^112 + x^111 + x^109 + x^108 + x^107 + x^104 + x^103 + x^101 + x^98 + x^95 + x^93 + x^92 + x^89 + x^88 + x^86 + x^82 + x^80 + x^76 + x^75 + x^69 + x^67 + x^65 + x^64 + x^60 + x^58 + x^57 + x^55 + x^53 + x^50 + x^49 + x^47 + x^46 + x^45 + x^44 + x^43 + x^42 + x^39 + x^37 + x^36 + x^35 + x^34 + x^32 + x^24 + x^22 + x^20 + 1 + +33-20-8 359 x^928 + x^898 + x^874 + x^870 + x^845 + x^840 + x^824 + x^808 + x^794 + x^790 + x^787 + x^786 + x^780 + x^778 + x^764 + x^762 + x^761 + x^760 + x^756 + x^752 + x^741 + x^737 + x^736 + x^734 + x^731 + x^729 + x^728 + x^726 + x^720 + x^716 + x^712 + x^710 + x^707 + x^706 + x^703 + x^700 + x^697 + x^696 + x^688 + x^686 + x^683 + x^682 + x^681 + x^675 + x^671 + x^669 + x^667 + x^666 + x^660 + x^658 + x^657 + x^654 + x^653 + x^652 + x^649 + x^647 + x^646 + x^640 + x^639 + x^637 + x^636 + x^634 + x^629 + x^628 + x^627 + x^625 + x^624 + x^618 + x^613 + x^609 + x^608 + x^607 + x^600 + x^598 + x^597 + x^596 + x^595 + x^594 + x^593 + x^592 + x^591 + x^589 + x^587 + x^586 + x^585 + x^583 + x^582 + x^581 + x^580 + x^579 + x^577 + x^570 + x^567 + x^562 + x^554 + x^553 + x^550 + x^549 + x^548 + x^547 + x^546 + x^545 + x^544 + x^542 + x^539 + x^537 + x^535 + x^533 + x^531 + x^530 + x^524 + x^523 + x^521 + x^520 + x^517 + x^516 + x^512 + x^511 + x^504 + x^503 + x^502 + x^501 + x^497 + x^494 + x^492 + x^490 + x^489 + x^486 + x^480 + x^478 + x^477 + x^476 + x^475 + x^472 + x^467 + x^465 + x^463 + x^462 + x^461 + x^460 + x^459 + x^458 + x^456 + x^452 + x^449 + x^447 + x^446 + x^442 + x^440 + x^439 + x^438 + x^437 + x^436 + x^435 + x^434 + x^433 + x^432 + x^431 + x^430 + x^429 + x^426 + x^425 + x^423 + x^422 + x^421 + x^420 + x^415 + x^414 + x^413 + x^412 + x^411 + x^410 + x^408 + x^404 + x^402 + x^400 + x^399 + x^398 + x^395 + x^394 + x^392 + x^390 + x^387 + x^386 + x^385 + x^383 + x^379 + x^377 + x^376 + x^374 + x^373 + x^372 + x^371 + x^368 + x^366 + x^365 + x^362 + x^361 + x^360 + x^359 + x^358 + x^350 + x^347 + x^342 + x^341 + x^338 + x^337 + x^335 + x^331 + x^330 + x^326 + x^325 + x^322 + x^321 + x^320 + x^319 + x^318 + x^317 + x^312 + x^311 + x^308 + x^307 + x^305 + x^303 + x^301 + x^298 + x^295 + x^292 + x^289 + x^286 + x^282 + x^280 + x^277 + x^273 + x^271 + x^270 + x^265 + x^263 + x^262 + x^261 + x^260 + x^257 + x^253 + x^252 + x^251 + x^249 + x^248 + x^247 + x^246 + x^245 + x^243 + x^240 + x^239 + x^236 + x^234 + x^230 + x^228 + x^226 + x^225 + x^224 + x^222 + x^220 + x^219 + x^218 + x^215 + x^214 + x^213 + x^210 + x^205 + x^203 + x^201 + x^195 + x^194 + x^192 + x^191 + x^187 + x^185 + x^184 + x^183 + x^181 + x^178 + x^176 + x^174 + x^173 + x^172 + x^171 + x^170 + x^168 + x^165 + x^164 + x^163 + x^160 + x^157 + x^155 + x^154 + x^153 + x^152 + x^151 + x^150 + x^149 + x^148 + x^139 + x^138 + x^135 + x^134 + x^132 + x^131 + x^129 + x^128 + x^127 + x^126 + x^125 + x^122 + x^121 + x^119 + x^118 + x^115 + x^114 + x^111 + x^110 + x^106 + x^103 + x^101 + x^99 + x^98 + x^95 + x^91 + x^90 + x^89 + x^83 + x^78 + x^77 + x^75 + x^74 + x^70 + x^65 + x^64 + x^61 + x^56 + x^55 + x^52 + x^51 + x^49 + x^46 + x^41 + x^38 + x^36 + x^35 + x^34 + x^32 + x^29 + x^28 + x^27 + x^23 + x^16 + x^12 + x^8 + x^4 + 1 + +6-42-43 359 x^928 + x^898 + x^877 + x^870 + x^847 + x^846 + x^844 + x^822 + x^808 + x^799 + x^794 + x^793 + x^790 + x^786 + x^778 + x^775 + x^774 + x^772 + x^771 + x^769 + x^766 + x^764 + x^750 + x^749 + x^748 + x^747 + x^744 + x^743 + x^742 + x^739 + x^736 + x^724 + x^721 + x^718 + x^715 + x^712 + x^710 + x^706 + x^702 + x^699 + x^695 + x^693 + x^692 + x^689 + x^687 + x^685 + x^684 + x^683 + x^682 + x^679 + x^674 + x^671 + x^670 + x^668 + x^667 + x^664 + x^662 + x^653 + x^652 + x^650 + x^649 + x^645 + x^644 + x^642 + x^639 + x^638 + x^637 + x^634 + x^633 + x^630 + x^624 + x^622 + x^619 + x^618 + x^616 + x^614 + x^613 + x^611 + x^610 + x^609 + x^608 + x^603 + x^601 + x^598 + x^595 + x^592 + x^589 + x^587 + x^586 + x^585 + x^584 + x^583 + x^582 + x^578 + x^574 + x^572 + x^568 + x^567 + x^565 + x^562 + x^561 + x^559 + x^558 + x^557 + x^555 + x^554 + x^550 + x^549 + x^547 + x^545 + x^541 + x^534 + x^528 + x^527 + x^523 + x^520 + x^518 + x^515 + x^510 + x^503 + x^502 + x^501 + x^500 + x^499 + x^498 + x^497 + x^496 + x^495 + x^494 + x^492 + x^489 + x^486 + x^483 + x^482 + x^479 + x^476 + x^475 + x^473 + x^472 + x^468 + x^467 + x^465 + x^464 + x^463 + x^461 + x^459 + x^458 + x^455 + x^452 + x^451 + x^450 + x^449 + x^448 + x^446 + x^445 + x^444 + x^443 + x^441 + x^440 + x^437 + x^434 + x^431 + x^430 + x^429 + x^425 + x^423 + x^422 + x^421 + x^419 + x^418 + x^417 + x^411 + x^408 + x^406 + x^405 + x^403 + x^401 + x^398 + x^397 + x^396 + x^393 + x^390 + x^382 + x^378 + x^376 + x^373 + x^372 + x^370 + x^369 + x^366 + x^361 + x^359 + x^358 + x^356 + x^354 + x^352 + x^351 + x^350 + x^349 + x^348 + x^347 + x^344 + x^340 + x^339 + x^336 + x^335 + x^333 + x^332 + x^329 + x^325 + x^324 + x^322 + x^321 + x^317 + x^316 + x^311 + x^310 + x^309 + x^303 + x^302 + x^301 + x^299 + x^298 + x^295 + x^294 + x^290 + x^289 + x^288 + x^287 + x^283 + x^282 + x^280 + x^279 + x^278 + x^277 + x^272 + x^271 + x^270 + x^268 + x^263 + x^261 + x^260 + x^259 + x^257 + x^256 + x^254 + x^250 + x^248 + x^247 + x^245 + x^244 + x^243 + x^241 + x^239 + x^238 + x^237 + x^235 + x^231 + x^230 + x^229 + x^228 + x^226 + x^225 + x^224 + x^221 + x^219 + x^217 + x^214 + x^210 + x^209 + x^208 + x^205 + x^204 + x^202 + x^199 + x^196 + x^191 + x^190 + x^189 + x^186 + x^184 + x^183 + x^180 + x^179 + x^178 + x^177 + x^175 + x^174 + x^170 + x^168 + x^167 + x^166 + x^165 + x^164 + x^162 + x^161 + x^160 + x^159 + x^157 + x^154 + x^153 + x^151 + x^143 + x^137 + x^135 + x^134 + x^129 + x^127 + x^126 + x^122 + x^121 + x^119 + x^118 + x^113 + x^111 + x^106 + x^104 + x^101 + x^100 + x^99 + x^98 + x^96 + x^92 + x^91 + x^90 + x^83 + x^82 + x^81 + x^79 + x^75 + x^74 + x^72 + x^71 + x^68 + x^64 + x^63 + x^60 + x^57 + x^56 + x^54 + x^52 + x^51 + x^49 + x^48 + x^46 + x^45 + x^43 + x^42 + x^39 + x^38 + x^37 + x^36 + x^32 + x^28 + x^26 + x^24 + x^16 + x^14 + 1 + +33-22-4 361 x^928 + x^898 + x^870 + x^832 + x^822 + x^812 + x^808 + x^805 + x^802 + x^792 + x^784 + x^778 + x^775 + x^774 + x^762 + x^757 + x^756 + x^754 + x^746 + x^744 + x^740 + x^738 + x^736 + x^732 + x^728 + x^726 + x^720 + x^716 + x^711 + x^701 + x^697 + x^692 + x^684 + x^682 + x^680 + x^678 + x^673 + x^672 + x^671 + x^670 + x^658 + x^655 + x^653 + x^652 + x^651 + x^644 + x^637 + x^636 + x^630 + x^626 + x^622 + x^615 + x^613 + x^612 + x^611 + x^610 + x^608 + x^607 + x^606 + x^600 + x^599 + x^594 + x^593 + x^592 + x^590 + x^589 + x^588 + x^587 + x^586 + x^585 + x^583 + x^580 + x^578 + x^577 + x^574 + x^569 + x^567 + x^564 + x^562 + x^561 + x^559 + x^554 + x^553 + x^552 + x^549 + x^548 + x^544 + x^543 + x^538 + x^537 + x^536 + x^535 + x^533 + x^530 + x^525 + x^524 + x^523 + x^522 + x^521 + x^519 + x^517 + x^516 + x^515 + x^514 + x^511 + x^507 + x^506 + x^500 + x^499 + x^498 + x^496 + x^494 + x^487 + x^486 + x^485 + x^483 + x^480 + x^473 + x^468 + x^466 + x^465 + x^464 + x^463 + x^461 + x^460 + x^454 + x^452 + x^448 + x^447 + x^445 + x^441 + x^440 + x^439 + x^438 + x^436 + x^435 + x^434 + x^433 + x^429 + x^427 + x^425 + x^423 + x^421 + x^420 + x^418 + x^416 + x^415 + x^413 + x^411 + x^410 + x^409 + x^408 + x^404 + x^402 + x^401 + x^399 + x^397 + x^396 + x^393 + x^391 + x^390 + x^388 + x^387 + x^386 + x^383 + x^381 + x^380 + x^379 + x^376 + x^375 + x^374 + x^371 + x^370 + x^369 + x^368 + x^366 + x^364 + x^363 + x^361 + x^359 + x^356 + x^355 + x^354 + x^353 + x^350 + x^349 + x^344 + x^341 + x^338 + x^337 + x^336 + x^335 + x^332 + x^330 + x^329 + x^328 + x^327 + x^323 + x^322 + x^319 + x^318 + x^315 + x^314 + x^313 + x^309 + x^308 + x^307 + x^306 + x^304 + x^303 + x^301 + x^300 + x^298 + x^296 + x^295 + x^294 + x^292 + x^290 + x^289 + x^288 + x^286 + x^284 + x^282 + x^280 + x^279 + x^278 + x^275 + x^273 + x^271 + x^268 + x^266 + x^262 + x^261 + x^259 + x^258 + x^256 + x^255 + x^254 + x^253 + x^250 + x^245 + x^241 + x^240 + x^239 + x^237 + x^235 + x^234 + x^232 + x^231 + x^229 + x^226 + x^225 + x^221 + x^220 + x^218 + x^217 + x^215 + x^213 + x^211 + x^209 + x^206 + x^205 + x^201 + x^200 + x^196 + x^195 + x^192 + x^189 + x^188 + x^187 + x^186 + x^184 + x^183 + x^181 + x^180 + x^178 + x^177 + x^176 + x^175 + x^174 + x^172 + x^171 + x^169 + x^166 + x^165 + x^162 + x^160 + x^157 + x^154 + x^152 + x^150 + x^149 + x^145 + x^141 + x^138 + x^137 + x^133 + x^131 + x^129 + x^127 + x^125 + x^124 + x^123 + x^120 + x^118 + x^117 + x^115 + x^114 + x^112 + x^109 + x^108 + x^105 + x^102 + x^100 + x^99 + x^97 + x^96 + x^91 + x^90 + x^88 + x^87 + x^86 + x^85 + x^82 + x^80 + x^79 + x^77 + x^75 + x^74 + x^70 + x^66 + x^65 + x^63 + x^62 + x^61 + x^60 + x^59 + x^54 + x^52 + x^50 + x^49 + x^46 + x^42 + x^41 + x^40 + x^39 + x^38 + x^37 + x^36 + x^35 + x^33 + x^31 + x^27 + x^26 + x^22 + x^21 + x^19 + x^16 + x^14 + x^12 + 1 + +46-48-7 361 x^928 + x^898 + x^870 + x^862 + x^848 + x^837 + x^829 + x^820 + x^815 + x^808 + x^807 + x^806 + x^802 + x^799 + x^798 + x^795 + x^790 + x^788 + x^787 + x^785 + x^776 + x^769 + x^765 + x^762 + x^760 + x^756 + x^755 + x^740 + x^739 + x^737 + x^732 + x^730 + x^727 + x^725 + x^717 + x^715 + x^714 + x^710 + x^709 + x^706 + x^704 + x^695 + x^688 + x^687 + x^686 + x^684 + x^682 + x^680 + x^679 + x^678 + x^676 + x^675 + x^667 + x^664 + x^661 + x^658 + x^657 + x^655 + x^654 + x^653 + x^652 + x^650 + x^649 + x^647 + x^646 + x^644 + x^643 + x^638 + x^637 + x^636 + x^634 + x^633 + x^627 + x^624 + x^623 + x^622 + x^620 + x^619 + x^618 + x^615 + x^612 + x^611 + x^606 + x^604 + x^601 + x^594 + x^591 + x^590 + x^589 + x^588 + x^587 + x^585 + x^583 + x^580 + x^575 + x^574 + x^572 + x^571 + x^569 + x^568 + x^567 + x^564 + x^563 + x^558 + x^557 + x^556 + x^552 + x^551 + x^546 + x^544 + x^542 + x^541 + x^535 + x^534 + x^532 + x^531 + x^530 + x^529 + x^528 + x^526 + x^525 + x^523 + x^521 + x^519 + x^518 + x^516 + x^514 + x^513 + x^506 + x^505 + x^504 + x^499 + x^498 + x^497 + x^496 + x^495 + x^493 + x^491 + x^490 + x^488 + x^487 + x^486 + x^485 + x^484 + x^481 + x^480 + x^477 + x^476 + x^473 + x^472 + x^471 + x^466 + x^465 + x^463 + x^462 + x^461 + x^460 + x^459 + x^455 + x^451 + x^449 + x^448 + x^447 + x^446 + x^444 + x^443 + x^441 + x^440 + x^439 + x^437 + x^431 + x^429 + x^428 + x^426 + x^424 + x^421 + x^420 + x^419 + x^418 + x^417 + x^416 + x^414 + x^411 + x^410 + x^408 + x^407 + x^406 + x^405 + x^403 + x^402 + x^401 + x^400 + x^397 + x^394 + x^392 + x^390 + x^388 + x^382 + x^381 + x^379 + x^378 + x^377 + x^376 + x^374 + x^367 + x^364 + x^363 + x^360 + x^357 + x^356 + x^354 + x^353 + x^352 + x^351 + x^350 + x^348 + x^343 + x^342 + x^341 + x^337 + x^334 + x^333 + x^330 + x^326 + x^316 + x^313 + x^309 + x^306 + x^305 + x^300 + x^299 + x^296 + x^294 + x^293 + x^292 + x^287 + x^285 + x^282 + x^281 + x^278 + x^276 + x^275 + x^271 + x^269 + x^267 + x^263 + x^262 + x^261 + x^260 + x^257 + x^256 + x^255 + x^253 + x^249 + x^248 + x^244 + x^243 + x^242 + x^240 + x^239 + x^238 + x^236 + x^235 + x^234 + x^231 + x^226 + x^224 + x^221 + x^217 + x^216 + x^214 + x^211 + x^210 + x^209 + x^208 + x^206 + x^205 + x^202 + x^200 + x^197 + x^196 + x^195 + x^190 + x^186 + x^184 + x^183 + x^181 + x^176 + x^172 + x^171 + x^167 + x^166 + x^165 + x^164 + x^159 + x^157 + x^154 + x^153 + x^151 + x^150 + x^146 + x^145 + x^144 + x^142 + x^141 + x^140 + x^139 + x^137 + x^136 + x^134 + x^131 + x^130 + x^129 + x^125 + x^124 + x^123 + x^120 + x^118 + x^117 + x^111 + x^110 + x^108 + x^107 + x^106 + x^105 + x^98 + x^96 + x^95 + x^94 + x^92 + x^88 + x^83 + x^81 + x^78 + x^76 + x^72 + x^71 + x^67 + x^66 + x^65 + x^63 + x^60 + x^58 + x^57 + x^56 + x^52 + x^51 + x^49 + x^48 + x^46 + x^44 + x^40 + x^38 + x^34 + x^33 + x^32 + x^31 + x^30 + x^28 + x^20 + 1 + +49-44-52 361 x^928 + x^898 + x^870 + x^864 + x^842 + x^837 + x^834 + x^812 + x^810 + x^808 + x^804 + x^788 + x^786 + x^783 + x^782 + x^780 + x^778 + x^774 + x^758 + x^756 + x^753 + x^752 + x^734 + x^732 + x^729 + x^727 + x^724 + x^723 + x^722 + x^717 + x^704 + x^702 + x^699 + x^696 + x^693 + x^692 + x^690 + x^680 + x^678 + x^672 + x^667 + x^665 + x^663 + x^662 + x^661 + x^660 + x^650 + x^646 + x^637 + x^636 + x^633 + x^631 + x^626 + x^621 + x^618 + x^616 + x^614 + x^612 + x^611 + x^610 + x^609 + x^605 + x^603 + x^602 + x^601 + x^599 + x^597 + x^596 + x^594 + x^589 + x^588 + x^586 + x^576 + x^575 + x^573 + x^572 + x^571 + x^569 + x^568 + x^565 + x^562 + x^561 + x^560 + x^558 + x^557 + x^555 + x^554 + x^551 + x^549 + x^548 + x^543 + x^542 + x^539 + x^536 + x^532 + x^529 + x^528 + x^527 + x^526 + x^525 + x^518 + x^517 + x^514 + x^513 + x^512 + x^511 + x^509 + x^508 + x^506 + x^505 + x^499 + x^496 + x^495 + x^493 + x^492 + x^491 + x^489 + x^486 + x^483 + x^482 + x^479 + x^477 + x^476 + x^474 + x^468 + x^467 + x^462 + x^461 + x^457 + x^456 + x^454 + x^453 + x^452 + x^451 + x^449 + x^444 + x^442 + x^441 + x^439 + x^438 + x^437 + x^436 + x^432 + x^431 + x^429 + x^428 + x^427 + x^425 + x^424 + x^423 + x^422 + x^420 + x^419 + x^418 + x^416 + x^414 + x^413 + x^409 + x^407 + x^406 + x^404 + x^403 + x^402 + x^401 + x^395 + x^394 + x^393 + x^392 + x^390 + x^389 + x^387 + x^386 + x^384 + x^380 + x^378 + x^377 + x^376 + x^372 + x^371 + x^370 + x^369 + x^365 + x^363 + x^362 + x^361 + x^358 + x^357 + x^356 + x^355 + x^352 + x^351 + x^350 + x^347 + x^346 + x^344 + x^343 + x^342 + x^340 + x^339 + x^338 + x^332 + x^331 + x^328 + x^326 + x^325 + x^323 + x^322 + x^321 + x^319 + x^317 + x^316 + x^314 + x^313 + x^309 + x^305 + x^303 + x^301 + x^300 + x^299 + x^298 + x^297 + x^294 + x^292 + x^290 + x^285 + x^284 + x^282 + x^280 + x^279 + x^278 + x^276 + x^272 + x^267 + x^266 + x^265 + x^264 + x^263 + x^262 + x^259 + x^256 + x^254 + x^253 + x^251 + x^250 + x^249 + x^248 + x^247 + x^245 + x^243 + x^241 + x^240 + x^238 + x^234 + x^231 + x^229 + x^228 + x^227 + x^224 + x^222 + x^221 + x^215 + x^214 + x^211 + x^210 + x^209 + x^207 + x^206 + x^205 + x^204 + x^197 + x^196 + x^194 + x^192 + x^191 + x^189 + x^188 + x^187 + x^186 + x^182 + x^180 + x^178 + x^175 + x^172 + x^168 + x^167 + x^166 + x^163 + x^162 + x^158 + x^154 + x^153 + x^150 + x^148 + x^146 + x^145 + x^144 + x^138 + x^136 + x^132 + x^131 + x^130 + x^129 + x^124 + x^122 + x^120 + x^116 + x^115 + x^113 + x^112 + x^109 + x^108 + x^107 + x^106 + x^103 + x^102 + x^101 + x^99 + x^98 + x^97 + x^96 + x^94 + x^92 + x^90 + x^86 + x^85 + x^84 + x^83 + x^82 + x^80 + x^79 + x^77 + x^76 + x^65 + x^60 + x^59 + x^57 + x^54 + x^52 + x^47 + x^45 + x^44 + x^42 + x^38 + x^37 + x^34 + x^33 + x^31 + x^29 + x^26 + x^25 + x^24 + x^22 + x^21 + x^19 + x^18 + x^17 + x^15 + x^14 + x^12 + x^10 + 1 + +34-42-39 365 x^928 + x^898 + x^870 + x^860 + x^850 + x^840 + x^831 + x^830 + x^811 + x^808 + x^801 + x^800 + x^792 + x^790 + x^780 + x^778 + x^771 + x^770 + x^762 + x^753 + x^744 + x^742 + x^740 + x^734 + x^720 + x^715 + x^712 + x^710 + x^703 + x^702 + x^694 + x^693 + x^691 + x^688 + x^685 + x^682 + x^680 + x^676 + x^674 + x^672 + x^664 + x^663 + x^662 + x^660 + x^658 + x^657 + x^656 + x^653 + x^652 + x^650 + x^647 + x^646 + x^645 + x^643 + x^637 + x^634 + x^628 + x^627 + x^623 + x^622 + x^618 + x^617 + x^615 + x^614 + x^608 + x^607 + x^604 + x^603 + x^600 + x^597 + x^593 + x^592 + x^589 + x^588 + x^587 + x^586 + x^585 + x^584 + x^579 + x^578 + x^577 + x^574 + x^570 + x^567 + x^565 + x^563 + x^562 + x^560 + x^558 + x^555 + x^550 + x^549 + x^547 + x^544 + x^542 + x^541 + x^536 + x^533 + x^532 + x^529 + x^527 + x^526 + x^521 + x^520 + x^518 + x^517 + x^516 + x^514 + x^510 + x^509 + x^508 + x^506 + x^504 + x^503 + x^502 + x^501 + x^497 + x^496 + x^489 + x^488 + x^487 + x^486 + x^484 + x^481 + x^480 + x^476 + x^475 + x^473 + x^472 + x^471 + x^470 + x^467 + x^466 + x^463 + x^462 + x^459 + x^458 + x^457 + x^456 + x^453 + x^451 + x^450 + x^446 + x^445 + x^444 + x^443 + x^442 + x^439 + x^438 + x^437 + x^436 + x^433 + x^432 + x^430 + x^428 + x^427 + x^425 + x^421 + x^420 + x^413 + x^412 + x^411 + x^408 + x^406 + x^405 + x^404 + x^403 + x^401 + x^399 + x^398 + x^396 + x^391 + x^388 + x^385 + x^381 + x^378 + x^376 + x^375 + x^374 + x^372 + x^370 + x^368 + x^364 + x^363 + x^360 + x^359 + x^357 + x^354 + x^353 + x^350 + x^349 + x^348 + x^347 + x^346 + x^344 + x^343 + x^342 + x^340 + x^339 + x^337 + x^335 + x^334 + x^333 + x^331 + x^330 + x^329 + x^327 + x^326 + x^325 + x^324 + x^323 + x^321 + x^320 + x^317 + x^312 + x^310 + x^309 + x^303 + x^300 + x^298 + x^296 + x^295 + x^292 + x^291 + x^290 + x^288 + x^287 + x^286 + x^285 + x^284 + x^282 + x^278 + x^277 + x^275 + x^274 + x^273 + x^267 + x^266 + x^265 + x^264 + x^263 + x^262 + x^261 + x^260 + x^257 + x^254 + x^250 + x^249 + x^244 + x^243 + x^242 + x^239 + x^238 + x^237 + x^236 + x^231 + x^230 + x^229 + x^228 + x^226 + x^224 + x^223 + x^222 + x^221 + x^220 + x^217 + x^214 + x^213 + x^212 + x^211 + x^208 + x^207 + x^206 + x^205 + x^204 + x^202 + x^198 + x^197 + x^195 + x^194 + x^192 + x^189 + x^187 + x^186 + x^185 + x^181 + x^180 + x^179 + x^178 + x^175 + x^173 + x^170 + x^168 + x^167 + x^166 + x^165 + x^164 + x^157 + x^155 + x^154 + x^150 + x^149 + x^148 + x^144 + x^143 + x^140 + x^138 + x^137 + x^135 + x^134 + x^132 + x^126 + x^124 + x^123 + x^120 + x^119 + x^117 + x^115 + x^113 + x^105 + x^104 + x^102 + x^100 + x^97 + x^96 + x^94 + x^91 + x^88 + x^87 + x^85 + x^84 + x^83 + x^82 + x^80 + x^79 + x^77 + x^76 + x^69 + x^68 + x^64 + x^61 + x^60 + x^55 + x^53 + x^51 + x^50 + x^48 + x^45 + x^44 + x^43 + x^40 + x^39 + x^38 + x^37 + x^36 + x^34 + x^33 + x^30 + x^22 + x^20 + x^19 + x^18 + x^16 + x^14 + 1 + +34-2-25 367 x^928 + x^898 + x^870 + x^868 + x^866 + x^850 + x^838 + x^832 + x^822 + x^820 + x^808 + x^804 + x^802 + x^797 + x^792 + x^790 + x^788 + x^786 + x^778 + x^770 + x^768 + x^767 + x^760 + x^754 + x^748 + x^747 + x^746 + x^744 + x^738 + x^736 + x^735 + x^728 + x^726 + x^720 + x^717 + x^712 + x^708 + x^696 + x^694 + x^692 + x^690 + x^688 + x^687 + x^685 + x^684 + x^678 + x^677 + x^675 + x^673 + x^669 + x^667 + x^662 + x^658 + x^657 + x^654 + x^652 + x^647 + x^643 + x^642 + x^640 + x^639 + x^638 + x^637 + x^634 + x^633 + x^624 + x^613 + x^612 + x^611 + x^610 + x^609 + x^607 + x^606 + x^603 + x^600 + x^598 + x^597 + x^593 + x^591 + x^590 + x^588 + x^587 + x^586 + x^584 + x^583 + x^582 + x^581 + x^577 + x^576 + x^574 + x^571 + x^570 + x^568 + x^567 + x^564 + x^563 + x^562 + x^560 + x^558 + x^557 + x^554 + x^553 + x^551 + x^549 + x^548 + x^547 + x^546 + x^545 + x^539 + x^538 + x^535 + x^533 + x^531 + x^530 + x^529 + x^527 + x^525 + x^524 + x^523 + x^522 + x^520 + x^517 + x^515 + x^514 + x^511 + x^510 + x^507 + x^506 + x^504 + x^503 + x^500 + x^499 + x^498 + x^496 + x^493 + x^491 + x^490 + x^489 + x^487 + x^485 + x^483 + x^482 + x^480 + x^478 + x^477 + x^474 + x^473 + x^471 + x^469 + x^468 + x^465 + x^464 + x^460 + x^459 + x^457 + x^455 + x^451 + x^450 + x^449 + x^447 + x^446 + x^442 + x^441 + x^438 + x^430 + x^428 + x^427 + x^426 + x^425 + x^421 + x^420 + x^417 + x^416 + x^415 + x^414 + x^411 + x^410 + x^409 + x^406 + x^405 + x^404 + x^402 + x^396 + x^394 + x^393 + x^390 + x^389 + x^388 + x^386 + x^384 + x^382 + x^381 + x^380 + x^379 + x^378 + x^377 + x^376 + x^374 + x^373 + x^372 + x^371 + x^369 + x^366 + x^365 + x^364 + x^363 + x^356 + x^354 + x^352 + x^351 + x^344 + x^342 + x^338 + x^336 + x^335 + x^334 + x^327 + x^323 + x^321 + x^319 + x^318 + x^317 + x^312 + x^308 + x^305 + x^303 + x^302 + x^301 + x^299 + x^298 + x^297 + x^289 + x^286 + x^285 + x^283 + x^277 + x^276 + x^275 + x^272 + x^271 + x^269 + x^268 + x^267 + x^266 + x^263 + x^258 + x^255 + x^254 + x^252 + x^251 + x^247 + x^246 + x^235 + x^233 + x^232 + x^229 + x^226 + x^225 + x^223 + x^220 + x^219 + x^216 + x^212 + x^210 + x^209 + x^208 + x^206 + x^205 + x^204 + x^202 + x^201 + x^200 + x^198 + x^196 + x^193 + x^190 + x^187 + x^185 + x^184 + x^182 + x^180 + x^179 + x^174 + x^173 + x^172 + x^171 + x^169 + x^167 + x^166 + x^165 + x^164 + x^159 + x^158 + x^156 + x^155 + x^154 + x^151 + x^150 + x^149 + x^148 + x^147 + x^146 + x^144 + x^143 + x^141 + x^137 + x^133 + x^132 + x^131 + x^130 + x^129 + x^126 + x^124 + x^123 + x^122 + x^118 + x^112 + x^111 + x^109 + x^108 + x^107 + x^105 + x^104 + x^103 + x^102 + x^99 + x^96 + x^93 + x^92 + x^91 + x^89 + x^86 + x^84 + x^83 + x^82 + x^81 + x^79 + x^77 + x^76 + x^75 + x^71 + x^70 + x^67 + x^65 + x^63 + x^53 + x^49 + x^44 + x^40 + x^38 + x^37 + x^36 + x^35 + x^34 + x^32 + x^31 + x^28 + x^26 + x^25 + x^24 + x^22 + x^18 + x^16 + x^10 + x^8 + x^4 + 1 + +34-39-1 367 x^928 + x^898 + x^894 + x^874 + x^870 + x^864 + x^855 + x^844 + x^840 + x^835 + x^834 + x^825 + x^820 + x^815 + x^814 + x^808 + x^805 + x^804 + x^801 + x^800 + x^796 + x^790 + x^785 + x^784 + x^778 + x^776 + x^770 + x^766 + x^761 + x^760 + x^756 + x^754 + x^751 + x^750 + x^746 + x^741 + x^740 + x^735 + x^732 + x^731 + x^730 + x^727 + x^724 + x^722 + x^721 + x^720 + x^716 + x^715 + x^712 + x^706 + x^705 + x^700 + x^697 + x^694 + x^692 + x^688 + x^686 + x^685 + x^682 + x^677 + x^676 + x^672 + x^667 + x^664 + x^657 + x^655 + x^653 + x^651 + x^650 + x^648 + x^642 + x^638 + x^632 + x^631 + x^630 + x^628 + x^627 + x^625 + x^622 + x^620 + x^618 + x^617 + x^616 + x^612 + x^611 + x^602 + x^600 + x^598 + x^597 + x^591 + x^588 + x^586 + x^582 + x^581 + x^579 + x^578 + x^577 + x^574 + x^573 + x^572 + x^568 + x^566 + x^564 + x^560 + x^552 + x^541 + x^538 + x^537 + x^535 + x^534 + x^532 + x^531 + x^529 + x^528 + x^523 + x^522 + x^521 + x^519 + x^518 + x^516 + x^515 + x^510 + x^509 + x^508 + x^506 + x^504 + x^500 + x^499 + x^498 + x^497 + x^495 + x^492 + x^490 + x^489 + x^488 + x^484 + x^482 + x^477 + x^475 + x^474 + x^472 + x^471 + x^470 + x^469 + x^467 + x^466 + x^465 + x^464 + x^462 + x^461 + x^460 + x^457 + x^455 + x^454 + x^453 + x^448 + x^447 + x^446 + x^444 + x^443 + x^442 + x^441 + x^440 + x^437 + x^431 + x^430 + x^429 + x^426 + x^424 + x^423 + x^421 + x^419 + x^418 + x^417 + x^416 + x^412 + x^411 + x^409 + x^408 + x^407 + x^406 + x^405 + x^403 + x^402 + x^397 + x^395 + x^394 + x^389 + x^388 + x^384 + x^383 + x^382 + x^380 + x^379 + x^377 + x^376 + x^375 + x^374 + x^373 + x^372 + x^370 + x^369 + x^368 + x^367 + x^366 + x^361 + x^360 + x^359 + x^357 + x^355 + x^353 + x^350 + x^349 + x^344 + x^342 + x^341 + x^340 + x^338 + x^337 + x^336 + x^335 + x^334 + x^333 + x^332 + x^331 + x^329 + x^328 + x^326 + x^321 + x^320 + x^318 + x^317 + x^316 + x^313 + x^311 + x^310 + x^309 + x^306 + x^303 + x^302 + x^300 + x^299 + x^296 + x^295 + x^294 + x^293 + x^292 + x^291 + x^288 + x^287 + x^286 + x^284 + x^279 + x^276 + x^274 + x^270 + x^269 + x^268 + x^265 + x^264 + x^262 + x^259 + x^257 + x^256 + x^255 + x^251 + x^248 + x^247 + x^245 + x^244 + x^243 + x^240 + x^237 + x^234 + x^232 + x^230 + x^227 + x^224 + x^223 + x^219 + x^214 + x^212 + x^206 + x^204 + x^201 + x^198 + x^197 + x^194 + x^192 + x^190 + x^189 + x^186 + x^185 + x^182 + x^179 + x^178 + x^174 + x^172 + x^171 + x^168 + x^166 + x^163 + x^162 + x^161 + x^159 + x^158 + x^154 + x^153 + x^152 + x^151 + x^149 + x^148 + x^146 + x^143 + x^142 + x^140 + x^138 + x^137 + x^135 + x^134 + x^133 + x^132 + x^131 + x^128 + x^126 + x^125 + x^124 + x^123 + x^119 + x^118 + x^116 + x^115 + x^109 + x^108 + x^106 + x^105 + x^103 + x^100 + x^98 + x^91 + x^89 + x^86 + x^84 + x^83 + x^81 + x^79 + x^78 + x^70 + x^69 + x^67 + x^66 + x^65 + x^59 + x^58 + x^56 + x^48 + x^45 + x^43 + x^42 + x^38 + x^37 + x^34 + x^29 + x^26 + x^21 + x^10 + 1 + +8-38-53 367 x^928 + x^898 + x^870 + x^843 + x^838 + x^833 + x^830 + x^820 + x^816 + x^811 + x^808 + x^800 + x^786 + x^781 + x^778 + x^773 + x^768 + x^760 + x^756 + x^752 + x^750 + x^743 + x^742 + x^738 + x^733 + x^732 + x^730 + x^728 + x^726 + x^723 + x^720 + x^718 + x^711 + x^710 + x^708 + x^700 + x^699 + x^698 + x^696 + x^695 + x^693 + x^691 + x^690 + x^688 + x^686 + x^685 + x^683 + x^681 + x^672 + x^670 + x^667 + x^666 + x^665 + x^664 + x^663 + x^662 + x^661 + x^658 + x^657 + x^655 + x^654 + x^650 + x^648 + x^646 + x^645 + x^644 + x^642 + x^640 + x^639 + x^638 + x^636 + x^634 + x^626 + x^622 + x^620 + x^617 + x^615 + x^614 + x^613 + x^612 + x^606 + x^605 + x^603 + x^600 + x^597 + x^596 + x^592 + x^591 + x^590 + x^589 + x^588 + x^585 + x^584 + x^581 + x^578 + x^577 + x^576 + x^575 + x^574 + x^572 + x^569 + x^568 + x^564 + x^562 + x^561 + x^556 + x^554 + x^551 + x^549 + x^544 + x^543 + x^542 + x^540 + x^539 + x^538 + x^533 + x^532 + x^530 + x^529 + x^527 + x^526 + x^524 + x^521 + x^520 + x^517 + x^515 + x^514 + x^510 + x^507 + x^504 + x^503 + x^502 + x^501 + x^500 + x^499 + x^498 + x^497 + x^495 + x^493 + x^489 + x^487 + x^486 + x^483 + x^482 + x^480 + x^477 + x^474 + x^471 + x^469 + x^468 + x^463 + x^461 + x^459 + x^456 + x^453 + x^450 + x^447 + x^443 + x^442 + x^441 + x^439 + x^438 + x^437 + x^436 + x^435 + x^431 + x^424 + x^422 + x^415 + x^411 + x^410 + x^409 + x^407 + x^406 + x^405 + x^404 + x^401 + x^400 + x^396 + x^395 + x^392 + x^391 + x^389 + x^388 + x^385 + x^382 + x^381 + x^380 + x^379 + x^377 + x^374 + x^369 + x^368 + x^367 + x^366 + x^365 + x^364 + x^359 + x^357 + x^353 + x^351 + x^349 + x^345 + x^344 + x^342 + x^341 + x^340 + x^339 + x^337 + x^335 + x^334 + x^333 + x^331 + x^330 + x^329 + x^327 + x^326 + x^324 + x^323 + x^322 + x^320 + x^319 + x^318 + x^314 + x^313 + x^312 + x^311 + x^309 + x^307 + x^304 + x^303 + x^300 + x^298 + x^294 + x^288 + x^285 + x^284 + x^281 + x^280 + x^279 + x^277 + x^276 + x^275 + x^273 + x^271 + x^270 + x^269 + x^268 + x^266 + x^261 + x^254 + x^253 + x^251 + x^250 + x^249 + x^248 + x^247 + x^245 + x^244 + x^242 + x^240 + x^238 + x^236 + x^233 + x^231 + x^230 + x^227 + x^226 + x^224 + x^223 + x^222 + x^221 + x^220 + x^219 + x^218 + x^216 + x^215 + x^213 + x^212 + x^209 + x^208 + x^207 + x^202 + x^198 + x^195 + x^189 + x^186 + x^183 + x^182 + x^180 + x^179 + x^177 + x^175 + x^172 + x^170 + x^168 + x^167 + x^159 + x^158 + x^154 + x^153 + x^150 + x^149 + x^144 + x^141 + x^140 + x^138 + x^136 + x^135 + x^134 + x^133 + x^132 + x^130 + x^129 + x^126 + x^125 + x^124 + x^123 + x^122 + x^121 + x^117 + x^116 + x^114 + x^113 + x^112 + x^111 + x^110 + x^108 + x^106 + x^105 + x^102 + x^98 + x^97 + x^96 + x^95 + x^92 + x^91 + x^90 + x^87 + x^79 + x^77 + x^74 + x^73 + x^72 + x^69 + x^64 + x^61 + x^60 + x^59 + x^58 + x^57 + x^56 + x^55 + x^53 + x^50 + x^49 + x^47 + x^46 + x^44 + x^43 + x^36 + x^35 + x^30 + x^29 + x^22 + x^16 + 1 + +40-6-51 369 x^928 + x^898 + x^870 + x^859 + x^840 + x^830 + x^819 + x^790 + x^780 + x^779 + x^771 + x^770 + x^768 + x^761 + x^760 + x^757 + x^752 + x^750 + x^748 + x^742 + x^741 + x^740 + x^739 + x^738 + x^732 + x^728 + x^727 + x^721 + x^719 + x^718 + x^717 + x^712 + x^709 + x^708 + x^702 + x^699 + x^692 + x^690 + x^687 + x^684 + x^678 + x^677 + x^673 + x^672 + x^671 + x^668 + x^666 + x^659 + x^658 + x^657 + x^655 + x^654 + x^653 + x^652 + x^651 + x^648 + x^647 + x^645 + x^644 + x^643 + x^637 + x^636 + x^634 + x^632 + x^631 + x^624 + x^622 + x^621 + x^620 + x^616 + x^614 + x^613 + x^612 + x^611 + x^609 + x^608 + x^607 + x^598 + x^596 + x^594 + x^593 + x^592 + x^591 + x^588 + x^585 + x^583 + x^579 + x^578 + x^576 + x^575 + x^574 + x^573 + x^572 + x^571 + x^570 + x^567 + x^566 + x^563 + x^562 + x^560 + x^557 + x^555 + x^554 + x^549 + x^547 + x^546 + x^544 + x^542 + x^538 + x^537 + x^535 + x^534 + x^533 + x^531 + x^529 + x^527 + x^526 + x^522 + x^520 + x^518 + x^517 + x^515 + x^514 + x^513 + x^511 + x^510 + x^509 + x^508 + x^505 + x^504 + x^500 + x^497 + x^496 + x^494 + x^492 + x^481 + x^477 + x^475 + x^473 + x^471 + x^470 + x^468 + x^467 + x^466 + x^465 + x^464 + x^463 + x^462 + x^460 + x^457 + x^456 + x^453 + x^451 + x^450 + x^449 + x^447 + x^446 + x^444 + x^443 + x^441 + x^436 + x^435 + x^434 + x^429 + x^428 + x^427 + x^423 + x^422 + x^421 + x^419 + x^418 + x^416 + x^414 + x^413 + x^412 + x^410 + x^408 + x^406 + x^405 + x^402 + x^401 + x^400 + x^399 + x^398 + x^396 + x^395 + x^394 + x^393 + x^391 + x^390 + x^388 + x^385 + x^384 + x^382 + x^381 + x^379 + x^378 + x^377 + x^376 + x^375 + x^374 + x^373 + x^372 + x^371 + x^370 + x^367 + x^366 + x^365 + x^363 + x^360 + x^358 + x^355 + x^354 + x^353 + x^352 + x^350 + x^347 + x^344 + x^342 + x^340 + x^339 + x^338 + x^335 + x^334 + x^331 + x^330 + x^328 + x^324 + x^323 + x^322 + x^321 + x^318 + x^316 + x^315 + x^313 + x^307 + x^306 + x^304 + x^303 + x^301 + x^300 + x^299 + x^298 + x^297 + x^290 + x^287 + x^285 + x^283 + x^282 + x^281 + x^280 + x^279 + x^278 + x^276 + x^275 + x^274 + x^270 + x^264 + x^263 + x^262 + x^261 + x^259 + x^258 + x^254 + x^253 + x^252 + x^244 + x^242 + x^241 + x^240 + x^239 + x^237 + x^236 + x^235 + x^230 + x^228 + x^224 + x^220 + x^217 + x^214 + x^210 + x^209 + x^208 + x^206 + x^203 + x^202 + x^201 + x^200 + x^199 + x^196 + x^195 + x^194 + x^192 + x^189 + x^188 + x^186 + x^185 + x^184 + x^181 + x^176 + x^175 + x^173 + x^172 + x^171 + x^170 + x^165 + x^163 + x^161 + x^159 + x^156 + x^154 + x^153 + x^144 + x^143 + x^137 + x^135 + x^134 + x^132 + x^130 + x^129 + x^126 + x^125 + x^122 + x^120 + x^118 + x^114 + x^113 + x^110 + x^109 + x^108 + x^107 + x^98 + x^97 + x^95 + x^92 + x^88 + x^87 + x^86 + x^85 + x^84 + x^83 + x^80 + x^76 + x^74 + x^71 + x^70 + x^69 + x^64 + x^63 + x^62 + x^61 + x^57 + x^53 + x^52 + x^50 + x^46 + x^40 + x^38 + x^37 + x^34 + x^33 + x^32 + x^31 + x^28 + x^25 + x^22 + x^18 + x^6 + 1 + +48-40-25 371 x^928 + x^898 + x^878 + x^870 + x^851 + x^844 + x^832 + x^828 + x^818 + x^809 + x^808 + x^802 + x^790 + x^782 + x^779 + x^772 + x^755 + x^751 + x^749 + x^744 + x^742 + x^740 + x^738 + x^732 + x^731 + x^728 + x^725 + x^719 + x^718 + x^714 + x^713 + x^712 + x^708 + x^705 + x^695 + x^691 + x^690 + x^688 + x^687 + x^686 + x^684 + x^683 + x^679 + x^676 + x^674 + x^671 + x^668 + x^665 + x^663 + x^662 + x^661 + x^656 + x^655 + x^652 + x^650 + x^649 + x^648 + x^647 + x^646 + x^641 + x^640 + x^638 + x^636 + x^635 + x^633 + x^630 + x^627 + x^626 + x^625 + x^624 + x^621 + x^620 + x^619 + x^618 + x^617 + x^614 + x^613 + x^608 + x^607 + x^606 + x^605 + x^604 + x^603 + x^602 + x^599 + x^597 + x^596 + x^591 + x^589 + x^586 + x^585 + x^584 + x^582 + x^578 + x^576 + x^572 + x^569 + x^566 + x^564 + x^558 + x^555 + x^553 + x^550 + x^549 + x^548 + x^545 + x^543 + x^541 + x^540 + x^538 + x^537 + x^533 + x^530 + x^527 + x^524 + x^523 + x^521 + x^520 + x^519 + x^517 + x^516 + x^515 + x^513 + x^510 + x^504 + x^503 + x^501 + x^500 + x^498 + x^497 + x^496 + x^494 + x^492 + x^491 + x^488 + x^485 + x^482 + x^478 + x^476 + x^474 + x^473 + x^471 + x^469 + x^468 + x^464 + x^461 + x^460 + x^459 + x^454 + x^450 + x^449 + x^448 + x^444 + x^439 + x^438 + x^437 + x^436 + x^435 + x^433 + x^431 + x^428 + x^426 + x^424 + x^420 + x^417 + x^416 + x^413 + x^411 + x^408 + x^407 + x^406 + x^403 + x^400 + x^399 + x^398 + x^397 + x^396 + x^392 + x^390 + x^389 + x^388 + x^387 + x^386 + x^385 + x^381 + x^378 + x^374 + x^370 + x^369 + x^368 + x^367 + x^365 + x^364 + x^359 + x^356 + x^354 + x^350 + x^348 + x^346 + x^344 + x^343 + x^342 + x^340 + x^338 + x^337 + x^334 + x^332 + x^330 + x^329 + x^327 + x^322 + x^317 + x^316 + x^314 + x^311 + x^308 + x^307 + x^301 + x^300 + x^299 + x^296 + x^295 + x^292 + x^291 + x^290 + x^286 + x^284 + x^282 + x^279 + x^278 + x^276 + x^273 + x^270 + x^269 + x^265 + x^263 + x^259 + x^258 + x^257 + x^256 + x^255 + x^253 + x^251 + x^249 + x^247 + x^245 + x^244 + x^243 + x^242 + x^241 + x^240 + x^238 + x^236 + x^231 + x^230 + x^229 + x^228 + x^226 + x^223 + x^221 + x^220 + x^219 + x^218 + x^217 + x^215 + x^214 + x^212 + x^211 + x^210 + x^208 + x^207 + x^206 + x^205 + x^202 + x^200 + x^198 + x^193 + x^192 + x^191 + x^187 + x^186 + x^184 + x^182 + x^179 + x^173 + x^171 + x^169 + x^168 + x^164 + x^163 + x^162 + x^161 + x^157 + x^156 + x^155 + x^154 + x^153 + x^151 + x^149 + x^147 + x^146 + x^145 + x^143 + x^140 + x^137 + x^136 + x^133 + x^127 + x^122 + x^121 + x^119 + x^115 + x^114 + x^112 + x^111 + x^110 + x^108 + x^104 + x^103 + x^101 + x^98 + x^94 + x^88 + x^87 + x^85 + x^83 + x^81 + x^80 + x^79 + x^78 + x^77 + x^75 + x^74 + x^73 + x^70 + x^69 + x^68 + x^67 + x^65 + x^63 + x^62 + x^60 + x^59 + x^58 + x^57 + x^56 + x^54 + x^53 + x^52 + x^50 + x^49 + x^47 + x^46 + x^45 + x^41 + x^39 + x^36 + x^34 + x^33 + x^32 + x^31 + x^30 + x^29 + x^28 + x^26 + x^25 + x^21 + x^16 + x^14 + x^4 + 1 + +19-22-10 373 x^928 + x^898 + x^870 + x^866 + x^842 + x^836 + x^834 + x^821 + x^808 + x^807 + x^806 + x^804 + x^802 + x^794 + x^792 + x^782 + x^778 + x^777 + x^776 + x^770 + x^766 + x^761 + x^757 + x^753 + x^748 + x^743 + x^739 + x^738 + x^736 + x^735 + x^734 + x^732 + x^728 + x^725 + x^722 + x^721 + x^718 + x^716 + x^713 + x^711 + x^710 + x^709 + x^708 + x^705 + x^703 + x^702 + x^693 + x^686 + x^681 + x^680 + x^677 + x^664 + x^662 + x^659 + x^652 + x^651 + x^649 + x^646 + x^644 + x^643 + x^641 + x^638 + x^636 + x^629 + x^627 + x^625 + x^623 + x^622 + x^619 + x^617 + x^616 + x^615 + x^614 + x^612 + x^611 + x^609 + x^606 + x^605 + x^600 + x^597 + x^596 + x^595 + x^592 + x^591 + x^590 + x^586 + x^582 + x^580 + x^577 + x^576 + x^574 + x^572 + x^570 + x^569 + x^567 + x^565 + x^563 + x^562 + x^560 + x^557 + x^556 + x^555 + x^554 + x^552 + x^549 + x^548 + x^546 + x^545 + x^544 + x^542 + x^540 + x^538 + x^536 + x^534 + x^533 + x^532 + x^530 + x^528 + x^525 + x^520 + x^519 + x^518 + x^516 + x^514 + x^513 + x^511 + x^509 + x^504 + x^501 + x^497 + x^496 + x^495 + x^488 + x^487 + x^485 + x^482 + x^481 + x^480 + x^479 + x^478 + x^476 + x^475 + x^474 + x^472 + x^470 + x^468 + x^466 + x^465 + x^463 + x^458 + x^453 + x^451 + x^449 + x^448 + x^447 + x^446 + x^445 + x^444 + x^443 + x^439 + x^438 + x^437 + x^436 + x^433 + x^428 + x^424 + x^418 + x^417 + x^414 + x^412 + x^411 + x^408 + x^407 + x^406 + x^404 + x^403 + x^401 + x^400 + x^399 + x^396 + x^395 + x^393 + x^390 + x^389 + x^388 + x^387 + x^386 + x^385 + x^384 + x^382 + x^380 + x^379 + x^375 + x^371 + x^370 + x^369 + x^365 + x^363 + x^362 + x^357 + x^355 + x^354 + x^351 + x^348 + x^346 + x^345 + x^344 + x^342 + x^340 + x^338 + x^336 + x^334 + x^331 + x^328 + x^327 + x^326 + x^324 + x^323 + x^322 + x^320 + x^315 + x^308 + x^306 + x^298 + x^295 + x^292 + x^287 + x^286 + x^285 + x^284 + x^282 + x^281 + x^280 + x^279 + x^278 + x^276 + x^274 + x^273 + x^272 + x^270 + x^269 + x^268 + x^267 + x^265 + x^262 + x^258 + x^256 + x^253 + x^252 + x^251 + x^250 + x^246 + x^245 + x^244 + x^243 + x^242 + x^240 + x^239 + x^238 + x^237 + x^235 + x^232 + x^228 + x^226 + x^224 + x^217 + x^215 + x^213 + x^211 + x^210 + x^208 + x^206 + x^201 + x^199 + x^196 + x^195 + x^192 + x^190 + x^187 + x^186 + x^185 + x^184 + x^182 + x^180 + x^178 + x^175 + x^174 + x^173 + x^172 + x^165 + x^160 + x^159 + x^158 + x^157 + x^155 + x^154 + x^153 + x^148 + x^146 + x^145 + x^143 + x^142 + x^137 + x^135 + x^133 + x^132 + x^131 + x^128 + x^127 + x^124 + x^123 + x^122 + x^121 + x^117 + x^115 + x^114 + x^113 + x^112 + x^108 + x^107 + x^106 + x^105 + x^103 + x^102 + x^101 + x^100 + x^94 + x^93 + x^92 + x^89 + x^87 + x^86 + x^85 + x^83 + x^80 + x^79 + x^78 + x^77 + x^75 + x^74 + x^72 + x^70 + x^67 + x^66 + x^62 + x^61 + x^59 + x^58 + x^57 + x^55 + x^51 + x^50 + x^49 + x^42 + x^41 + x^39 + x^36 + x^34 + x^32 + x^30 + x^26 + x^24 + x^23 + x^22 + x^21 + x^18 + x^16 + x^14 + x^10 + x^9 + x^8 + x^2 + 1 + +52-17-25 375 x^928 + x^898 + x^870 + x^861 + x^860 + x^859 + x^858 + x^849 + x^848 + x^847 + x^846 + x^842 + x^836 + x^835 + x^831 + x^830 + x^829 + x^828 + x^823 + x^822 + x^811 + x^810 + x^808 + x^807 + x^806 + x^804 + x^800 + x^799 + x^793 + x^792 + x^789 + x^788 + x^786 + x^780 + x^778 + x^777 + x^775 + x^774 + x^770 + x^769 + x^768 + x^764 + x^763 + x^762 + x^758 + x^757 + x^756 + x^755 + x^751 + x^750 + x^744 + x^740 + x^739 + x^738 + x^733 + x^731 + x^728 + x^721 + x^720 + x^716 + x^715 + x^714 + x^713 + x^710 + x^709 + x^702 + x^701 + x^697 + x^696 + x^695 + x^692 + x^688 + x^686 + x^684 + x^683 + x^680 + x^679 + x^678 + x^677 + x^674 + x^673 + x^672 + x^665 + x^662 + x^658 + x^654 + x^647 + x^643 + x^636 + x^635 + x^631 + x^628 + x^626 + x^625 + x^622 + x^621 + x^619 + x^618 + x^617 + x^614 + x^613 + x^612 + x^611 + x^609 + x^608 + x^598 + x^595 + x^591 + x^589 + x^588 + x^587 + x^586 + x^584 + x^582 + x^576 + x^574 + x^571 + x^569 + x^567 + x^564 + x^563 + x^560 + x^557 + x^550 + x^549 + x^548 + x^547 + x^546 + x^544 + x^539 + x^537 + x^536 + x^529 + x^527 + x^526 + x^523 + x^522 + x^521 + x^516 + x^513 + x^512 + x^509 + x^508 + x^505 + x^503 + x^502 + x^499 + x^493 + x^492 + x^491 + x^487 + x^486 + x^484 + x^482 + x^479 + x^478 + x^477 + x^475 + x^474 + x^471 + x^468 + x^467 + x^465 + x^463 + x^462 + x^461 + x^460 + x^455 + x^452 + x^445 + x^443 + x^442 + x^437 + x^433 + x^432 + x^429 + x^427 + x^426 + x^424 + x^423 + x^422 + x^421 + x^417 + x^411 + x^410 + x^408 + x^407 + x^403 + x^402 + x^400 + x^399 + x^393 + x^391 + x^390 + x^389 + x^386 + x^384 + x^381 + x^379 + x^375 + x^373 + x^372 + x^369 + x^367 + x^366 + x^364 + x^363 + x^362 + x^361 + x^360 + x^357 + x^356 + x^353 + x^352 + x^348 + x^345 + x^344 + x^342 + x^341 + x^340 + x^335 + x^332 + x^330 + x^329 + x^328 + x^327 + x^322 + x^321 + x^319 + x^318 + x^317 + x^315 + x^312 + x^311 + x^310 + x^308 + x^305 + x^302 + x^300 + x^299 + x^298 + x^297 + x^296 + x^292 + x^290 + x^289 + x^288 + x^286 + x^284 + x^283 + x^282 + x^281 + x^278 + x^277 + x^276 + x^274 + x^272 + x^269 + x^267 + x^263 + x^262 + x^260 + x^259 + x^256 + x^254 + x^253 + x^251 + x^250 + x^249 + x^248 + x^242 + x^241 + x^240 + x^235 + x^234 + x^231 + x^226 + x^225 + x^221 + x^219 + x^214 + x^212 + x^211 + x^209 + x^207 + x^206 + x^205 + x^202 + x^199 + x^198 + x^195 + x^190 + x^189 + x^187 + x^185 + x^182 + x^181 + x^180 + x^179 + x^177 + x^176 + x^175 + x^174 + x^171 + x^169 + x^165 + x^164 + x^163 + x^162 + x^161 + x^160 + x^155 + x^154 + x^153 + x^152 + x^146 + x^145 + x^142 + x^139 + x^137 + x^136 + x^128 + x^127 + x^124 + x^121 + x^120 + x^119 + x^117 + x^114 + x^113 + x^112 + x^110 + x^109 + x^108 + x^104 + x^102 + x^101 + x^99 + x^97 + x^87 + x^85 + x^84 + x^83 + x^82 + x^78 + x^76 + x^73 + x^72 + x^69 + x^64 + x^62 + x^59 + x^52 + x^51 + x^50 + x^49 + x^46 + x^45 + x^44 + x^42 + x^41 + x^40 + x^36 + x^35 + x^33 + x^28 + x^26 + x^25 + x^21 + x^19 + x^13 + x^12 + x^7 + x^6 + 1 + +35-28-44 377 x^928 + x^898 + x^894 + x^870 + x^859 + x^829 + x^828 + x^825 + x^808 + x^804 + x^800 + x^799 + x^795 + x^794 + x^789 + x^778 + x^774 + x^769 + x^768 + x^759 + x^758 + x^755 + x^754 + x^748 + x^744 + x^739 + x^735 + x^734 + x^727 + x^726 + x^722 + x^718 + x^714 + x^713 + x^709 + x^708 + x^705 + x^704 + x^700 + x^694 + x^691 + x^690 + x^688 + x^687 + x^685 + x^682 + x^681 + x^675 + x^668 + x^665 + x^659 + x^657 + x^656 + x^655 + x^653 + x^652 + x^651 + x^649 + x^647 + x^644 + x^640 + x^638 + x^635 + x^634 + x^631 + x^630 + x^629 + x^628 + x^625 + x^624 + x^619 + x^617 + x^614 + x^613 + x^612 + x^611 + x^610 + x^608 + x^602 + x^599 + x^598 + x^593 + x^592 + x^591 + x^590 + x^586 + x^585 + x^583 + x^582 + x^581 + x^580 + x^579 + x^578 + x^576 + x^575 + x^573 + x^569 + x^566 + x^560 + x^559 + x^558 + x^557 + x^553 + x^552 + x^551 + x^549 + x^542 + x^536 + x^535 + x^533 + x^524 + x^522 + x^521 + x^520 + x^518 + x^517 + x^516 + x^515 + x^512 + x^509 + x^507 + x^506 + x^504 + x^502 + x^500 + x^499 + x^498 + x^497 + x^496 + x^491 + x^489 + x^485 + x^484 + x^483 + x^482 + x^481 + x^480 + x^477 + x^476 + x^474 + x^467 + x^466 + x^465 + x^462 + x^460 + x^459 + x^454 + x^450 + x^447 + x^446 + x^445 + x^443 + x^439 + x^434 + x^433 + x^432 + x^431 + x^430 + x^429 + x^428 + x^427 + x^426 + x^425 + x^424 + x^423 + x^421 + x^419 + x^417 + x^413 + x^412 + x^411 + x^409 + x^408 + x^407 + x^406 + x^405 + x^404 + x^403 + x^401 + x^400 + x^394 + x^389 + x^386 + x^383 + x^381 + x^380 + x^379 + x^378 + x^376 + x^375 + x^373 + x^372 + x^371 + x^369 + x^368 + x^367 + x^366 + x^365 + x^364 + x^362 + x^359 + x^356 + x^349 + x^347 + x^346 + x^345 + x^343 + x^342 + x^341 + x^340 + x^339 + x^338 + x^337 + x^336 + x^335 + x^332 + x^331 + x^330 + x^329 + x^325 + x^324 + x^320 + x^319 + x^318 + x^317 + x^316 + x^315 + x^311 + x^309 + x^307 + x^304 + x^303 + x^301 + x^300 + x^298 + x^297 + x^296 + x^295 + x^294 + x^292 + x^291 + x^289 + x^288 + x^285 + x^283 + x^282 + x^281 + x^274 + x^272 + x^271 + x^269 + x^262 + x^261 + x^260 + x^258 + x^257 + x^252 + x^251 + x^250 + x^249 + x^247 + x^246 + x^244 + x^241 + x^236 + x^234 + x^231 + x^230 + x^227 + x^226 + x^219 + x^218 + x^216 + x^215 + x^213 + x^206 + x^205 + x^204 + x^203 + x^202 + x^201 + x^200 + x^198 + x^196 + x^195 + x^194 + x^193 + x^192 + x^190 + x^186 + x^185 + x^184 + x^182 + x^176 + x^174 + x^167 + x^166 + x^164 + x^163 + x^162 + x^161 + x^159 + x^155 + x^154 + x^153 + x^151 + x^150 + x^146 + x^145 + x^143 + x^140 + x^138 + x^135 + x^131 + x^130 + x^129 + x^125 + x^124 + x^123 + x^118 + x^116 + x^113 + x^110 + x^107 + x^101 + x^100 + x^99 + x^95 + x^94 + x^93 + x^92 + x^91 + x^88 + x^86 + x^82 + x^81 + x^80 + x^79 + x^78 + x^76 + x^75 + x^73 + x^72 + x^71 + x^70 + x^69 + x^68 + x^67 + x^65 + x^63 + x^62 + x^60 + x^58 + x^57 + x^56 + x^55 + x^52 + x^50 + x^48 + x^46 + x^44 + x^42 + x^40 + x^39 + x^38 + x^33 + x^31 + x^30 + x^29 + x^28 + x^27 + x^24 + x^21 + x^16 + x^13 + x^8 + 1 + +45-7-6 377 x^928 + x^898 + x^897 + x^870 + x^868 + x^854 + x^853 + x^850 + x^836 + x^835 + x^832 + x^821 + x^810 + x^809 + x^808 + x^807 + x^805 + x^803 + x^802 + x^801 + x^798 + x^794 + x^792 + x^791 + x^790 + x^780 + x^778 + x^775 + x^770 + x^767 + x^766 + x^765 + x^764 + x^760 + x^754 + x^750 + x^743 + x^740 + x^737 + x^735 + x^732 + x^726 + x^723 + x^722 + x^721 + x^720 + x^716 + x^714 + x^710 + x^707 + x^706 + x^703 + x^701 + x^700 + x^699 + x^695 + x^691 + x^689 + x^687 + x^686 + x^685 + x^683 + x^680 + x^676 + x^675 + x^672 + x^671 + x^670 + x^668 + x^667 + x^666 + x^665 + x^662 + x^658 + x^655 + x^653 + x^652 + x^648 + x^644 + x^643 + x^642 + x^637 + x^629 + x^628 + x^627 + x^626 + x^624 + x^621 + x^617 + x^616 + x^612 + x^610 + x^609 + x^608 + x^607 + x^603 + x^602 + x^600 + x^595 + x^594 + x^592 + x^591 + x^589 + x^586 + x^585 + x^582 + x^578 + x^576 + x^574 + x^573 + x^571 + x^570 + x^566 + x^563 + x^561 + x^560 + x^558 + x^557 + x^556 + x^555 + x^554 + x^553 + x^552 + x^551 + x^549 + x^548 + x^545 + x^538 + x^533 + x^530 + x^524 + x^520 + x^519 + x^512 + x^509 + x^508 + x^505 + x^503 + x^502 + x^497 + x^495 + x^493 + x^492 + x^486 + x^485 + x^484 + x^482 + x^480 + x^479 + x^477 + x^472 + x^470 + x^469 + x^466 + x^464 + x^462 + x^460 + x^459 + x^458 + x^456 + x^455 + x^454 + x^452 + x^451 + x^448 + x^446 + x^442 + x^441 + x^440 + x^437 + x^431 + x^430 + x^427 + x^426 + x^425 + x^424 + x^423 + x^420 + x^417 + x^415 + x^414 + x^413 + x^412 + x^410 + x^408 + x^406 + x^405 + x^404 + x^403 + x^401 + x^399 + x^395 + x^393 + x^392 + x^384 + x^381 + x^375 + x^370 + x^368 + x^367 + x^364 + x^363 + x^361 + x^360 + x^356 + x^355 + x^354 + x^353 + x^351 + x^349 + x^346 + x^343 + x^341 + x^340 + x^339 + x^335 + x^334 + x^333 + x^330 + x^328 + x^326 + x^325 + x^324 + x^323 + x^319 + x^318 + x^316 + x^315 + x^312 + x^306 + x^303 + x^302 + x^301 + x^299 + x^294 + x^292 + x^291 + x^290 + x^289 + x^283 + x^281 + x^276 + x^275 + x^274 + x^273 + x^271 + x^270 + x^269 + x^268 + x^267 + x^265 + x^264 + x^263 + x^262 + x^261 + x^260 + x^259 + x^258 + x^257 + x^255 + x^254 + x^253 + x^252 + x^250 + x^247 + x^246 + x^245 + x^243 + x^241 + x^240 + x^239 + x^235 + x^228 + x^224 + x^223 + x^220 + x^217 + x^216 + x^215 + x^209 + x^205 + x^204 + x^203 + x^202 + x^197 + x^196 + x^195 + x^192 + x^191 + x^189 + x^187 + x^183 + x^181 + x^180 + x^179 + x^178 + x^173 + x^172 + x^170 + x^165 + x^163 + x^161 + x^159 + x^156 + x^155 + x^153 + x^148 + x^146 + x^141 + x^138 + x^134 + x^133 + x^131 + x^130 + x^127 + x^126 + x^125 + x^123 + x^122 + x^119 + x^117 + x^116 + x^110 + x^109 + x^108 + x^107 + x^106 + x^95 + x^93 + x^92 + x^91 + x^88 + x^87 + x^86 + x^84 + x^82 + x^80 + x^79 + x^76 + x^75 + x^73 + x^72 + x^69 + x^68 + x^64 + x^63 + x^61 + x^59 + x^57 + x^55 + x^54 + x^53 + x^51 + x^49 + x^46 + x^43 + x^41 + x^40 + x^36 + x^35 + x^34 + x^33 + x^31 + x^27 + x^25 + x^23 + x^22 + x^21 + x^19 + x^15 + x^13 + x^12 + x^11 + x^10 + x^9 + x^8 + 1 + +23-49-24 379 x^928 + x^898 + x^872 + x^870 + x^856 + x^843 + x^842 + x^827 + x^826 + x^824 + x^818 + x^813 + x^812 + x^811 + x^797 + x^796 + x^795 + x^792 + x^789 + x^788 + x^781 + x^780 + x^772 + x^767 + x^765 + x^759 + x^757 + x^752 + x^743 + x^742 + x^741 + x^740 + x^737 + x^735 + x^730 + x^729 + x^728 + x^727 + x^724 + x^722 + x^721 + x^720 + x^718 + x^714 + x^712 + x^710 + x^707 + x^706 + x^704 + x^703 + x^702 + x^700 + x^699 + x^698 + x^696 + x^694 + x^692 + x^691 + x^690 + x^689 + x^688 + x^686 + x^682 + x^681 + x^679 + x^677 + x^675 + x^674 + x^665 + x^663 + x^661 + x^660 + x^657 + x^656 + x^655 + x^651 + x^649 + x^647 + x^646 + x^644 + x^642 + x^640 + x^637 + x^635 + x^633 + x^625 + x^622 + x^620 + x^619 + x^617 + x^616 + x^615 + x^613 + x^612 + x^610 + x^609 + x^607 + x^606 + x^605 + x^601 + x^600 + x^597 + x^596 + x^595 + x^593 + x^592 + x^590 + x^588 + x^584 + x^583 + x^582 + x^579 + x^578 + x^574 + x^571 + x^570 + x^567 + x^566 + x^564 + x^561 + x^560 + x^559 + x^558 + x^554 + x^553 + x^547 + x^543 + x^539 + x^536 + x^535 + x^534 + x^533 + x^529 + x^527 + x^526 + x^525 + x^518 + x^516 + x^515 + x^510 + x^509 + x^508 + x^506 + x^505 + x^504 + x^503 + x^499 + x^498 + x^497 + x^495 + x^493 + x^491 + x^489 + x^488 + x^487 + x^486 + x^485 + x^484 + x^482 + x^479 + x^475 + x^473 + x^472 + x^469 + x^468 + x^464 + x^463 + x^462 + x^459 + x^457 + x^455 + x^453 + x^448 + x^446 + x^444 + x^443 + x^442 + x^438 + x^435 + x^431 + x^429 + x^427 + x^424 + x^423 + x^418 + x^416 + x^414 + x^406 + x^404 + x^403 + x^401 + x^397 + x^396 + x^390 + x^387 + x^384 + x^380 + x^377 + x^375 + x^373 + x^369 + x^366 + x^365 + x^364 + x^358 + x^356 + x^355 + x^354 + x^353 + x^352 + x^348 + x^346 + x^340 + x^338 + x^336 + x^334 + x^329 + x^326 + x^325 + x^322 + x^320 + x^318 + x^312 + x^307 + x^305 + x^302 + x^301 + x^299 + x^297 + x^296 + x^294 + x^293 + x^291 + x^288 + x^287 + x^286 + x^281 + x^278 + x^277 + x^276 + x^274 + x^273 + x^272 + x^271 + x^269 + x^268 + x^267 + x^266 + x^262 + x^261 + x^258 + x^256 + x^255 + x^253 + x^251 + x^250 + x^249 + x^248 + x^247 + x^243 + x^240 + x^237 + x^236 + x^235 + x^233 + x^231 + x^230 + x^223 + x^222 + x^220 + x^218 + x^216 + x^212 + x^211 + x^208 + x^207 + x^205 + x^203 + x^202 + x^201 + x^200 + x^199 + x^198 + x^197 + x^195 + x^193 + x^192 + x^191 + x^187 + x^186 + x^183 + x^181 + x^180 + x^179 + x^176 + x^174 + x^173 + x^169 + x^166 + x^161 + x^159 + x^158 + x^157 + x^156 + x^154 + x^152 + x^151 + x^149 + x^145 + x^144 + x^143 + x^142 + x^141 + x^139 + x^136 + x^135 + x^131 + x^128 + x^127 + x^126 + x^125 + x^123 + x^122 + x^120 + x^118 + x^117 + x^116 + x^114 + x^111 + x^110 + x^108 + x^107 + x^106 + x^103 + x^100 + x^99 + x^97 + x^96 + x^95 + x^93 + x^92 + x^91 + x^90 + x^87 + x^84 + x^81 + x^80 + x^79 + x^78 + x^77 + x^74 + x^73 + x^71 + x^67 + x^64 + x^61 + x^59 + x^58 + x^54 + x^49 + x^47 + x^44 + x^43 + x^41 + x^38 + x^37 + x^33 + x^30 + x^25 + x^17 + x^16 + x^15 + x^14 + x^12 + x^11 + x^8 + x^7 + x^4 + 1 + +19-18-26 381 x^928 + x^898 + x^870 + x^844 + x^836 + x^832 + x^820 + x^814 + x^808 + x^802 + x^798 + x^794 + x^791 + x^790 + x^784 + x^778 + x^774 + x^770 + x^769 + x^766 + x^764 + x^761 + x^754 + x^748 + x^743 + x^734 + x^732 + x^731 + x^727 + x^726 + x^724 + x^723 + x^721 + x^718 + x^716 + x^715 + x^713 + x^711 + x^709 + x^706 + x^704 + x^703 + x^702 + x^701 + x^700 + x^699 + x^697 + x^696 + x^695 + x^693 + x^692 + x^691 + x^690 + x^685 + x^684 + x^676 + x^674 + x^667 + x^664 + x^663 + x^658 + x^657 + x^656 + x^654 + x^653 + x^651 + x^650 + x^646 + x^645 + x^644 + x^643 + x^639 + x^638 + x^637 + x^633 + x^631 + x^630 + x^627 + x^626 + x^625 + x^624 + x^620 + x^611 + x^610 + x^607 + x^606 + x^604 + x^603 + x^598 + x^597 + x^595 + x^593 + x^590 + x^589 + x^588 + x^586 + x^585 + x^584 + x^582 + x^580 + x^577 + x^575 + x^570 + x^569 + x^568 + x^563 + x^562 + x^561 + x^558 + x^557 + x^556 + x^555 + x^552 + x^551 + x^550 + x^548 + x^547 + x^546 + x^545 + x^542 + x^541 + x^540 + x^539 + x^538 + x^537 + x^535 + x^534 + x^533 + x^531 + x^530 + x^528 + x^523 + x^522 + x^521 + x^517 + x^515 + x^514 + x^512 + x^510 + x^509 + x^508 + x^506 + x^497 + x^494 + x^493 + x^490 + x^487 + x^486 + x^483 + x^481 + x^479 + x^476 + x^474 + x^472 + x^471 + x^467 + x^465 + x^464 + x^463 + x^462 + x^460 + x^456 + x^454 + x^452 + x^450 + x^449 + x^448 + x^447 + x^445 + x^444 + x^440 + x^439 + x^438 + x^437 + x^436 + x^432 + x^431 + x^430 + x^429 + x^428 + x^427 + x^425 + x^424 + x^422 + x^420 + x^419 + x^418 + x^416 + x^414 + x^413 + x^409 + x^407 + x^405 + x^404 + x^402 + x^397 + x^396 + x^392 + x^391 + x^389 + x^386 + x^385 + x^384 + x^381 + x^377 + x^376 + x^373 + x^371 + x^368 + x^366 + x^364 + x^362 + x^360 + x^359 + x^357 + x^355 + x^354 + x^353 + x^352 + x^351 + x^350 + x^349 + x^348 + x^346 + x^343 + x^341 + x^340 + x^338 + x^337 + x^336 + x^334 + x^331 + x^329 + x^328 + x^327 + x^325 + x^324 + x^323 + x^320 + x^316 + x^314 + x^313 + x^311 + x^309 + x^307 + x^306 + x^305 + x^304 + x^302 + x^301 + x^299 + x^297 + x^291 + x^289 + x^288 + x^285 + x^283 + x^282 + x^280 + x^279 + x^272 + x^271 + x^270 + x^269 + x^268 + x^264 + x^263 + x^260 + x^259 + x^257 + x^249 + x^245 + x^242 + x^241 + x^238 + x^234 + x^233 + x^232 + x^230 + x^229 + x^227 + x^226 + x^225 + x^223 + x^221 + x^214 + x^211 + x^210 + x^209 + x^206 + x^205 + x^204 + x^203 + x^201 + x^200 + x^199 + x^197 + x^195 + x^188 + x^186 + x^181 + x^180 + x^177 + x^174 + x^173 + x^171 + x^160 + x^159 + x^158 + x^157 + x^154 + x^148 + x^147 + x^144 + x^142 + x^141 + x^140 + x^139 + x^135 + x^134 + x^129 + x^124 + x^123 + x^120 + x^118 + x^117 + x^116 + x^113 + x^112 + x^110 + x^109 + x^108 + x^107 + x^106 + x^104 + x^103 + x^100 + x^97 + x^95 + x^94 + x^89 + x^86 + x^84 + x^83 + x^79 + x^76 + x^75 + x^73 + x^72 + x^70 + x^67 + x^66 + x^65 + x^63 + x^61 + x^60 + x^59 + x^58 + x^56 + x^53 + x^52 + x^51 + x^50 + x^45 + x^41 + x^40 + x^39 + x^37 + x^36 + x^35 + x^29 + x^26 + x^22 + x^19 + x^18 + x^17 + x^16 + x^13 + x^11 + x^6 + 1 + +26-3-47 381 x^928 + x^898 + x^878 + x^870 + x^869 + x^860 + x^859 + x^858 + x^850 + x^849 + x^848 + x^840 + x^839 + x^838 + x^830 + x^820 + x^818 + x^810 + x^808 + x^807 + x^798 + x^788 + x^780 + x^778 + x^777 + x^769 + x^767 + x^758 + x^750 + x^740 + x^730 + x^729 + x^728 + x^727 + x^721 + x^720 + x^719 + x^717 + x^716 + x^711 + x^701 + x^700 + x^696 + x^690 + x^689 + x^687 + x^686 + x^681 + x^679 + x^678 + x^676 + x^670 + x^668 + x^667 + x^661 + x^660 + x^657 + x^656 + x^652 + x^650 + x^649 + x^648 + x^647 + x^646 + x^641 + x^638 + x^637 + x^636 + x^632 + x^629 + x^627 + x^625 + x^622 + x^620 + x^619 + x^617 + x^616 + x^610 + x^607 + x^606 + x^602 + x^601 + x^600 + x^599 + x^597 + x^596 + x^592 + x^591 + x^590 + x^588 + x^587 + x^586 + x^585 + x^581 + x^578 + x^576 + x^572 + x^571 + x^570 + x^567 + x^562 + x^560 + x^559 + x^558 + x^557 + x^555 + x^551 + x^550 + x^549 + x^548 + x^546 + x^542 + x^541 + x^536 + x^534 + x^531 + x^530 + x^528 + x^525 + x^523 + x^522 + x^520 + x^519 + x^518 + x^515 + x^514 + x^511 + x^510 + x^509 + x^508 + x^505 + x^504 + x^502 + x^501 + x^500 + x^499 + x^494 + x^493 + x^485 + x^484 + x^481 + x^480 + x^479 + x^477 + x^476 + x^475 + x^473 + x^466 + x^464 + x^462 + x^461 + x^460 + x^458 + x^457 + x^453 + x^447 + x^444 + x^442 + x^441 + x^440 + x^439 + x^438 + x^433 + x^432 + x^429 + x^428 + x^426 + x^425 + x^424 + x^422 + x^421 + x^420 + x^419 + x^418 + x^414 + x^412 + x^411 + x^407 + x^404 + x^401 + x^398 + x^396 + x^393 + x^392 + x^391 + x^389 + x^387 + x^385 + x^383 + x^382 + x^381 + x^380 + x^378 + x^377 + x^374 + x^372 + x^371 + x^368 + x^365 + x^363 + x^362 + x^359 + x^357 + x^356 + x^355 + x^354 + x^350 + x^347 + x^344 + x^343 + x^341 + x^338 + x^337 + x^335 + x^334 + x^333 + x^331 + x^330 + x^329 + x^328 + x^324 + x^322 + x^319 + x^317 + x^314 + x^310 + x^306 + x^300 + x^299 + x^295 + x^294 + x^291 + x^290 + x^289 + x^288 + x^287 + x^286 + x^285 + x^284 + x^282 + x^281 + x^279 + x^278 + x^275 + x^272 + x^271 + x^270 + x^269 + x^266 + x^264 + x^262 + x^260 + x^259 + x^258 + x^254 + x^250 + x^249 + x^247 + x^246 + x^244 + x^243 + x^241 + x^239 + x^237 + x^236 + x^233 + x^232 + x^231 + x^229 + x^227 + x^226 + x^221 + x^220 + x^218 + x^217 + x^216 + x^215 + x^214 + x^213 + x^212 + x^210 + x^209 + x^207 + x^206 + x^202 + x^201 + x^200 + x^199 + x^198 + x^197 + x^195 + x^193 + x^192 + x^189 + x^188 + x^180 + x^176 + x^174 + x^173 + x^172 + x^170 + x^167 + x^165 + x^164 + x^162 + x^160 + x^157 + x^155 + x^153 + x^152 + x^151 + x^149 + x^148 + x^147 + x^144 + x^142 + x^140 + x^139 + x^136 + x^134 + x^133 + x^128 + x^127 + x^124 + x^123 + x^122 + x^121 + x^119 + x^117 + x^115 + x^113 + x^112 + x^110 + x^107 + x^105 + x^101 + x^100 + x^99 + x^98 + x^94 + x^92 + x^91 + x^90 + x^89 + x^87 + x^85 + x^83 + x^81 + x^80 + x^78 + x^77 + x^74 + x^72 + x^71 + x^70 + x^69 + x^65 + x^64 + x^63 + x^62 + x^61 + x^57 + x^56 + x^55 + x^54 + x^53 + x^49 + x^48 + x^42 + x^41 + x^39 + x^36 + x^34 + x^32 + x^31 + x^24 + x^23 + x^21 + x^14 + x^10 + 1 + +37-33-14 381 x^928 + x^898 + x^874 + x^870 + x^849 + x^845 + x^841 + x^837 + x^836 + x^832 + x^824 + x^820 + x^812 + x^811 + x^804 + x^802 + x^794 + x^789 + x^785 + x^783 + x^781 + x^777 + x^775 + x^771 + x^770 + x^764 + x^761 + x^757 + x^754 + x^751 + x^746 + x^742 + x^741 + x^740 + x^738 + x^736 + x^734 + x^733 + x^731 + x^727 + x^726 + x^725 + x^724 + x^723 + x^721 + x^720 + x^718 + x^717 + x^716 + x^715 + x^714 + x^713 + x^712 + x^711 + x^709 + x^705 + x^704 + x^703 + x^701 + x^695 + x^692 + x^691 + x^690 + x^688 + x^687 + x^686 + x^685 + x^684 + x^682 + x^680 + x^679 + x^676 + x^671 + x^670 + x^665 + x^663 + x^662 + x^661 + x^660 + x^659 + x^653 + x^652 + x^651 + x^650 + x^648 + x^644 + x^643 + x^642 + x^641 + x^637 + x^635 + x^634 + x^631 + x^630 + x^629 + x^627 + x^626 + x^623 + x^622 + x^621 + x^619 + x^617 + x^616 + x^614 + x^613 + x^611 + x^610 + x^603 + x^601 + x^600 + x^599 + x^597 + x^596 + x^594 + x^593 + x^592 + x^591 + x^589 + x^587 + x^585 + x^583 + x^582 + x^579 + x^575 + x^573 + x^571 + x^570 + x^567 + x^566 + x^560 + x^557 + x^556 + x^554 + x^553 + x^552 + x^551 + x^548 + x^546 + x^541 + x^540 + x^538 + x^535 + x^534 + x^531 + x^527 + x^526 + x^525 + x^524 + x^523 + x^517 + x^516 + x^511 + x^509 + x^507 + x^499 + x^496 + x^493 + x^491 + x^490 + x^485 + x^482 + x^481 + x^479 + x^477 + x^475 + x^474 + x^470 + x^469 + x^468 + x^465 + x^462 + x^461 + x^459 + x^458 + x^453 + x^451 + x^446 + x^444 + x^441 + x^436 + x^435 + x^434 + x^432 + x^429 + x^420 + x^419 + x^416 + x^414 + x^411 + x^410 + x^409 + x^407 + x^405 + x^404 + x^403 + x^402 + x^401 + x^397 + x^395 + x^393 + x^392 + x^390 + x^388 + x^384 + x^381 + x^379 + x^376 + x^374 + x^368 + x^367 + x^365 + x^362 + x^361 + x^360 + x^359 + x^357 + x^355 + x^351 + x^349 + x^346 + x^345 + x^344 + x^342 + x^341 + x^339 + x^337 + x^336 + x^335 + x^333 + x^331 + x^326 + x^320 + x^318 + x^316 + x^315 + x^314 + x^313 + x^312 + x^311 + x^310 + x^309 + x^306 + x^304 + x^303 + x^302 + x^301 + x^300 + x^298 + x^297 + x^296 + x^295 + x^294 + x^293 + x^292 + x^290 + x^289 + x^288 + x^284 + x^283 + x^282 + x^278 + x^277 + x^274 + x^273 + x^272 + x^270 + x^266 + x^265 + x^261 + x^258 + x^256 + x^255 + x^254 + x^248 + x^246 + x^245 + x^239 + x^237 + x^234 + x^230 + x^229 + x^225 + x^224 + x^222 + x^221 + x^217 + x^215 + x^214 + x^213 + x^212 + x^208 + x^206 + x^204 + x^203 + x^202 + x^201 + x^199 + x^198 + x^195 + x^194 + x^191 + x^189 + x^188 + x^186 + x^184 + x^182 + x^181 + x^179 + x^177 + x^176 + x^175 + x^171 + x^169 + x^168 + x^162 + x^161 + x^159 + x^157 + x^156 + x^151 + x^150 + x^148 + x^146 + x^145 + x^142 + x^140 + x^139 + x^138 + x^135 + x^134 + x^131 + x^129 + x^126 + x^124 + x^119 + x^118 + x^117 + x^116 + x^108 + x^106 + x^105 + x^103 + x^99 + x^97 + x^93 + x^92 + x^91 + x^90 + x^87 + x^86 + x^84 + x^83 + x^78 + x^76 + x^73 + x^66 + x^65 + x^64 + x^62 + x^61 + x^58 + x^53 + x^52 + x^51 + x^48 + x^47 + x^46 + x^45 + x^44 + x^42 + x^41 + x^38 + x^34 + x^29 + x^28 + x^26 + x^23 + x^16 + x^8 + x^6 + 1 + +36-46-33 383 x^928 + x^898 + x^870 + x^857 + x^828 + x^827 + x^816 + x^815 + x^809 + x^808 + x^804 + x^797 + x^792 + x^786 + x^785 + x^780 + x^778 + x^775 + x^773 + x^768 + x^767 + x^762 + x^755 + x^751 + x^750 + x^737 + x^734 + x^732 + x^728 + x^727 + x^725 + x^722 + x^716 + x^715 + x^714 + x^710 + x^708 + x^707 + x^704 + x^703 + x^701 + x^698 + x^697 + x^695 + x^689 + x^688 + x^685 + x^681 + x^680 + x^679 + x^678 + x^677 + x^674 + x^665 + x^658 + x^656 + x^651 + x^650 + x^649 + x^648 + x^647 + x^640 + x^637 + x^635 + x^634 + x^633 + x^631 + x^628 + x^626 + x^624 + x^620 + x^619 + x^617 + x^616 + x^610 + x^607 + x^605 + x^603 + x^602 + x^601 + x^598 + x^591 + x^590 + x^587 + x^584 + x^582 + x^580 + x^577 + x^574 + x^573 + x^571 + x^567 + x^563 + x^562 + x^560 + x^559 + x^558 + x^557 + x^555 + x^553 + x^551 + x^550 + x^549 + x^548 + x^547 + x^544 + x^542 + x^540 + x^539 + x^538 + x^537 + x^535 + x^534 + x^533 + x^531 + x^528 + x^527 + x^525 + x^519 + x^517 + x^516 + x^515 + x^510 + x^509 + x^508 + x^507 + x^505 + x^500 + x^499 + x^496 + x^495 + x^494 + x^493 + x^489 + x^488 + x^487 + x^486 + x^485 + x^484 + x^482 + x^480 + x^479 + x^477 + x^475 + x^473 + x^470 + x^468 + x^467 + x^466 + x^465 + x^463 + x^461 + x^460 + x^455 + x^451 + x^450 + x^448 + x^444 + x^443 + x^442 + x^439 + x^438 + x^437 + x^435 + x^434 + x^433 + x^427 + x^426 + x^425 + x^423 + x^421 + x^420 + x^419 + x^418 + x^416 + x^413 + x^411 + x^410 + x^408 + x^407 + x^404 + x^402 + x^400 + x^399 + x^398 + x^397 + x^396 + x^395 + x^394 + x^393 + x^387 + x^386 + x^382 + x^379 + x^378 + x^377 + x^376 + x^375 + x^374 + x^372 + x^371 + x^370 + x^369 + x^366 + x^364 + x^363 + x^362 + x^361 + x^359 + x^358 + x^357 + x^356 + x^355 + x^354 + x^352 + x^347 + x^346 + x^345 + x^344 + x^343 + x^341 + x^340 + x^336 + x^335 + x^332 + x^330 + x^329 + x^328 + x^325 + x^324 + x^323 + x^322 + x^321 + x^320 + x^318 + x^316 + x^315 + x^312 + x^302 + x^301 + x^298 + x^295 + x^288 + x^287 + x^286 + x^285 + x^280 + x^279 + x^278 + x^275 + x^273 + x^272 + x^270 + x^267 + x^266 + x^264 + x^261 + x^259 + x^258 + x^255 + x^253 + x^252 + x^250 + x^249 + x^248 + x^247 + x^244 + x^243 + x^242 + x^241 + x^240 + x^237 + x^236 + x^235 + x^231 + x^230 + x^228 + x^225 + x^224 + x^222 + x^220 + x^218 + x^217 + x^216 + x^215 + x^212 + x^211 + x^205 + x^204 + x^203 + x^201 + x^196 + x^194 + x^193 + x^192 + x^191 + x^190 + x^187 + x^186 + x^184 + x^183 + x^182 + x^180 + x^179 + x^178 + x^174 + x^173 + x^172 + x^164 + x^163 + x^162 + x^161 + x^160 + x^159 + x^158 + x^156 + x^154 + x^153 + x^150 + x^149 + x^148 + x^146 + x^144 + x^143 + x^140 + x^134 + x^131 + x^127 + x^125 + x^124 + x^120 + x^119 + x^118 + x^116 + x^115 + x^113 + x^112 + x^109 + x^108 + x^107 + x^106 + x^103 + x^100 + x^99 + x^97 + x^96 + x^95 + x^94 + x^91 + x^87 + x^86 + x^83 + x^81 + x^80 + x^78 + x^77 + x^75 + x^74 + x^72 + x^71 + x^69 + x^68 + x^66 + x^64 + x^63 + x^59 + x^57 + x^55 + x^53 + x^48 + x^47 + x^44 + x^42 + x^41 + x^40 + x^37 + x^28 + x^27 + x^26 + x^20 + x^18 + x^16 + 1 + +40-1-13 383 x^928 + x^898 + x^870 + x^864 + x^861 + x^858 + x^855 + x^852 + x^837 + x^834 + x^825 + x^808 + x^804 + x^801 + x^798 + x^792 + x^791 + x^788 + x^785 + x^778 + x^777 + x^774 + x^773 + x^758 + x^755 + x^752 + x^743 + x^740 + x^738 + x^737 + x^735 + x^734 + x^725 + x^721 + x^719 + x^718 + x^716 + x^715 + x^713 + x^710 + x^707 + x^705 + x^704 + x^703 + x^698 + x^695 + x^691 + x^689 + x^686 + x^682 + x^680 + x^678 + x^677 + x^676 + x^674 + x^673 + x^671 + x^668 + x^667 + x^664 + x^662 + x^661 + x^659 + x^658 + x^656 + x^652 + x^650 + x^648 + x^647 + x^645 + x^644 + x^643 + x^642 + x^641 + x^640 + x^636 + x^633 + x^630 + x^629 + x^628 + x^627 + x^626 + x^623 + x^621 + x^618 + x^615 + x^612 + x^608 + x^607 + x^605 + x^603 + x^602 + x^600 + x^598 + x^596 + x^594 + x^593 + x^588 + x^583 + x^582 + x^579 + x^574 + x^573 + x^569 + x^568 + x^567 + x^565 + x^564 + x^562 + x^558 + x^557 + x^556 + x^555 + x^552 + x^551 + x^548 + x^545 + x^544 + x^542 + x^540 + x^537 + x^535 + x^534 + x^533 + x^532 + x^531 + x^530 + x^526 + x^525 + x^524 + x^523 + x^521 + x^520 + x^519 + x^518 + x^517 + x^515 + x^513 + x^509 + x^508 + x^507 + x^502 + x^500 + x^499 + x^496 + x^495 + x^491 + x^489 + x^488 + x^485 + x^482 + x^480 + x^478 + x^476 + x^474 + x^471 + x^468 + x^466 + x^463 + x^462 + x^461 + x^460 + x^459 + x^457 + x^456 + x^453 + x^451 + x^450 + x^447 + x^443 + x^440 + x^439 + x^436 + x^435 + x^434 + x^433 + x^432 + x^426 + x^425 + x^424 + x^422 + x^421 + x^419 + x^417 + x^416 + x^410 + x^409 + x^408 + x^406 + x^405 + x^402 + x^401 + x^399 + x^398 + x^396 + x^389 + x^388 + x^387 + x^385 + x^383 + x^382 + x^381 + x^379 + x^378 + x^377 + x^376 + x^375 + x^373 + x^371 + x^368 + x^366 + x^364 + x^363 + x^360 + x^358 + x^356 + x^355 + x^353 + x^350 + x^349 + x^346 + x^345 + x^344 + x^340 + x^339 + x^338 + x^332 + x^331 + x^329 + x^326 + x^324 + x^323 + x^321 + x^316 + x^315 + x^313 + x^312 + x^310 + x^309 + x^307 + x^306 + x^305 + x^303 + x^302 + x^300 + x^299 + x^296 + x^294 + x^293 + x^292 + x^289 + x^288 + x^286 + x^285 + x^283 + x^282 + x^279 + x^277 + x^274 + x^273 + x^272 + x^270 + x^269 + x^268 + x^267 + x^265 + x^264 + x^261 + x^259 + x^256 + x^251 + x^249 + x^245 + x^241 + x^240 + x^239 + x^238 + x^236 + x^235 + x^234 + x^232 + x^230 + x^226 + x^222 + x^220 + x^217 + x^215 + x^211 + x^210 + x^208 + x^203 + x^200 + x^199 + x^197 + x^195 + x^194 + x^193 + x^190 + x^187 + x^186 + x^184 + x^183 + x^182 + x^178 + x^176 + x^174 + x^169 + x^168 + x^167 + x^164 + x^162 + x^161 + x^160 + x^159 + x^158 + x^154 + x^153 + x^147 + x^142 + x^138 + x^137 + x^129 + x^128 + x^127 + x^126 + x^124 + x^123 + x^121 + x^119 + x^116 + x^115 + x^112 + x^110 + x^109 + x^107 + x^106 + x^103 + x^101 + x^100 + x^99 + x^98 + x^95 + x^94 + x^93 + x^92 + x^91 + x^89 + x^85 + x^83 + x^80 + x^79 + x^77 + x^75 + x^73 + x^72 + x^71 + x^63 + x^61 + x^59 + x^58 + x^57 + x^55 + x^51 + x^48 + x^47 + x^46 + x^45 + x^39 + x^38 + x^37 + x^34 + x^33 + x^32 + x^29 + x^28 + x^25 + x^23 + x^20 + x^17 + x^11 + x^10 + x^7 + 1 + +46-53-31 383 x^928 + x^898 + x^896 + x^875 + x^870 + x^864 + x^856 + x^854 + x^848 + x^845 + x^834 + x^833 + x^827 + x^817 + x^814 + x^812 + x^808 + x^807 + x^806 + x^804 + x^801 + x^796 + x^794 + x^791 + x^787 + x^786 + x^778 + x^777 + x^776 + x^774 + x^773 + x^772 + x^770 + x^767 + x^765 + x^764 + x^761 + x^757 + x^756 + x^754 + x^746 + x^745 + x^744 + x^741 + x^735 + x^734 + x^730 + x^727 + x^725 + x^723 + x^716 + x^715 + x^714 + x^713 + x^712 + x^705 + x^704 + x^703 + x^702 + x^697 + x^696 + x^692 + x^688 + x^685 + x^684 + x^683 + x^682 + x^681 + x^674 + x^673 + x^672 + x^671 + x^670 + x^667 + x^666 + x^664 + x^663 + x^661 + x^660 + x^658 + x^655 + x^654 + x^653 + x^652 + x^651 + x^650 + x^645 + x^639 + x^637 + x^634 + x^631 + x^630 + x^626 + x^625 + x^624 + x^621 + x^620 + x^618 + x^615 + x^610 + x^608 + x^607 + x^606 + x^605 + x^604 + x^602 + x^596 + x^594 + x^588 + x^586 + x^584 + x^582 + x^580 + x^578 + x^574 + x^573 + x^571 + x^570 + x^568 + x^567 + x^560 + x^553 + x^551 + x^548 + x^545 + x^543 + x^542 + x^538 + x^537 + x^536 + x^534 + x^533 + x^532 + x^530 + x^529 + x^528 + x^523 + x^521 + x^513 + x^512 + x^509 + x^508 + x^505 + x^502 + x^491 + x^490 + x^489 + x^487 + x^481 + x^479 + x^468 + x^466 + x^465 + x^464 + x^461 + x^460 + x^459 + x^457 + x^455 + x^454 + x^453 + x^451 + x^450 + x^444 + x^442 + x^440 + x^438 + x^431 + x^430 + x^429 + x^428 + x^427 + x^426 + x^425 + x^424 + x^423 + x^422 + x^421 + x^416 + x^415 + x^414 + x^412 + x^410 + x^409 + x^408 + x^402 + x^400 + x^399 + x^398 + x^396 + x^395 + x^393 + x^392 + x^391 + x^388 + x^385 + x^384 + x^381 + x^380 + x^379 + x^377 + x^376 + x^374 + x^373 + x^371 + x^368 + x^366 + x^365 + x^364 + x^362 + x^358 + x^357 + x^356 + x^351 + x^348 + x^345 + x^342 + x^339 + x^338 + x^336 + x^334 + x^331 + x^330 + x^328 + x^327 + x^323 + x^320 + x^319 + x^318 + x^312 + x^311 + x^307 + x^306 + x^305 + x^304 + x^303 + x^302 + x^301 + x^298 + x^295 + x^293 + x^292 + x^291 + x^287 + x^286 + x^285 + x^284 + x^283 + x^280 + x^279 + x^278 + x^276 + x^275 + x^271 + x^269 + x^266 + x^264 + x^263 + x^262 + x^260 + x^257 + x^255 + x^254 + x^253 + x^252 + x^251 + x^250 + x^248 + x^242 + x^241 + x^238 + x^234 + x^231 + x^225 + x^223 + x^222 + x^221 + x^220 + x^219 + x^218 + x^217 + x^214 + x^211 + x^209 + x^208 + x^207 + x^206 + x^204 + x^200 + x^196 + x^194 + x^191 + x^188 + x^186 + x^185 + x^182 + x^181 + x^178 + x^176 + x^175 + x^174 + x^169 + x^168 + x^166 + x^164 + x^160 + x^159 + x^158 + x^155 + x^154 + x^153 + x^151 + x^149 + x^147 + x^145 + x^143 + x^138 + x^135 + x^132 + x^131 + x^130 + x^128 + x^127 + x^124 + x^123 + x^121 + x^120 + x^119 + x^117 + x^116 + x^115 + x^112 + x^111 + x^110 + x^109 + x^106 + x^105 + x^104 + x^101 + x^99 + x^96 + x^87 + x^85 + x^84 + x^83 + x^81 + x^80 + x^79 + x^75 + x^73 + x^70 + x^69 + x^68 + x^67 + x^66 + x^63 + x^61 + x^60 + x^58 + x^56 + x^54 + x^53 + x^51 + x^49 + x^47 + x^45 + x^44 + x^43 + x^42 + x^41 + x^39 + x^36 + x^34 + x^31 + x^30 + x^29 + x^27 + x^24 + x^22 + x^14 + x^13 + x^12 + 1 + +1-41-10 385 x^928 + x^898 + x^879 + x^870 + x^855 + x^850 + x^849 + x^831 + x^826 + x^825 + x^819 + x^816 + x^808 + x^807 + x^802 + x^801 + x^800 + x^796 + x^792 + x^789 + x^777 + x^776 + x^768 + x^766 + x^762 + x^758 + x^756 + x^754 + x^748 + x^746 + x^744 + x^742 + x^738 + x^736 + x^735 + x^733 + x^729 + x^728 + x^725 + x^720 + x^716 + x^710 + x^705 + x^704 + x^701 + x^700 + x^696 + x^688 + x^686 + x^682 + x^681 + x^680 + x^679 + x^678 + x^677 + x^676 + x^675 + x^674 + x^671 + x^670 + x^669 + x^666 + x^665 + x^661 + x^660 + x^658 + x^655 + x^653 + x^652 + x^649 + x^648 + x^645 + x^640 + x^637 + x^633 + x^629 + x^628 + x^624 + x^623 + x^621 + x^620 + x^619 + x^618 + x^617 + x^616 + x^610 + x^609 + x^607 + x^605 + x^604 + x^601 + x^600 + x^595 + x^591 + x^589 + x^586 + x^584 + x^581 + x^579 + x^575 + x^573 + x^569 + x^568 + x^561 + x^559 + x^556 + x^555 + x^546 + x^542 + x^540 + x^538 + x^537 + x^534 + x^533 + x^531 + x^530 + x^528 + x^526 + x^525 + x^523 + x^518 + x^517 + x^513 + x^511 + x^507 + x^497 + x^496 + x^494 + x^493 + x^491 + x^489 + x^488 + x^487 + x^485 + x^484 + x^483 + x^482 + x^479 + x^477 + x^476 + x^475 + x^474 + x^471 + x^470 + x^468 + x^466 + x^465 + x^464 + x^463 + x^462 + x^461 + x^459 + x^458 + x^454 + x^453 + x^452 + x^451 + x^449 + x^447 + x^446 + x^442 + x^441 + x^440 + x^434 + x^431 + x^428 + x^427 + x^426 + x^424 + x^421 + x^418 + x^417 + x^416 + x^414 + x^412 + x^411 + x^408 + x^405 + x^401 + x^400 + x^399 + x^398 + x^395 + x^394 + x^393 + x^392 + x^391 + x^388 + x^386 + x^385 + x^381 + x^380 + x^379 + x^378 + x^376 + x^375 + x^373 + x^372 + x^370 + x^367 + x^366 + x^365 + x^364 + x^361 + x^360 + x^359 + x^356 + x^354 + x^353 + x^349 + x^348 + x^346 + x^344 + x^340 + x^335 + x^332 + x^330 + x^325 + x^324 + x^321 + x^318 + x^316 + x^315 + x^314 + x^312 + x^310 + x^309 + x^307 + x^304 + x^301 + x^300 + x^299 + x^298 + x^297 + x^295 + x^291 + x^290 + x^289 + x^288 + x^287 + x^286 + x^285 + x^284 + x^281 + x^280 + x^276 + x^274 + x^269 + x^268 + x^267 + x^265 + x^264 + x^261 + x^260 + x^259 + x^257 + x^256 + x^255 + x^254 + x^251 + x^250 + x^245 + x^242 + x^239 + x^236 + x^233 + x^229 + x^227 + x^225 + x^223 + x^222 + x^219 + x^217 + x^214 + x^213 + x^210 + x^207 + x^205 + x^204 + x^203 + x^202 + x^201 + x^200 + x^199 + x^198 + x^197 + x^196 + x^195 + x^192 + x^190 + x^189 + x^181 + x^180 + x^178 + x^177 + x^175 + x^174 + x^173 + x^172 + x^169 + x^168 + x^165 + x^162 + x^159 + x^158 + x^154 + x^153 + x^152 + x^150 + x^148 + x^144 + x^143 + x^142 + x^141 + x^139 + x^138 + x^137 + x^133 + x^132 + x^128 + x^127 + x^124 + x^123 + x^122 + x^121 + x^118 + x^116 + x^106 + x^102 + x^101 + x^98 + x^96 + x^92 + x^91 + x^90 + x^89 + x^88 + x^86 + x^85 + x^84 + x^83 + x^81 + x^80 + x^78 + x^76 + x^73 + x^72 + x^71 + x^69 + x^67 + x^66 + x^65 + x^63 + x^58 + x^54 + x^53 + x^52 + x^51 + x^50 + x^47 + x^46 + x^45 + x^44 + x^43 + x^40 + x^39 + x^38 + x^37 + x^34 + x^33 + x^32 + x^30 + x^25 + x^23 + x^21 + x^20 + x^19 + x^17 + x^16 + x^15 + x^12 + x^9 + x^8 + x^4 + 1 + +11-33-48 385 x^928 + x^898 + x^882 + x^870 + x^860 + x^853 + x^844 + x^835 + x^828 + x^827 + x^822 + x^819 + x^815 + x^811 + x^808 + x^806 + x^805 + x^794 + x^793 + x^786 + x^784 + x^781 + x^778 + x^773 + x^772 + x^769 + x^765 + x^762 + x^760 + x^759 + x^756 + x^755 + x^748 + x^746 + x^743 + x^739 + x^735 + x^734 + x^733 + x^731 + x^729 + x^727 + x^723 + x^718 + x^713 + x^709 + x^707 + x^706 + x^704 + x^702 + x^699 + x^696 + x^693 + x^692 + x^691 + x^686 + x^685 + x^683 + x^679 + x^677 + x^673 + x^670 + x^669 + x^666 + x^657 + x^656 + x^655 + x^649 + x^648 + x^646 + x^641 + x^639 + x^638 + x^635 + x^633 + x^632 + x^630 + x^628 + x^627 + x^625 + x^620 + x^616 + x^614 + x^613 + x^611 + x^610 + x^609 + x^607 + x^606 + x^600 + x^595 + x^593 + x^592 + x^589 + x^587 + x^584 + x^581 + x^580 + x^579 + x^577 + x^572 + x^570 + x^568 + x^567 + x^566 + x^564 + x^560 + x^559 + x^557 + x^555 + x^554 + x^552 + x^551 + x^546 + x^542 + x^538 + x^535 + x^534 + x^533 + x^531 + x^530 + x^528 + x^526 + x^524 + x^523 + x^520 + x^519 + x^511 + x^509 + x^506 + x^504 + x^503 + x^502 + x^501 + x^500 + x^499 + x^498 + x^495 + x^494 + x^491 + x^490 + x^489 + x^488 + x^487 + x^485 + x^484 + x^479 + x^475 + x^473 + x^466 + x^459 + x^457 + x^456 + x^455 + x^454 + x^451 + x^450 + x^449 + x^448 + x^447 + x^446 + x^444 + x^443 + x^442 + x^441 + x^440 + x^433 + x^432 + x^430 + x^428 + x^427 + x^425 + x^423 + x^419 + x^418 + x^417 + x^414 + x^409 + x^408 + x^406 + x^404 + x^402 + x^400 + x^398 + x^397 + x^396 + x^395 + x^393 + x^389 + x^388 + x^385 + x^379 + x^377 + x^375 + x^374 + x^369 + x^365 + x^364 + x^363 + x^362 + x^361 + x^359 + x^355 + x^354 + x^350 + x^348 + x^345 + x^344 + x^339 + x^336 + x^335 + x^334 + x^333 + x^332 + x^331 + x^330 + x^329 + x^328 + x^326 + x^319 + x^318 + x^316 + x^315 + x^314 + x^313 + x^312 + x^309 + x^308 + x^304 + x^303 + x^302 + x^299 + x^298 + x^297 + x^295 + x^294 + x^293 + x^292 + x^291 + x^290 + x^288 + x^285 + x^282 + x^281 + x^279 + x^277 + x^276 + x^274 + x^272 + x^271 + x^270 + x^269 + x^268 + x^267 + x^265 + x^264 + x^262 + x^261 + x^260 + x^257 + x^253 + x^251 + x^250 + x^247 + x^246 + x^243 + x^242 + x^241 + x^240 + x^237 + x^236 + x^233 + x^232 + x^230 + x^229 + x^226 + x^223 + x^222 + x^221 + x^220 + x^217 + x^215 + x^211 + x^210 + x^209 + x^207 + x^206 + x^205 + x^203 + x^201 + x^200 + x^198 + x^196 + x^195 + x^193 + x^189 + x^188 + x^186 + x^184 + x^180 + x^179 + x^177 + x^176 + x^170 + x^168 + x^163 + x^162 + x^160 + x^159 + x^158 + x^157 + x^156 + x^154 + x^151 + x^149 + x^148 + x^145 + x^144 + x^141 + x^140 + x^138 + x^137 + x^136 + x^135 + x^134 + x^133 + x^132 + x^131 + x^130 + x^129 + x^125 + x^124 + x^121 + x^117 + x^116 + x^110 + x^109 + x^108 + x^106 + x^104 + x^103 + x^102 + x^99 + x^98 + x^96 + x^95 + x^93 + x^92 + x^90 + x^89 + x^88 + x^85 + x^83 + x^81 + x^78 + x^76 + x^72 + x^71 + x^70 + x^68 + x^67 + x^66 + x^62 + x^59 + x^58 + x^56 + x^51 + x^48 + x^46 + x^45 + x^44 + x^43 + x^41 + x^40 + x^38 + x^37 + x^35 + x^34 + x^33 + x^32 + x^27 + x^22 + x^19 + x^13 + 1 + +19-8-4 387 x^928 + x^898 + x^870 + x^844 + x^838 + x^834 + x^828 + x^818 + x^814 + x^806 + x^804 + x^803 + x^797 + x^782 + x^778 + x^776 + x^774 + x^773 + x^772 + x^768 + x^767 + x^764 + x^752 + x^751 + x^745 + x^744 + x^742 + x^740 + x^738 + x^736 + x^733 + x^730 + x^721 + x^718 + x^715 + x^714 + x^710 + x^707 + x^704 + x^702 + x^700 + x^699 + x^698 + x^697 + x^694 + x^692 + x^691 + x^688 + x^686 + x^685 + x^682 + x^676 + x^673 + x^671 + x^670 + x^669 + x^666 + x^662 + x^661 + x^658 + x^656 + x^655 + x^654 + x^652 + x^649 + x^648 + x^647 + x^645 + x^644 + x^642 + x^640 + x^638 + x^636 + x^632 + x^631 + x^630 + x^623 + x^622 + x^621 + x^616 + x^614 + x^612 + x^611 + x^606 + x^605 + x^602 + x^598 + x^597 + x^594 + x^593 + x^591 + x^590 + x^588 + x^587 + x^586 + x^584 + x^582 + x^581 + x^579 + x^577 + x^574 + x^572 + x^571 + x^568 + x^567 + x^566 + x^563 + x^562 + x^560 + x^559 + x^558 + x^556 + x^553 + x^551 + x^550 + x^549 + x^546 + x^545 + x^543 + x^542 + x^538 + x^536 + x^535 + x^534 + x^533 + x^531 + x^530 + x^529 + x^527 + x^525 + x^524 + x^522 + x^521 + x^520 + x^518 + x^517 + x^513 + x^512 + x^510 + x^509 + x^507 + x^506 + x^505 + x^504 + x^501 + x^500 + x^499 + x^493 + x^492 + x^491 + x^489 + x^488 + x^486 + x^485 + x^484 + x^480 + x^479 + x^478 + x^477 + x^474 + x^473 + x^472 + x^470 + x^468 + x^464 + x^462 + x^458 + x^456 + x^455 + x^454 + x^453 + x^452 + x^451 + x^450 + x^449 + x^447 + x^446 + x^445 + x^441 + x^439 + x^438 + x^437 + x^436 + x^434 + x^433 + x^432 + x^431 + x^429 + x^426 + x^425 + x^424 + x^423 + x^421 + x^419 + x^418 + x^416 + x^414 + x^412 + x^410 + x^407 + x^405 + x^404 + x^398 + x^397 + x^394 + x^393 + x^392 + x^391 + x^387 + x^386 + x^385 + x^382 + x^381 + x^379 + x^377 + x^375 + x^374 + x^369 + x^368 + x^365 + x^364 + x^363 + x^362 + x^361 + x^359 + x^358 + x^356 + x^355 + x^352 + x^351 + x^350 + x^345 + x^343 + x^341 + x^340 + x^338 + x^333 + x^331 + x^330 + x^329 + x^328 + x^327 + x^325 + x^324 + x^323 + x^322 + x^318 + x^315 + x^312 + x^310 + x^308 + x^305 + x^304 + x^303 + x^299 + x^296 + x^295 + x^294 + x^290 + x^289 + x^288 + x^285 + x^284 + x^281 + x^280 + x^275 + x^274 + x^273 + x^271 + x^268 + x^266 + x^264 + x^263 + x^262 + x^261 + x^260 + x^258 + x^256 + x^255 + x^254 + x^251 + x^245 + x^240 + x^237 + x^236 + x^231 + x^230 + x^229 + x^228 + x^223 + x^221 + x^219 + x^218 + x^215 + x^210 + x^208 + x^207 + x^206 + x^204 + x^202 + x^197 + x^196 + x^195 + x^194 + x^193 + x^192 + x^187 + x^186 + x^184 + x^183 + x^182 + x^180 + x^178 + x^177 + x^175 + x^174 + x^169 + x^168 + x^166 + x^165 + x^163 + x^160 + x^159 + x^156 + x^155 + x^152 + x^150 + x^145 + x^140 + x^138 + x^137 + x^136 + x^135 + x^134 + x^133 + x^131 + x^128 + x^127 + x^126 + x^122 + x^121 + x^118 + x^117 + x^114 + x^109 + x^105 + x^102 + x^101 + x^100 + x^99 + x^96 + x^94 + x^93 + x^91 + x^90 + x^89 + x^88 + x^87 + x^81 + x^79 + x^76 + x^74 + x^71 + x^70 + x^64 + x^63 + x^61 + x^59 + x^56 + x^55 + x^49 + x^47 + x^46 + x^44 + x^43 + x^40 + x^35 + x^34 + x^33 + x^31 + x^28 + x^26 + x^23 + x^21 + x^18 + x^13 + 1 + +36-23-57 387 x^928 + x^898 + x^897 + x^890 + x^870 + x^860 + x^859 + x^854 + x^853 + x^846 + x^836 + x^835 + x^824 + x^816 + x^810 + x^809 + x^808 + x^804 + x^802 + x^792 + x^791 + x^785 + x^784 + x^780 + x^778 + x^777 + x^776 + x^775 + x^774 + x^773 + x^765 + x^755 + x^753 + x^748 + x^747 + x^744 + x^742 + x^733 + x^732 + x^728 + x^725 + x^724 + x^721 + x^716 + x^714 + x^713 + x^703 + x^701 + x^700 + x^699 + x^697 + x^693 + x^692 + x^691 + x^690 + x^689 + x^680 + x^679 + x^675 + x^673 + x^672 + x^666 + x^663 + x^662 + x^661 + x^658 + x^656 + x^655 + x^653 + x^652 + x^649 + x^647 + x^645 + x^642 + x^641 + x^639 + x^638 + x^637 + x^633 + x^630 + x^629 + x^625 + x^620 + x^619 + x^614 + x^613 + x^612 + x^611 + x^610 + x^606 + x^605 + x^604 + x^603 + x^601 + x^600 + x^599 + x^597 + x^593 + x^591 + x^589 + x^587 + x^586 + x^584 + x^583 + x^581 + x^579 + x^576 + x^572 + x^571 + x^566 + x^565 + x^563 + x^559 + x^554 + x^552 + x^549 + x^548 + x^545 + x^544 + x^543 + x^540 + x^535 + x^534 + x^533 + x^531 + x^529 + x^524 + x^522 + x^517 + x^516 + x^513 + x^512 + x^511 + x^509 + x^507 + x^506 + x^505 + x^503 + x^502 + x^496 + x^495 + x^494 + x^492 + x^491 + x^490 + x^489 + x^487 + x^478 + x^477 + x^475 + x^474 + x^472 + x^471 + x^470 + x^469 + x^466 + x^463 + x^461 + x^459 + x^458 + x^457 + x^456 + x^455 + x^454 + x^453 + x^452 + x^448 + x^447 + x^446 + x^444 + x^442 + x^440 + x^439 + x^438 + x^437 + x^434 + x^431 + x^428 + x^427 + x^426 + x^425 + x^424 + x^423 + x^414 + x^409 + x^408 + x^407 + x^403 + x^401 + x^400 + x^396 + x^395 + x^394 + x^390 + x^389 + x^388 + x^387 + x^386 + x^379 + x^378 + x^377 + x^375 + x^374 + x^370 + x^366 + x^360 + x^359 + x^358 + x^357 + x^356 + x^355 + x^353 + x^351 + x^350 + x^349 + x^348 + x^347 + x^345 + x^343 + x^342 + x^341 + x^340 + x^339 + x^338 + x^336 + x^335 + x^330 + x^329 + x^327 + x^325 + x^324 + x^323 + x^322 + x^319 + x^318 + x^315 + x^313 + x^309 + x^305 + x^304 + x^302 + x^300 + x^298 + x^296 + x^291 + x^289 + x^286 + x^285 + x^284 + x^283 + x^282 + x^281 + x^279 + x^275 + x^274 + x^272 + x^271 + x^265 + x^264 + x^263 + x^259 + x^258 + x^257 + x^253 + x^252 + x^249 + x^246 + x^245 + x^244 + x^242 + x^239 + x^233 + x^232 + x^231 + x^230 + x^229 + x^228 + x^221 + x^217 + x^216 + x^212 + x^211 + x^209 + x^208 + x^205 + x^202 + x^201 + x^200 + x^199 + x^198 + x^197 + x^196 + x^195 + x^193 + x^192 + x^191 + x^189 + x^187 + x^186 + x^183 + x^182 + x^181 + x^177 + x^174 + x^172 + x^170 + x^166 + x^164 + x^163 + x^162 + x^160 + x^156 + x^155 + x^152 + x^151 + x^147 + x^144 + x^143 + x^142 + x^141 + x^138 + x^137 + x^135 + x^133 + x^130 + x^129 + x^126 + x^125 + x^123 + x^121 + x^120 + x^117 + x^114 + x^112 + x^107 + x^106 + x^103 + x^102 + x^100 + x^98 + x^95 + x^94 + x^93 + x^91 + x^90 + x^89 + x^87 + x^85 + x^84 + x^82 + x^81 + x^80 + x^78 + x^76 + x^75 + x^71 + x^70 + x^69 + x^68 + x^66 + x^65 + x^61 + x^60 + x^59 + x^58 + x^56 + x^55 + x^51 + x^49 + x^48 + x^46 + x^44 + x^42 + x^36 + x^34 + x^33 + x^32 + x^29 + x^24 + x^23 + x^20 + x^14 + x^13 + x^12 + x^11 + x^10 + 1 + +44-4-19 389 x^928 + x^898 + x^874 + x^870 + x^836 + x^832 + x^824 + x^812 + x^809 + x^802 + x^794 + x^793 + x^789 + x^785 + x^781 + x^766 + x^764 + x^763 + x^762 + x^759 + x^758 + x^751 + x^750 + x^746 + x^744 + x^742 + x^738 + x^734 + x^733 + x^732 + x^729 + x^721 + x^719 + x^716 + x^715 + x^714 + x^713 + x^710 + x^706 + x^705 + x^703 + x^702 + x^700 + x^699 + x^698 + x^696 + x^693 + x^691 + x^690 + x^689 + x^688 + x^684 + x^680 + x^678 + x^677 + x^674 + x^672 + x^669 + x^667 + x^665 + x^663 + x^662 + x^660 + x^659 + x^658 + x^656 + x^652 + x^651 + x^649 + x^646 + x^645 + x^642 + x^641 + x^640 + x^633 + x^632 + x^630 + x^627 + x^626 + x^622 + x^617 + x^612 + x^611 + x^610 + x^606 + x^605 + x^604 + x^603 + x^602 + x^601 + x^599 + x^597 + x^596 + x^594 + x^593 + x^592 + x^591 + x^582 + x^581 + x^580 + x^578 + x^577 + x^575 + x^574 + x^573 + x^568 + x^567 + x^564 + x^560 + x^559 + x^558 + x^556 + x^554 + x^550 + x^548 + x^547 + x^545 + x^544 + x^541 + x^540 + x^539 + x^538 + x^536 + x^535 + x^534 + x^532 + x^531 + x^529 + x^523 + x^521 + x^518 + x^517 + x^516 + x^514 + x^511 + x^510 + x^508 + x^507 + x^505 + x^504 + x^499 + x^498 + x^495 + x^494 + x^492 + x^491 + x^489 + x^488 + x^487 + x^486 + x^484 + x^483 + x^482 + x^481 + x^478 + x^477 + x^476 + x^475 + x^471 + x^470 + x^468 + x^465 + x^464 + x^460 + x^459 + x^455 + x^454 + x^453 + x^451 + x^445 + x^438 + x^434 + x^432 + x^431 + x^430 + x^429 + x^428 + x^427 + x^424 + x^423 + x^419 + x^418 + x^417 + x^416 + x^415 + x^413 + x^411 + x^409 + x^408 + x^406 + x^405 + x^404 + x^403 + x^402 + x^400 + x^397 + x^395 + x^390 + x^389 + x^387 + x^386 + x^385 + x^384 + x^382 + x^380 + x^379 + x^377 + x^376 + x^374 + x^373 + x^370 + x^369 + x^366 + x^365 + x^364 + x^362 + x^360 + x^359 + x^357 + x^356 + x^355 + x^354 + x^352 + x^349 + x^345 + x^342 + x^341 + x^340 + x^338 + x^334 + x^333 + x^326 + x^324 + x^322 + x^321 + x^319 + x^318 + x^317 + x^315 + x^312 + x^311 + x^309 + x^305 + x^304 + x^302 + x^298 + x^297 + x^296 + x^294 + x^293 + x^290 + x^289 + x^287 + x^286 + x^285 + x^279 + x^277 + x^275 + x^273 + x^268 + x^265 + x^264 + x^263 + x^260 + x^258 + x^254 + x^251 + x^249 + x^248 + x^247 + x^245 + x^244 + x^242 + x^241 + x^240 + x^239 + x^232 + x^229 + x^226 + x^225 + x^223 + x^222 + x^220 + x^219 + x^218 + x^215 + x^214 + x^213 + x^212 + x^210 + x^209 + x^206 + x^203 + x^202 + x^199 + x^197 + x^195 + x^194 + x^192 + x^191 + x^189 + x^188 + x^185 + x^184 + x^181 + x^177 + x^176 + x^175 + x^174 + x^172 + x^171 + x^170 + x^169 + x^167 + x^164 + x^162 + x^160 + x^159 + x^157 + x^156 + x^154 + x^153 + x^152 + x^150 + x^148 + x^145 + x^144 + x^139 + x^138 + x^137 + x^136 + x^133 + x^126 + x^124 + x^123 + x^122 + x^121 + x^120 + x^119 + x^113 + x^111 + x^108 + x^105 + x^104 + x^103 + x^102 + x^101 + x^98 + x^97 + x^96 + x^95 + x^92 + x^86 + x^85 + x^83 + x^81 + x^80 + x^77 + x^74 + x^73 + x^72 + x^71 + x^70 + x^68 + x^67 + x^66 + x^64 + x^62 + x^61 + x^60 + x^59 + x^57 + x^53 + x^51 + x^43 + x^41 + x^40 + x^39 + x^35 + x^30 + x^29 + x^27 + x^26 + x^24 + x^20 + x^18 + x^14 + x^6 + 1 + +8-3-35 389 x^928 + x^898 + x^895 + x^870 + x^865 + x^835 + x^832 + x^824 + x^821 + x^812 + x^810 + x^808 + x^807 + x^805 + x^804 + x^794 + x^791 + x^788 + x^778 + x^777 + x^774 + x^772 + x^771 + x^763 + x^755 + x^752 + x^750 + x^747 + x^746 + x^742 + x^739 + x^733 + x^731 + x^728 + x^725 + x^722 + x^721 + x^720 + x^719 + x^716 + x^714 + x^713 + x^712 + x^711 + x^708 + x^705 + x^704 + x^701 + x^700 + x^698 + x^695 + x^694 + x^692 + x^689 + x^683 + x^680 + x^675 + x^674 + x^670 + x^668 + x^667 + x^664 + x^662 + x^661 + x^658 + x^656 + x^654 + x^651 + x^650 + x^645 + x^643 + x^640 + x^639 + x^637 + x^635 + x^634 + x^633 + x^631 + x^630 + x^628 + x^621 + x^620 + x^616 + x^615 + x^614 + x^613 + x^608 + x^605 + x^604 + x^602 + x^601 + x^598 + x^597 + x^591 + x^590 + x^588 + x^587 + x^581 + x^579 + x^578 + x^577 + x^576 + x^573 + x^572 + x^571 + x^570 + x^569 + x^568 + x^566 + x^564 + x^563 + x^562 + x^559 + x^557 + x^553 + x^550 + x^549 + x^547 + x^546 + x^540 + x^537 + x^536 + x^534 + x^530 + x^527 + x^526 + x^525 + x^524 + x^522 + x^521 + x^520 + x^519 + x^518 + x^510 + x^509 + x^505 + x^504 + x^498 + x^496 + x^493 + x^492 + x^490 + x^489 + x^488 + x^487 + x^484 + x^483 + x^482 + x^481 + x^480 + x^479 + x^477 + x^475 + x^474 + x^472 + x^470 + x^468 + x^465 + x^464 + x^463 + x^461 + x^458 + x^457 + x^455 + x^454 + x^453 + x^452 + x^451 + x^450 + x^448 + x^447 + x^445 + x^441 + x^440 + x^437 + x^435 + x^434 + x^433 + x^432 + x^431 + x^425 + x^421 + x^418 + x^415 + x^413 + x^412 + x^411 + x^410 + x^409 + x^408 + x^404 + x^403 + x^402 + x^399 + x^398 + x^394 + x^393 + x^391 + x^390 + x^389 + x^384 + x^383 + x^381 + x^380 + x^379 + x^378 + x^375 + x^374 + x^372 + x^370 + x^369 + x^367 + x^365 + x^364 + x^363 + x^362 + x^358 + x^357 + x^355 + x^354 + x^353 + x^351 + x^349 + x^347 + x^343 + x^342 + x^338 + x^337 + x^336 + x^334 + x^332 + x^329 + x^328 + x^327 + x^326 + x^323 + x^321 + x^320 + x^319 + x^317 + x^315 + x^311 + x^308 + x^305 + x^304 + x^303 + x^300 + x^298 + x^294 + x^293 + x^292 + x^290 + x^283 + x^281 + x^280 + x^279 + x^276 + x^274 + x^273 + x^269 + x^268 + x^267 + x^266 + x^264 + x^263 + x^259 + x^256 + x^248 + x^247 + x^246 + x^245 + x^244 + x^243 + x^242 + x^241 + x^240 + x^239 + x^235 + x^234 + x^231 + x^228 + x^227 + x^226 + x^225 + x^224 + x^220 + x^219 + x^218 + x^217 + x^216 + x^213 + x^211 + x^210 + x^206 + x^203 + x^200 + x^197 + x^196 + x^195 + x^193 + x^192 + x^191 + x^185 + x^184 + x^180 + x^178 + x^176 + x^175 + x^173 + x^170 + x^169 + x^167 + x^166 + x^165 + x^164 + x^162 + x^161 + x^160 + x^159 + x^154 + x^153 + x^152 + x^147 + x^141 + x^140 + x^138 + x^135 + x^133 + x^132 + x^131 + x^129 + x^127 + x^126 + x^125 + x^119 + x^115 + x^112 + x^111 + x^108 + x^106 + x^99 + x^97 + x^95 + x^94 + x^93 + x^92 + x^91 + x^89 + x^87 + x^84 + x^82 + x^81 + x^78 + x^76 + x^74 + x^69 + x^68 + x^67 + x^65 + x^63 + x^61 + x^58 + x^57 + x^56 + x^55 + x^54 + x^53 + x^51 + x^49 + x^48 + x^47 + x^46 + x^43 + x^40 + x^39 + x^35 + x^34 + x^32 + x^30 + x^28 + x^27 + x^25 + x^24 + x^21 + x^20 + x^15 + x^14 + x^10 + 1 + +1-12-46 393 x^928 + x^898 + x^896 + x^870 + x^866 + x^836 + x^834 + x^828 + x^826 + x^808 + x^806 + x^804 + x^802 + x^796 + x^794 + x^778 + x^774 + x^767 + x^766 + x^758 + x^756 + x^744 + x^742 + x^740 + x^738 + x^736 + x^732 + x^731 + x^730 + x^727 + x^726 + x^718 + x^712 + x^710 + x^708 + x^707 + x^706 + x^704 + x^703 + x^701 + x^700 + x^699 + x^698 + x^697 + x^696 + x^695 + x^692 + x^686 + x^685 + x^680 + x^676 + x^672 + x^669 + x^668 + x^665 + x^664 + x^662 + x^661 + x^657 + x^656 + x^655 + x^653 + x^647 + x^646 + x^643 + x^640 + x^639 + x^638 + x^635 + x^628 + x^626 + x^623 + x^622 + x^621 + x^615 + x^612 + x^610 + x^609 + x^608 + x^607 + x^605 + x^604 + x^603 + x^602 + x^600 + x^599 + x^598 + x^597 + x^590 + x^589 + x^587 + x^586 + x^584 + x^583 + x^582 + x^581 + x^579 + x^578 + x^577 + x^576 + x^574 + x^573 + x^571 + x^567 + x^563 + x^561 + x^560 + x^559 + x^556 + x^554 + x^550 + x^548 + x^545 + x^544 + x^542 + x^535 + x^530 + x^529 + x^528 + x^526 + x^524 + x^522 + x^521 + x^519 + x^517 + x^516 + x^515 + x^512 + x^508 + x^507 + x^506 + x^505 + x^504 + x^499 + x^498 + x^497 + x^496 + x^495 + x^494 + x^489 + x^488 + x^487 + x^486 + x^484 + x^483 + x^481 + x^477 + x^476 + x^475 + x^474 + x^473 + x^471 + x^469 + x^466 + x^465 + x^464 + x^461 + x^460 + x^459 + x^457 + x^455 + x^452 + x^451 + x^449 + x^448 + x^447 + x^446 + x^445 + x^443 + x^442 + x^440 + x^439 + x^438 + x^433 + x^427 + x^425 + x^424 + x^422 + x^421 + x^419 + x^416 + x^415 + x^414 + x^413 + x^412 + x^408 + x^405 + x^404 + x^403 + x^402 + x^399 + x^398 + x^397 + x^396 + x^394 + x^393 + x^392 + x^391 + x^389 + x^388 + x^387 + x^386 + x^384 + x^381 + x^380 + x^376 + x^375 + x^374 + x^373 + x^366 + x^363 + x^362 + x^359 + x^358 + x^357 + x^356 + x^355 + x^353 + x^352 + x^351 + x^347 + x^344 + x^343 + x^342 + x^341 + x^339 + x^337 + x^336 + x^333 + x^331 + x^329 + x^328 + x^327 + x^326 + x^325 + x^321 + x^320 + x^317 + x^316 + x^315 + x^312 + x^311 + x^310 + x^307 + x^306 + x^303 + x^300 + x^298 + x^296 + x^295 + x^292 + x^291 + x^290 + x^287 + x^283 + x^279 + x^278 + x^277 + x^276 + x^273 + x^272 + x^271 + x^270 + x^269 + x^263 + x^262 + x^257 + x^256 + x^255 + x^254 + x^252 + x^249 + x^247 + x^245 + x^242 + x^241 + x^240 + x^235 + x^234 + x^233 + x^232 + x^231 + x^230 + x^229 + x^227 + x^225 + x^224 + x^223 + x^222 + x^219 + x^218 + x^217 + x^216 + x^215 + x^214 + x^213 + x^211 + x^206 + x^205 + x^203 + x^200 + x^199 + x^196 + x^194 + x^191 + x^186 + x^185 + x^183 + x^180 + x^179 + x^177 + x^176 + x^175 + x^174 + x^172 + x^171 + x^170 + x^169 + x^167 + x^160 + x^159 + x^158 + x^157 + x^156 + x^155 + x^154 + x^152 + x^146 + x^145 + x^144 + x^143 + x^142 + x^141 + x^139 + x^138 + x^137 + x^134 + x^133 + x^129 + x^126 + x^125 + x^124 + x^122 + x^121 + x^119 + x^118 + x^117 + x^113 + x^111 + x^110 + x^109 + x^108 + x^99 + x^98 + x^97 + x^95 + x^93 + x^92 + x^90 + x^85 + x^78 + x^76 + x^75 + x^70 + x^69 + x^65 + x^64 + x^63 + x^62 + x^59 + x^56 + x^54 + x^53 + x^51 + x^50 + x^49 + x^45 + x^44 + x^42 + x^41 + x^39 + x^36 + x^35 + x^34 + x^32 + x^31 + x^30 + x^23 + x^20 + x^16 + x^13 + x^12 + 1 + +21-29-14 393 x^928 + x^898 + x^885 + x^870 + x^858 + x^856 + x^855 + x^831 + x^829 + x^828 + x^826 + x^815 + x^813 + x^812 + x^808 + x^804 + x^802 + x^801 + x^799 + x^798 + x^796 + x^788 + x^785 + x^778 + x^777 + x^775 + x^774 + x^772 + x^769 + x^768 + x^766 + x^765 + x^758 + x^752 + x^748 + x^747 + x^742 + x^741 + x^739 + x^738 + x^736 + x^735 + x^732 + x^728 + x^726 + x^725 + x^721 + x^718 + x^717 + x^714 + x^712 + x^709 + x^708 + x^706 + x^705 + x^702 + x^701 + x^699 + x^698 + x^694 + x^691 + x^687 + x^685 + x^684 + x^682 + x^681 + x^680 + x^679 + x^676 + x^675 + x^674 + x^671 + x^667 + x^664 + x^658 + x^657 + x^655 + x^653 + x^652 + x^650 + x^649 + x^648 + x^647 + x^646 + x^644 + x^641 + x^640 + x^637 + x^635 + x^632 + x^627 + x^625 + x^623 + x^622 + x^619 + x^618 + x^617 + x^613 + x^610 + x^608 + x^601 + x^595 + x^593 + x^592 + x^586 + x^585 + x^583 + x^582 + x^577 + x^572 + x^571 + x^570 + x^568 + x^565 + x^563 + x^561 + x^559 + x^557 + x^555 + x^553 + x^550 + x^548 + x^545 + x^544 + x^538 + x^537 + x^534 + x^532 + x^529 + x^526 + x^525 + x^521 + x^520 + x^518 + x^512 + x^511 + x^510 + x^508 + x^507 + x^503 + x^501 + x^500 + x^499 + x^498 + x^497 + x^496 + x^494 + x^489 + x^486 + x^485 + x^481 + x^480 + x^477 + x^476 + x^475 + x^466 + x^464 + x^462 + x^461 + x^460 + x^459 + x^458 + x^454 + x^452 + x^451 + x^449 + x^447 + x^446 + x^444 + x^441 + x^440 + x^439 + x^433 + x^431 + x^430 + x^429 + x^427 + x^425 + x^424 + x^422 + x^416 + x^414 + x^412 + x^408 + x^406 + x^403 + x^402 + x^401 + x^399 + x^398 + x^395 + x^393 + x^390 + x^389 + x^387 + x^386 + x^385 + x^384 + x^383 + x^381 + x^380 + x^378 + x^375 + x^374 + x^372 + x^371 + x^370 + x^368 + x^367 + x^362 + x^361 + x^358 + x^356 + x^354 + x^353 + x^351 + x^350 + x^347 + x^346 + x^343 + x^340 + x^339 + x^337 + x^335 + x^331 + x^330 + x^329 + x^328 + x^326 + x^325 + x^323 + x^322 + x^319 + x^318 + x^316 + x^315 + x^311 + x^310 + x^309 + x^308 + x^306 + x^304 + x^303 + x^298 + x^296 + x^295 + x^294 + x^292 + x^291 + x^290 + x^289 + x^287 + x^284 + x^281 + x^280 + x^276 + x^274 + x^272 + x^271 + x^270 + x^268 + x^267 + x^259 + x^258 + x^257 + x^256 + x^255 + x^254 + x^249 + x^248 + x^247 + x^246 + x^244 + x^243 + x^241 + x^240 + x^236 + x^234 + x^233 + x^232 + x^229 + x^224 + x^221 + x^220 + x^217 + x^216 + x^215 + x^213 + x^211 + x^210 + x^208 + x^207 + x^200 + x^199 + x^198 + x^197 + x^196 + x^195 + x^193 + x^192 + x^191 + x^190 + x^185 + x^184 + x^183 + x^181 + x^179 + x^176 + x^173 + x^170 + x^169 + x^168 + x^164 + x^162 + x^160 + x^157 + x^155 + x^154 + x^152 + x^151 + x^150 + x^149 + x^146 + x^144 + x^143 + x^142 + x^140 + x^136 + x^134 + x^132 + x^130 + x^129 + x^128 + x^126 + x^123 + x^120 + x^117 + x^112 + x^111 + x^108 + x^107 + x^105 + x^103 + x^100 + x^99 + x^96 + x^95 + x^91 + x^90 + x^89 + x^87 + x^86 + x^84 + x^83 + x^81 + x^79 + x^78 + x^76 + x^69 + x^68 + x^67 + x^66 + x^62 + x^61 + x^57 + x^56 + x^55 + x^53 + x^52 + x^51 + x^49 + x^48 + x^46 + x^43 + x^40 + x^38 + x^37 + x^36 + x^35 + x^34 + x^33 + x^31 + x^30 + x^29 + x^28 + x^27 + x^26 + x^24 + x^21 + x^20 + x^17 + x^13 + 1 + +37-29-44 393 x^928 + x^898 + x^885 + x^870 + x^858 + x^856 + x^855 + x^831 + x^829 + x^828 + x^826 + x^815 + x^813 + x^812 + x^808 + x^804 + x^802 + x^801 + x^799 + x^798 + x^796 + x^788 + x^785 + x^778 + x^777 + x^775 + x^774 + x^772 + x^769 + x^768 + x^766 + x^765 + x^758 + x^752 + x^748 + x^747 + x^742 + x^741 + x^739 + x^738 + x^736 + x^735 + x^732 + x^728 + x^726 + x^725 + x^721 + x^718 + x^717 + x^714 + x^712 + x^709 + x^708 + x^706 + x^705 + x^702 + x^701 + x^699 + x^698 + x^694 + x^691 + x^687 + x^685 + x^684 + x^682 + x^681 + x^680 + x^679 + x^676 + x^675 + x^674 + x^671 + x^667 + x^664 + x^658 + x^657 + x^655 + x^653 + x^652 + x^650 + x^649 + x^648 + x^647 + x^646 + x^644 + x^641 + x^640 + x^637 + x^635 + x^632 + x^627 + x^625 + x^623 + x^622 + x^619 + x^618 + x^617 + x^613 + x^610 + x^608 + x^601 + x^595 + x^593 + x^592 + x^586 + x^585 + x^583 + x^582 + x^577 + x^572 + x^571 + x^570 + x^568 + x^565 + x^563 + x^561 + x^559 + x^557 + x^555 + x^553 + x^550 + x^548 + x^545 + x^544 + x^538 + x^537 + x^534 + x^532 + x^529 + x^526 + x^525 + x^521 + x^520 + x^518 + x^512 + x^511 + x^510 + x^508 + x^507 + x^503 + x^501 + x^500 + x^499 + x^498 + x^497 + x^496 + x^494 + x^489 + x^486 + x^485 + x^481 + x^480 + x^477 + x^476 + x^475 + x^466 + x^464 + x^462 + x^461 + x^460 + x^459 + x^458 + x^454 + x^452 + x^451 + x^449 + x^447 + x^446 + x^444 + x^441 + x^440 + x^439 + x^433 + x^431 + x^430 + x^429 + x^427 + x^425 + x^424 + x^422 + x^416 + x^414 + x^412 + x^408 + x^406 + x^403 + x^402 + x^401 + x^399 + x^398 + x^395 + x^393 + x^390 + x^389 + x^387 + x^386 + x^385 + x^384 + x^383 + x^381 + x^380 + x^378 + x^375 + x^374 + x^372 + x^371 + x^370 + x^368 + x^367 + x^362 + x^361 + x^358 + x^356 + x^354 + x^353 + x^351 + x^350 + x^347 + x^346 + x^343 + x^340 + x^339 + x^337 + x^335 + x^331 + x^330 + x^329 + x^328 + x^326 + x^325 + x^323 + x^322 + x^319 + x^318 + x^316 + x^315 + x^311 + x^310 + x^309 + x^308 + x^306 + x^304 + x^303 + x^298 + x^296 + x^295 + x^294 + x^292 + x^291 + x^290 + x^289 + x^287 + x^284 + x^281 + x^280 + x^276 + x^274 + x^272 + x^271 + x^270 + x^268 + x^267 + x^259 + x^258 + x^257 + x^256 + x^255 + x^254 + x^249 + x^248 + x^247 + x^246 + x^244 + x^243 + x^241 + x^240 + x^236 + x^234 + x^233 + x^232 + x^229 + x^224 + x^221 + x^220 + x^217 + x^216 + x^215 + x^213 + x^211 + x^210 + x^208 + x^207 + x^200 + x^199 + x^198 + x^197 + x^196 + x^195 + x^193 + x^192 + x^191 + x^190 + x^185 + x^184 + x^183 + x^181 + x^179 + x^176 + x^173 + x^170 + x^169 + x^168 + x^164 + x^162 + x^160 + x^157 + x^155 + x^154 + x^152 + x^151 + x^150 + x^149 + x^146 + x^144 + x^143 + x^142 + x^140 + x^136 + x^134 + x^132 + x^130 + x^129 + x^128 + x^126 + x^123 + x^120 + x^117 + x^112 + x^111 + x^108 + x^107 + x^105 + x^103 + x^100 + x^99 + x^96 + x^95 + x^91 + x^90 + x^89 + x^87 + x^86 + x^84 + x^83 + x^81 + x^79 + x^78 + x^76 + x^69 + x^68 + x^67 + x^66 + x^62 + x^61 + x^57 + x^56 + x^55 + x^53 + x^52 + x^51 + x^49 + x^48 + x^46 + x^43 + x^40 + x^38 + x^37 + x^36 + x^35 + x^34 + x^33 + x^31 + x^30 + x^29 + x^28 + x^27 + x^26 + x^24 + x^21 + x^20 + x^17 + x^13 + 1 + +7-48-42 393 x^928 + x^898 + x^892 + x^870 + x^862 + x^843 + x^820 + x^813 + x^808 + x^800 + x^794 + x^784 + x^783 + x^780 + x^778 + x^774 + x^771 + x^770 + x^767 + x^760 + x^757 + x^753 + x^750 + x^748 + x^743 + x^741 + x^740 + x^737 + x^731 + x^727 + x^724 + x^723 + x^718 + x^713 + x^707 + x^705 + x^704 + x^701 + x^700 + x^697 + x^695 + x^693 + x^688 + x^684 + x^683 + x^682 + x^681 + x^678 + x^677 + x^674 + x^671 + x^669 + x^667 + x^664 + x^663 + x^660 + x^653 + x^652 + x^651 + x^650 + x^649 + x^645 + x^644 + x^641 + x^639 + x^638 + x^637 + x^634 + x^633 + x^630 + x^628 + x^625 + x^623 + x^621 + x^620 + x^619 + x^615 + x^612 + x^611 + x^608 + x^607 + x^605 + x^604 + x^603 + x^599 + x^595 + x^593 + x^585 + x^584 + x^581 + x^580 + x^578 + x^577 + x^575 + x^573 + x^572 + x^569 + x^568 + x^565 + x^564 + x^563 + x^561 + x^558 + x^557 + x^551 + x^550 + x^548 + x^547 + x^544 + x^538 + x^537 + x^534 + x^533 + x^532 + x^530 + x^529 + x^524 + x^522 + x^521 + x^519 + x^518 + x^516 + x^515 + x^514 + x^513 + x^512 + x^510 + x^508 + x^506 + x^505 + x^502 + x^501 + x^500 + x^499 + x^496 + x^493 + x^492 + x^491 + x^488 + x^487 + x^483 + x^482 + x^481 + x^479 + x^478 + x^476 + x^475 + x^474 + x^473 + x^472 + x^471 + x^470 + x^469 + x^467 + x^463 + x^460 + x^458 + x^453 + x^452 + x^451 + x^447 + x^446 + x^445 + x^439 + x^435 + x^434 + x^433 + x^432 + x^431 + x^430 + x^428 + x^427 + x^425 + x^421 + x^420 + x^419 + x^417 + x^415 + x^413 + x^412 + x^410 + x^407 + x^406 + x^405 + x^404 + x^402 + x^400 + x^397 + x^395 + x^394 + x^393 + x^392 + x^391 + x^390 + x^388 + x^385 + x^381 + x^378 + x^377 + x^376 + x^374 + x^373 + x^370 + x^369 + x^368 + x^366 + x^365 + x^364 + x^363 + x^361 + x^358 + x^356 + x^355 + x^354 + x^351 + x^350 + x^348 + x^347 + x^344 + x^341 + x^340 + x^337 + x^335 + x^334 + x^329 + x^326 + x^325 + x^324 + x^322 + x^321 + x^319 + x^318 + x^314 + x^312 + x^311 + x^310 + x^309 + x^308 + x^307 + x^303 + x^302 + x^301 + x^298 + x^293 + x^292 + x^291 + x^288 + x^281 + x^280 + x^279 + x^278 + x^275 + x^271 + x^270 + x^267 + x^265 + x^264 + x^263 + x^262 + x^261 + x^260 + x^259 + x^258 + x^257 + x^256 + x^253 + x^252 + x^251 + x^250 + x^249 + x^248 + x^247 + x^244 + x^243 + x^242 + x^240 + x^237 + x^236 + x^233 + x^227 + x^226 + x^224 + x^222 + x^221 + x^220 + x^219 + x^218 + x^217 + x^216 + x^213 + x^212 + x^210 + x^208 + x^207 + x^206 + x^205 + x^202 + x^201 + x^200 + x^197 + x^195 + x^194 + x^193 + x^192 + x^191 + x^190 + x^189 + x^186 + x^184 + x^183 + x^182 + x^181 + x^178 + x^177 + x^176 + x^170 + x^168 + x^167 + x^166 + x^162 + x^159 + x^157 + x^149 + x^148 + x^147 + x^143 + x^141 + x^139 + x^138 + x^136 + x^135 + x^131 + x^127 + x^121 + x^119 + x^117 + x^114 + x^110 + x^109 + x^107 + x^103 + x^99 + x^98 + x^96 + x^95 + x^94 + x^92 + x^90 + x^89 + x^88 + x^87 + x^86 + x^85 + x^84 + x^82 + x^80 + x^79 + x^78 + x^77 + x^76 + x^74 + x^71 + x^69 + x^68 + x^66 + x^64 + x^62 + x^61 + x^58 + x^57 + x^53 + x^50 + x^46 + x^44 + x^43 + x^42 + x^41 + x^40 + x^37 + x^35 + x^34 + x^32 + x^31 + x^29 + x^27 + x^26 + x^25 + x^22 + x^20 + x^16 + x^12 + x^10 + 1 + +57-51-22 397 x^928 + x^898 + x^895 + x^870 + x^866 + x^865 + x^862 + x^860 + x^857 + x^835 + x^833 + x^832 + x^828 + x^822 + x^819 + x^810 + x^808 + x^805 + x^797 + x^796 + x^791 + x^790 + x^789 + x^786 + x^784 + x^782 + x^781 + x^778 + x^777 + x^773 + x^767 + x^766 + x^762 + x^757 + x^752 + x^743 + x^742 + x^737 + x^734 + x^717 + x^715 + x^713 + x^708 + x^707 + x^706 + x^705 + x^703 + x^702 + x^701 + x^698 + x^696 + x^690 + x^688 + x^685 + x^683 + x^682 + x^680 + x^677 + x^673 + x^671 + x^668 + x^667 + x^666 + x^663 + x^662 + x^661 + x^656 + x^654 + x^653 + x^649 + x^647 + x^646 + x^645 + x^643 + x^637 + x^635 + x^632 + x^630 + x^629 + x^627 + x^626 + x^625 + x^624 + x^622 + x^619 + x^613 + x^610 + x^608 + x^602 + x^597 + x^594 + x^593 + x^591 + x^590 + x^587 + x^586 + x^584 + x^583 + x^582 + x^580 + x^577 + x^575 + x^574 + x^573 + x^570 + x^569 + x^568 + x^562 + x^556 + x^553 + x^552 + x^551 + x^550 + x^547 + x^546 + x^545 + x^543 + x^541 + x^539 + x^538 + x^535 + x^534 + x^533 + x^528 + x^527 + x^526 + x^524 + x^522 + x^521 + x^515 + x^512 + x^509 + x^505 + x^503 + x^502 + x^501 + x^500 + x^499 + x^498 + x^497 + x^493 + x^491 + x^483 + x^480 + x^477 + x^474 + x^473 + x^472 + x^471 + x^470 + x^468 + x^467 + x^463 + x^459 + x^458 + x^457 + x^455 + x^453 + x^452 + x^451 + x^448 + x^447 + x^446 + x^445 + x^444 + x^443 + x^440 + x^439 + x^436 + x^435 + x^434 + x^432 + x^430 + x^429 + x^428 + x^427 + x^426 + x^423 + x^422 + x^421 + x^420 + x^417 + x^416 + x^415 + x^414 + x^413 + x^411 + x^410 + x^407 + x^406 + x^403 + x^401 + x^400 + x^391 + x^390 + x^387 + x^386 + x^385 + x^381 + x^380 + x^379 + x^377 + x^374 + x^373 + x^370 + x^369 + x^362 + x^359 + x^357 + x^356 + x^355 + x^354 + x^349 + x^346 + x^345 + x^344 + x^343 + x^340 + x^339 + x^338 + x^337 + x^336 + x^334 + x^331 + x^330 + x^328 + x^323 + x^322 + x^321 + x^319 + x^318 + x^317 + x^316 + x^314 + x^313 + x^312 + x^311 + x^310 + x^305 + x^303 + x^299 + x^295 + x^294 + x^293 + x^290 + x^289 + x^286 + x^285 + x^284 + x^281 + x^280 + x^278 + x^275 + x^272 + x^271 + x^269 + x^267 + x^266 + x^263 + x^262 + x^261 + x^260 + x^258 + x^257 + x^256 + x^255 + x^254 + x^253 + x^252 + x^251 + x^247 + x^246 + x^241 + x^240 + x^239 + x^238 + x^236 + x^233 + x^232 + x^230 + x^228 + x^224 + x^223 + x^220 + x^216 + x^215 + x^213 + x^212 + x^211 + x^208 + x^206 + x^205 + x^204 + x^201 + x^199 + x^197 + x^195 + x^194 + x^193 + x^191 + x^189 + x^188 + x^187 + x^186 + x^185 + x^184 + x^182 + x^179 + x^178 + x^176 + x^172 + x^171 + x^170 + x^169 + x^168 + x^167 + x^166 + x^163 + x^162 + x^161 + x^160 + x^157 + x^156 + x^155 + x^153 + x^152 + x^148 + x^145 + x^142 + x^140 + x^139 + x^137 + x^134 + x^133 + x^131 + x^123 + x^119 + x^118 + x^114 + x^112 + x^111 + x^110 + x^108 + x^107 + x^105 + x^104 + x^102 + x^99 + x^98 + x^95 + x^94 + x^93 + x^89 + x^88 + x^86 + x^84 + x^82 + x^80 + x^78 + x^76 + x^73 + x^71 + x^70 + x^68 + x^67 + x^66 + x^64 + x^62 + x^60 + x^59 + x^57 + x^53 + x^52 + x^49 + x^48 + x^47 + x^46 + x^40 + x^37 + x^36 + x^35 + x^34 + x^32 + x^31 + x^28 + x^26 + x^25 + x^24 + x^22 + x^20 + x^19 + x^18 + x^15 + x^10 + x^8 + x^5 + 1 + +41-40-20 399 x^928 + x^898 + x^870 + x^846 + x^844 + x^821 + x^820 + x^816 + x^814 + x^808 + x^804 + x^801 + x^800 + x^798 + x^795 + x^778 + x^771 + x^770 + x^769 + x^768 + x^765 + x^760 + x^749 + x^748 + x^744 + x^743 + x^741 + x^738 + x^734 + x^729 + x^726 + x^719 + x^717 + x^716 + x^715 + x^714 + x^713 + x^712 + x^711 + x^710 + x^709 + x^706 + x^704 + x^703 + x^702 + x^701 + x^694 + x^693 + x^691 + x^690 + x^685 + x^684 + x^682 + x^681 + x^680 + x^678 + x^677 + x^676 + x^675 + x^674 + x^673 + x^671 + x^669 + x^667 + x^665 + x^661 + x^660 + x^659 + x^658 + x^656 + x^652 + x^651 + x^649 + x^648 + x^643 + x^642 + x^639 + x^637 + x^634 + x^631 + x^629 + x^628 + x^624 + x^620 + x^619 + x^618 + x^617 + x^616 + x^615 + x^614 + x^613 + x^611 + x^610 + x^605 + x^604 + x^603 + x^601 + x^599 + x^598 + x^595 + x^594 + x^593 + x^590 + x^589 + x^585 + x^584 + x^583 + x^579 + x^577 + x^575 + x^573 + x^572 + x^571 + x^570 + x^569 + x^566 + x^564 + x^562 + x^561 + x^560 + x^557 + x^555 + x^554 + x^553 + x^552 + x^551 + x^550 + x^549 + x^547 + x^545 + x^542 + x^540 + x^539 + x^538 + x^537 + x^536 + x^535 + x^533 + x^532 + x^530 + x^529 + x^524 + x^518 + x^516 + x^514 + x^513 + x^512 + x^510 + x^509 + x^504 + x^503 + x^501 + x^493 + x^490 + x^489 + x^487 + x^486 + x^485 + x^484 + x^481 + x^479 + x^478 + x^475 + x^473 + x^471 + x^470 + x^468 + x^467 + x^466 + x^462 + x^460 + x^455 + x^454 + x^451 + x^450 + x^449 + x^447 + x^446 + x^445 + x^444 + x^443 + x^442 + x^441 + x^440 + x^439 + x^438 + x^436 + x^430 + x^429 + x^424 + x^423 + x^420 + x^419 + x^418 + x^415 + x^414 + x^413 + x^410 + x^409 + x^408 + x^407 + x^406 + x^405 + x^403 + x^402 + x^400 + x^398 + x^397 + x^396 + x^395 + x^394 + x^392 + x^391 + x^389 + x^387 + x^386 + x^385 + x^383 + x^382 + x^381 + x^379 + x^375 + x^374 + x^369 + x^367 + x^366 + x^365 + x^363 + x^360 + x^359 + x^358 + x^357 + x^355 + x^354 + x^351 + x^339 + x^338 + x^337 + x^336 + x^335 + x^334 + x^332 + x^330 + x^329 + x^323 + x^320 + x^317 + x^315 + x^312 + x^311 + x^309 + x^305 + x^302 + x^301 + x^300 + x^299 + x^298 + x^297 + x^296 + x^295 + x^292 + x^291 + x^288 + x^286 + x^285 + x^284 + x^283 + x^282 + x^280 + x^279 + x^278 + x^273 + x^268 + x^267 + x^264 + x^263 + x^258 + x^257 + x^256 + x^255 + x^254 + x^253 + x^250 + x^249 + x^248 + x^244 + x^243 + x^242 + x^238 + x^237 + x^236 + x^234 + x^233 + x^231 + x^230 + x^222 + x^218 + x^215 + x^214 + x^211 + x^208 + x^206 + x^205 + x^201 + x^200 + x^198 + x^197 + x^195 + x^194 + x^193 + x^191 + x^189 + x^185 + x^182 + x^179 + x^176 + x^174 + x^173 + x^164 + x^163 + x^162 + x^159 + x^158 + x^156 + x^155 + x^152 + x^150 + x^148 + x^142 + x^140 + x^139 + x^138 + x^137 + x^135 + x^134 + x^133 + x^132 + x^131 + x^128 + x^124 + x^122 + x^121 + x^120 + x^119 + x^117 + x^116 + x^114 + x^111 + x^110 + x^108 + x^107 + x^105 + x^104 + x^103 + x^101 + x^95 + x^94 + x^93 + x^91 + x^89 + x^88 + x^87 + x^84 + x^83 + x^82 + x^81 + x^80 + x^79 + x^78 + x^77 + x^76 + x^74 + x^73 + x^72 + x^66 + x^65 + x^64 + x^59 + x^57 + x^56 + x^53 + x^49 + x^47 + x^44 + x^43 + x^39 + x^38 + x^37 + x^34 + x^33 + x^32 + x^31 + x^28 + x^23 + x^22 + x^19 + x^12 + x^8 + 1 + +55-27-24 399 x^928 + x^898 + x^864 + x^847 + x^846 + x^834 + x^824 + x^818 + x^817 + x^816 + x^812 + x^810 + x^808 + x^806 + x^801 + x^800 + x^795 + x^794 + x^788 + x^783 + x^781 + x^777 + x^776 + x^775 + x^771 + x^765 + x^764 + x^753 + x^750 + x^749 + x^747 + x^746 + x^745 + x^743 + x^742 + x^741 + x^737 + x^734 + x^731 + x^728 + x^726 + x^725 + x^724 + x^721 + x^719 + x^717 + x^716 + x^714 + x^712 + x^706 + x^704 + x^703 + x^702 + x^701 + x^699 + x^698 + x^697 + x^695 + x^694 + x^691 + x^690 + x^688 + x^687 + x^684 + x^683 + x^681 + x^680 + x^673 + x^672 + x^670 + x^667 + x^666 + x^664 + x^663 + x^661 + x^659 + x^657 + x^655 + x^650 + x^649 + x^647 + x^640 + x^639 + x^637 + x^636 + x^634 + x^630 + x^626 + x^625 + x^623 + x^622 + x^620 + x^618 + x^617 + x^615 + x^612 + x^611 + x^609 + x^608 + x^607 + x^605 + x^604 + x^602 + x^600 + x^599 + x^596 + x^594 + x^592 + x^590 + x^587 + x^581 + x^578 + x^576 + x^575 + x^574 + x^570 + x^567 + x^566 + x^565 + x^564 + x^563 + x^561 + x^560 + x^555 + x^554 + x^553 + x^552 + x^551 + x^550 + x^549 + x^548 + x^543 + x^537 + x^536 + x^533 + x^532 + x^531 + x^529 + x^528 + x^527 + x^525 + x^524 + x^523 + x^522 + x^519 + x^518 + x^516 + x^515 + x^514 + x^513 + x^512 + x^511 + x^509 + x^507 + x^505 + x^500 + x^498 + x^497 + x^496 + x^494 + x^489 + x^486 + x^481 + x^480 + x^479 + x^476 + x^471 + x^469 + x^467 + x^466 + x^463 + x^459 + x^455 + x^451 + x^450 + x^449 + x^448 + x^447 + x^445 + x^444 + x^441 + x^440 + x^432 + x^430 + x^429 + x^428 + x^426 + x^424 + x^420 + x^418 + x^416 + x^415 + x^413 + x^411 + x^408 + x^406 + x^405 + x^404 + x^400 + x^398 + x^392 + x^390 + x^387 + x^386 + x^382 + x^379 + x^378 + x^377 + x^376 + x^374 + x^373 + x^371 + x^370 + x^368 + x^367 + x^365 + x^364 + x^363 + x^362 + x^360 + x^358 + x^351 + x^350 + x^344 + x^343 + x^341 + x^340 + x^339 + x^338 + x^337 + x^332 + x^331 + x^329 + x^327 + x^323 + x^321 + x^320 + x^318 + x^317 + x^314 + x^313 + x^312 + x^311 + x^309 + x^308 + x^306 + x^304 + x^301 + x^300 + x^299 + x^296 + x^295 + x^294 + x^291 + x^290 + x^287 + x^286 + x^285 + x^280 + x^275 + x^274 + x^271 + x^270 + x^268 + x^264 + x^263 + x^258 + x^257 + x^256 + x^254 + x^252 + x^249 + x^248 + x^246 + x^245 + x^242 + x^241 + x^239 + x^238 + x^236 + x^233 + x^231 + x^228 + x^226 + x^225 + x^222 + x^219 + x^218 + x^217 + x^216 + x^215 + x^213 + x^210 + x^209 + x^207 + x^205 + x^203 + x^202 + x^198 + x^197 + x^196 + x^193 + x^192 + x^189 + x^188 + x^187 + x^184 + x^183 + x^182 + x^181 + x^179 + x^178 + x^177 + x^175 + x^172 + x^171 + x^170 + x^169 + x^164 + x^162 + x^161 + x^160 + x^159 + x^155 + x^154 + x^152 + x^151 + x^148 + x^147 + x^143 + x^140 + x^139 + x^137 + x^134 + x^133 + x^132 + x^131 + x^126 + x^125 + x^120 + x^119 + x^118 + x^117 + x^113 + x^111 + x^110 + x^108 + x^104 + x^102 + x^101 + x^100 + x^97 + x^96 + x^90 + x^89 + x^85 + x^82 + x^81 + x^77 + x^76 + x^73 + x^69 + x^68 + x^67 + x^65 + x^64 + x^63 + x^62 + x^61 + x^55 + x^54 + x^53 + x^52 + x^49 + x^47 + x^46 + x^40 + x^39 + x^37 + x^36 + x^35 + x^34 + x^33 + x^32 + x^31 + x^29 + x^26 + x^25 + x^24 + x^23 + x^22 + x^19 + x^17 + x^16 + x^12 + x^9 + x^8 + x^5 + 1 + +39-32-32 401 x^928 + x^898 + x^870 + x^867 + x^842 + x^837 + x^836 + x^832 + x^807 + x^806 + x^805 + x^802 + x^801 + x^780 + x^778 + x^775 + x^773 + x^771 + x^770 + x^768 + x^766 + x^758 + x^752 + x^749 + x^747 + x^740 + x^730 + x^724 + x^721 + x^719 + x^717 + x^716 + x^715 + x^714 + x^713 + x^711 + x^709 + x^708 + x^707 + x^704 + x^702 + x^700 + x^696 + x^693 + x^691 + x^689 + x^687 + x^686 + x^685 + x^683 + x^682 + x^680 + x^679 + x^677 + x^676 + x^674 + x^670 + x^669 + x^668 + x^665 + x^663 + x^661 + x^660 + x^659 + x^655 + x^653 + x^652 + x^651 + x^648 + x^647 + x^646 + x^645 + x^644 + x^642 + x^641 + x^640 + x^639 + x^638 + x^637 + x^635 + x^634 + x^633 + x^632 + x^631 + x^630 + x^625 + x^624 + x^620 + x^618 + x^615 + x^613 + x^611 + x^606 + x^605 + x^604 + x^602 + x^599 + x^597 + x^596 + x^593 + x^592 + x^591 + x^590 + x^589 + x^585 + x^584 + x^583 + x^580 + x^578 + x^577 + x^576 + x^575 + x^574 + x^572 + x^571 + x^570 + x^568 + x^567 + x^566 + x^563 + x^562 + x^560 + x^556 + x^554 + x^552 + x^549 + x^548 + x^547 + x^546 + x^544 + x^538 + x^536 + x^535 + x^534 + x^533 + x^530 + x^529 + x^528 + x^527 + x^526 + x^525 + x^524 + x^521 + x^519 + x^517 + x^516 + x^514 + x^513 + x^512 + x^510 + x^509 + x^507 + x^506 + x^504 + x^502 + x^500 + x^499 + x^498 + x^497 + x^493 + x^492 + x^490 + x^489 + x^485 + x^484 + x^481 + x^476 + x^475 + x^473 + x^467 + x^463 + x^462 + x^461 + x^459 + x^456 + x^453 + x^450 + x^449 + x^448 + x^445 + x^443 + x^442 + x^440 + x^438 + x^436 + x^433 + x^432 + x^431 + x^430 + x^429 + x^427 + x^421 + x^420 + x^419 + x^416 + x^415 + x^414 + x^413 + x^411 + x^410 + x^408 + x^407 + x^406 + x^405 + x^403 + x^400 + x^398 + x^397 + x^396 + x^394 + x^390 + x^389 + x^388 + x^385 + x^384 + x^383 + x^379 + x^376 + x^374 + x^373 + x^372 + x^370 + x^369 + x^368 + x^364 + x^362 + x^358 + x^355 + x^354 + x^353 + x^351 + x^348 + x^347 + x^344 + x^343 + x^339 + x^337 + x^336 + x^334 + x^333 + x^329 + x^328 + x^323 + x^321 + x^320 + x^319 + x^318 + x^317 + x^315 + x^314 + x^311 + x^309 + x^307 + x^306 + x^304 + x^302 + x^300 + x^297 + x^295 + x^294 + x^291 + x^289 + x^287 + x^285 + x^284 + x^283 + x^282 + x^281 + x^280 + x^278 + x^277 + x^276 + x^270 + x^269 + x^268 + x^267 + x^263 + x^261 + x^259 + x^257 + x^255 + x^253 + x^252 + x^250 + x^248 + x^247 + x^245 + x^244 + x^242 + x^241 + x^240 + x^238 + x^234 + x^233 + x^230 + x^227 + x^226 + x^224 + x^220 + x^219 + x^217 + x^216 + x^215 + x^209 + x^208 + x^207 + x^206 + x^205 + x^204 + x^203 + x^201 + x^199 + x^198 + x^197 + x^196 + x^193 + x^191 + x^187 + x^184 + x^183 + x^182 + x^173 + x^172 + x^169 + x^167 + x^166 + x^165 + x^162 + x^161 + x^160 + x^158 + x^157 + x^156 + x^155 + x^154 + x^152 + x^151 + x^149 + x^147 + x^141 + x^138 + x^137 + x^136 + x^135 + x^134 + x^131 + x^129 + x^126 + x^125 + x^122 + x^121 + x^117 + x^116 + x^113 + x^110 + x^109 + x^108 + x^104 + x^102 + x^101 + x^100 + x^98 + x^97 + x^95 + x^94 + x^87 + x^83 + x^82 + x^78 + x^77 + x^74 + x^73 + x^72 + x^70 + x^69 + x^66 + x^65 + x^62 + x^60 + x^55 + x^54 + x^52 + x^51 + x^49 + x^46 + x^45 + x^41 + x^40 + x^37 + x^33 + x^31 + x^29 + x^27 + x^26 + x^24 + x^20 + x^18 + x^17 + x^15 + x^6 + 1 + +40-7-11 401 x^928 + x^898 + x^897 + x^882 + x^870 + x^866 + x^860 + x^854 + x^852 + x^851 + x^836 + x^832 + x^830 + x^829 + x^826 + x^824 + x^820 + x^814 + x^812 + x^811 + x^808 + x^806 + x^804 + x^800 + x^796 + x^793 + x^792 + x^790 + x^786 + x^780 + x^778 + x^777 + x^774 + x^770 + x^769 + x^767 + x^762 + x^761 + x^760 + x^756 + x^749 + x^748 + x^746 + x^742 + x^736 + x^733 + x^730 + x^728 + x^726 + x^725 + x^724 + x^723 + x^719 + x^718 + x^717 + x^716 + x^715 + x^712 + x^709 + x^708 + x^707 + x^705 + x^701 + x^700 + x^699 + x^698 + x^695 + x^689 + x^688 + x^687 + x^684 + x^683 + x^680 + x^678 + x^675 + x^672 + x^670 + x^669 + x^666 + x^664 + x^663 + x^656 + x^655 + x^654 + x^651 + x^647 + x^641 + x^639 + x^638 + x^636 + x^635 + x^633 + x^632 + x^631 + x^630 + x^629 + x^627 + x^626 + x^625 + x^623 + x^621 + x^620 + x^619 + x^617 + x^613 + x^611 + x^610 + x^609 + x^608 + x^607 + x^606 + x^604 + x^603 + x^600 + x^598 + x^596 + x^595 + x^592 + x^591 + x^590 + x^587 + x^586 + x^585 + x^584 + x^582 + x^581 + x^578 + x^577 + x^576 + x^575 + x^574 + x^569 + x^566 + x^565 + x^564 + x^560 + x^558 + x^557 + x^556 + x^554 + x^552 + x^550 + x^549 + x^548 + x^545 + x^539 + x^538 + x^536 + x^535 + x^534 + x^533 + x^532 + x^527 + x^524 + x^523 + x^522 + x^520 + x^518 + x^517 + x^514 + x^513 + x^512 + x^510 + x^507 + x^504 + x^501 + x^499 + x^497 + x^496 + x^494 + x^492 + x^491 + x^490 + x^489 + x^487 + x^485 + x^483 + x^479 + x^478 + x^477 + x^476 + x^475 + x^473 + x^472 + x^470 + x^469 + x^468 + x^465 + x^464 + x^462 + x^460 + x^457 + x^452 + x^449 + x^448 + x^447 + x^445 + x^443 + x^441 + x^440 + x^436 + x^434 + x^432 + x^430 + x^428 + x^426 + x^424 + x^420 + x^416 + x^415 + x^410 + x^408 + x^407 + x^406 + x^404 + x^397 + x^395 + x^394 + x^390 + x^389 + x^387 + x^384 + x^382 + x^380 + x^374 + x^370 + x^368 + x^366 + x^365 + x^364 + x^362 + x^360 + x^359 + x^358 + x^356 + x^352 + x^350 + x^349 + x^347 + x^346 + x^344 + x^342 + x^341 + x^337 + x^336 + x^332 + x^331 + x^327 + x^325 + x^323 + x^322 + x^320 + x^317 + x^316 + x^315 + x^314 + x^312 + x^309 + x^307 + x^306 + x^305 + x^304 + x^303 + x^301 + x^300 + x^298 + x^296 + x^294 + x^293 + x^290 + x^289 + x^287 + x^285 + x^282 + x^281 + x^279 + x^277 + x^273 + x^265 + x^264 + x^259 + x^258 + x^257 + x^255 + x^254 + x^253 + x^251 + x^250 + x^249 + x^248 + x^247 + x^246 + x^244 + x^237 + x^235 + x^233 + x^232 + x^231 + x^230 + x^229 + x^228 + x^227 + x^225 + x^221 + x^215 + x^214 + x^211 + x^208 + x^206 + x^204 + x^200 + x^199 + x^198 + x^196 + x^192 + x^188 + x^186 + x^185 + x^184 + x^183 + x^181 + x^179 + x^176 + x^175 + x^173 + x^171 + x^170 + x^168 + x^165 + x^164 + x^160 + x^158 + x^155 + x^154 + x^152 + x^151 + x^149 + x^147 + x^144 + x^141 + x^140 + x^139 + x^137 + x^136 + x^133 + x^131 + x^130 + x^128 + x^127 + x^126 + x^125 + x^123 + x^122 + x^121 + x^120 + x^119 + x^117 + x^116 + x^115 + x^112 + x^109 + x^106 + x^102 + x^101 + x^98 + x^92 + x^89 + x^87 + x^86 + x^81 + x^74 + x^72 + x^71 + x^69 + x^65 + x^61 + x^57 + x^55 + x^53 + x^51 + x^50 + x^47 + x^45 + x^44 + x^43 + x^42 + x^40 + x^39 + x^38 + x^37 + x^35 + x^31 + x^30 + x^29 + x^27 + x^24 + x^22 + x^21 + x^20 + x^19 + 1 + +44-24-27 401 x^928 + x^898 + x^870 + x^858 + x^824 + x^821 + x^820 + x^798 + x^792 + x^791 + x^790 + x^789 + x^784 + x^776 + x^771 + x^770 + x^768 + x^764 + x^762 + x^761 + x^760 + x^759 + x^755 + x^754 + x^751 + x^742 + x^738 + x^735 + x^734 + x^731 + x^730 + x^729 + x^728 + x^725 + x^722 + x^721 + x^715 + x^711 + x^710 + x^706 + x^705 + x^701 + x^700 + x^698 + x^695 + x^692 + x^691 + x^689 + x^686 + x^685 + x^681 + x^676 + x^675 + x^674 + x^672 + x^671 + x^669 + x^668 + x^667 + x^666 + x^665 + x^658 + x^656 + x^654 + x^653 + x^651 + x^650 + x^648 + x^646 + x^644 + x^642 + x^635 + x^634 + x^633 + x^629 + x^626 + x^625 + x^624 + x^622 + x^621 + x^620 + x^618 + x^616 + x^614 + x^612 + x^608 + x^607 + x^606 + x^605 + x^604 + x^603 + x^602 + x^594 + x^593 + x^590 + x^589 + x^588 + x^586 + x^585 + x^584 + x^583 + x^582 + x^580 + x^577 + x^576 + x^575 + x^574 + x^569 + x^568 + x^566 + x^565 + x^564 + x^561 + x^559 + x^556 + x^554 + x^553 + x^552 + x^547 + x^543 + x^542 + x^541 + x^540 + x^539 + x^538 + x^536 + x^535 + x^534 + x^531 + x^530 + x^529 + x^527 + x^526 + x^524 + x^519 + x^514 + x^512 + x^511 + x^507 + x^504 + x^503 + x^502 + x^499 + x^498 + x^494 + x^493 + x^491 + x^489 + x^488 + x^486 + x^485 + x^483 + x^482 + x^480 + x^478 + x^476 + x^473 + x^468 + x^465 + x^463 + x^461 + x^460 + x^459 + x^458 + x^457 + x^455 + x^454 + x^453 + x^452 + x^451 + x^450 + x^449 + x^447 + x^446 + x^445 + x^443 + x^442 + x^439 + x^437 + x^436 + x^435 + x^433 + x^432 + x^429 + x^424 + x^423 + x^421 + x^418 + x^416 + x^412 + x^411 + x^406 + x^404 + x^403 + x^401 + x^400 + x^399 + x^398 + x^397 + x^394 + x^390 + x^388 + x^384 + x^383 + x^382 + x^381 + x^380 + x^378 + x^377 + x^376 + x^374 + x^373 + x^372 + x^368 + x^367 + x^366 + x^363 + x^361 + x^360 + x^359 + x^355 + x^353 + x^351 + x^350 + x^349 + x^348 + x^347 + x^345 + x^344 + x^342 + x^341 + x^340 + x^338 + x^337 + x^334 + x^329 + x^328 + x^326 + x^325 + x^319 + x^316 + x^314 + x^312 + x^311 + x^310 + x^308 + x^307 + x^305 + x^300 + x^296 + x^292 + x^291 + x^290 + x^287 + x^286 + x^284 + x^282 + x^280 + x^279 + x^277 + x^275 + x^274 + x^273 + x^272 + x^269 + x^268 + x^265 + x^264 + x^263 + x^262 + x^261 + x^258 + x^257 + x^256 + x^255 + x^254 + x^253 + x^252 + x^250 + x^249 + x^244 + x^242 + x^239 + x^238 + x^237 + x^235 + x^234 + x^232 + x^231 + x^228 + x^226 + x^225 + x^224 + x^221 + x^220 + x^219 + x^218 + x^216 + x^214 + x^213 + x^211 + x^210 + x^207 + x^205 + x^204 + x^203 + x^202 + x^200 + x^199 + x^198 + x^194 + x^186 + x^185 + x^184 + x^183 + x^182 + x^180 + x^179 + x^177 + x^174 + x^171 + x^169 + x^168 + x^165 + x^164 + x^163 + x^161 + x^159 + x^158 + x^157 + x^154 + x^152 + x^151 + x^148 + x^146 + x^145 + x^144 + x^142 + x^141 + x^139 + x^136 + x^135 + x^134 + x^132 + x^131 + x^130 + x^129 + x^128 + x^127 + x^124 + x^123 + x^120 + x^119 + x^117 + x^116 + x^114 + x^112 + x^106 + x^103 + x^101 + x^100 + x^98 + x^94 + x^93 + x^91 + x^86 + x^83 + x^82 + x^78 + x^76 + x^74 + x^73 + x^71 + x^69 + x^66 + x^65 + x^61 + x^60 + x^59 + x^58 + x^57 + x^56 + x^55 + x^54 + x^47 + x^45 + x^44 + x^43 + x^42 + x^40 + x^39 + x^38 + x^34 + x^31 + x^29 + x^28 + x^19 + x^16 + x^12 + x^6 + 1 + +10-49-23 403 x^928 + x^900 + x^898 + x^893 + x^872 + x^870 + x^865 + x^858 + x^842 + x^840 + x^835 + x^833 + x^826 + x^823 + x^821 + x^816 + x^814 + x^809 + x^808 + x^807 + x^805 + x^802 + x^796 + x^793 + x^782 + x^780 + x^778 + x^775 + x^772 + x^770 + x^767 + x^766 + x^765 + x^760 + x^758 + x^756 + x^754 + x^753 + x^751 + x^747 + x^744 + x^740 + x^739 + x^738 + x^736 + x^735 + x^732 + x^728 + x^724 + x^723 + x^722 + x^720 + x^718 + x^717 + x^714 + x^712 + x^709 + x^708 + x^707 + x^706 + x^705 + x^702 + x^701 + x^700 + x^698 + x^697 + x^696 + x^694 + x^693 + x^691 + x^689 + x^688 + x^684 + x^682 + x^681 + x^680 + x^679 + x^677 + x^676 + x^674 + x^673 + x^672 + x^670 + x^668 + x^667 + x^666 + x^664 + x^661 + x^658 + x^657 + x^655 + x^652 + x^649 + x^646 + x^645 + x^643 + x^641 + x^640 + x^639 + x^638 + x^636 + x^635 + x^634 + x^632 + x^631 + x^628 + x^623 + x^622 + x^620 + x^619 + x^615 + x^612 + x^610 + x^607 + x^604 + x^602 + x^599 + x^598 + x^596 + x^593 + x^590 + x^588 + x^587 + x^585 + x^582 + x^581 + x^578 + x^576 + x^575 + x^573 + x^571 + x^570 + x^569 + x^568 + x^567 + x^565 + x^564 + x^563 + x^559 + x^557 + x^556 + x^552 + x^550 + x^549 + x^548 + x^546 + x^545 + x^543 + x^537 + x^536 + x^535 + x^534 + x^533 + x^532 + x^529 + x^526 + x^525 + x^523 + x^521 + x^520 + x^518 + x^515 + x^513 + x^512 + x^511 + x^508 + x^503 + x^502 + x^499 + x^498 + x^495 + x^494 + x^493 + x^492 + x^491 + x^480 + x^479 + x^475 + x^474 + x^472 + x^470 + x^469 + x^466 + x^465 + x^460 + x^453 + x^452 + x^451 + x^447 + x^444 + x^443 + x^438 + x^436 + x^431 + x^428 + x^426 + x^424 + x^423 + x^421 + x^420 + x^418 + x^417 + x^414 + x^413 + x^411 + x^410 + x^404 + x^401 + x^396 + x^395 + x^390 + x^389 + x^386 + x^384 + x^380 + x^378 + x^377 + x^376 + x^373 + x^372 + x^371 + x^369 + x^364 + x^361 + x^360 + x^359 + x^357 + x^356 + x^354 + x^353 + x^348 + x^347 + x^346 + x^342 + x^341 + x^339 + x^338 + x^333 + x^332 + x^331 + x^329 + x^328 + x^324 + x^322 + x^320 + x^315 + x^314 + x^312 + x^311 + x^310 + x^307 + x^306 + x^303 + x^295 + x^294 + x^292 + x^289 + x^288 + x^283 + x^281 + x^279 + x^274 + x^272 + x^271 + x^270 + x^268 + x^267 + x^266 + x^265 + x^264 + x^261 + x^258 + x^257 + x^256 + x^255 + x^252 + x^251 + x^248 + x^245 + x^243 + x^242 + x^238 + x^237 + x^236 + x^235 + x^233 + x^232 + x^231 + x^229 + x^228 + x^226 + x^225 + x^224 + x^222 + x^221 + x^220 + x^216 + x^215 + x^213 + x^212 + x^210 + x^209 + x^208 + x^202 + x^200 + x^199 + x^197 + x^194 + x^193 + x^192 + x^191 + x^190 + x^189 + x^187 + x^186 + x^184 + x^183 + x^181 + x^179 + x^178 + x^177 + x^176 + x^174 + x^173 + x^172 + x^171 + x^169 + x^166 + x^164 + x^163 + x^160 + x^158 + x^157 + x^155 + x^152 + x^151 + x^150 + x^148 + x^146 + x^145 + x^143 + x^142 + x^137 + x^133 + x^129 + x^126 + x^118 + x^116 + x^114 + x^113 + x^112 + x^110 + x^106 + x^102 + x^99 + x^98 + x^97 + x^96 + x^92 + x^91 + x^88 + x^86 + x^85 + x^83 + x^82 + x^81 + x^79 + x^78 + x^77 + x^76 + x^73 + x^71 + x^67 + x^66 + x^65 + x^64 + x^63 + x^61 + x^60 + x^58 + x^53 + x^52 + x^51 + x^48 + x^47 + x^44 + x^43 + x^42 + x^41 + x^39 + x^37 + x^36 + x^35 + x^34 + x^33 + x^31 + x^26 + x^21 + x^20 + x^16 + x^10 + x^5 + 1 + +16-2-49 403 x^928 + x^898 + x^870 + x^858 + x^855 + x^846 + x^831 + x^828 + x^822 + x^818 + x^808 + x^800 + x^798 + x^795 + x^788 + x^786 + x^784 + x^783 + x^776 + x^775 + x^770 + x^768 + x^764 + x^759 + x^753 + x^751 + x^750 + x^746 + x^743 + x^740 + x^736 + x^735 + x^734 + x^724 + x^723 + x^719 + x^712 + x^708 + x^706 + x^703 + x^699 + x^698 + x^696 + x^695 + x^694 + x^693 + x^691 + x^690 + x^688 + x^686 + x^684 + x^683 + x^676 + x^675 + x^673 + x^672 + x^671 + x^670 + x^668 + x^664 + x^663 + x^662 + x^660 + x^655 + x^654 + x^652 + x^649 + x^644 + x^642 + x^636 + x^634 + x^633 + x^623 + x^622 + x^620 + x^619 + x^616 + x^614 + x^613 + x^611 + x^609 + x^608 + x^606 + x^604 + x^603 + x^602 + x^601 + x^600 + x^599 + x^598 + x^596 + x^595 + x^594 + x^592 + x^591 + x^588 + x^586 + x^585 + x^584 + x^583 + x^579 + x^578 + x^577 + x^576 + x^575 + x^574 + x^572 + x^569 + x^562 + x^560 + x^555 + x^553 + x^550 + x^549 + x^548 + x^547 + x^546 + x^545 + x^544 + x^543 + x^542 + x^538 + x^536 + x^533 + x^532 + x^531 + x^530 + x^529 + x^528 + x^527 + x^526 + x^523 + x^518 + x^516 + x^515 + x^511 + x^509 + x^507 + x^503 + x^501 + x^500 + x^498 + x^497 + x^496 + x^494 + x^491 + x^490 + x^487 + x^485 + x^484 + x^483 + x^482 + x^478 + x^477 + x^475 + x^474 + x^472 + x^470 + x^468 + x^467 + x^466 + x^465 + x^464 + x^462 + x^461 + x^460 + x^458 + x^457 + x^456 + x^455 + x^454 + x^453 + x^448 + x^446 + x^444 + x^443 + x^442 + x^441 + x^440 + x^437 + x^436 + x^435 + x^434 + x^433 + x^432 + x^431 + x^430 + x^429 + x^428 + x^427 + x^426 + x^425 + x^422 + x^421 + x^420 + x^419 + x^417 + x^414 + x^413 + x^411 + x^410 + x^406 + x^405 + x^400 + x^399 + x^397 + x^395 + x^394 + x^393 + x^392 + x^391 + x^389 + x^387 + x^382 + x^380 + x^379 + x^377 + x^375 + x^371 + x^370 + x^366 + x^365 + x^364 + x^363 + x^362 + x^358 + x^356 + x^354 + x^353 + x^351 + x^348 + x^346 + x^344 + x^343 + x^342 + x^340 + x^339 + x^337 + x^334 + x^333 + x^332 + x^331 + x^328 + x^327 + x^326 + x^325 + x^324 + x^323 + x^322 + x^321 + x^320 + x^318 + x^317 + x^316 + x^315 + x^314 + x^312 + x^310 + x^306 + x^304 + x^303 + x^301 + x^297 + x^295 + x^293 + x^290 + x^285 + x^283 + x^282 + x^273 + x^271 + x^268 + x^266 + x^262 + x^260 + x^259 + x^258 + x^253 + x^252 + x^251 + x^249 + x^248 + x^246 + x^245 + x^244 + x^243 + x^242 + x^241 + x^240 + x^239 + x^238 + x^237 + x^234 + x^233 + x^231 + x^229 + x^228 + x^227 + x^223 + x^222 + x^217 + x^216 + x^214 + x^213 + x^211 + x^209 + x^208 + x^203 + x^202 + x^201 + x^198 + x^197 + x^196 + x^194 + x^193 + x^189 + x^188 + x^187 + x^183 + x^182 + x^181 + x^177 + x^176 + x^174 + x^170 + x^166 + x^163 + x^162 + x^161 + x^160 + x^158 + x^157 + x^155 + x^153 + x^152 + x^151 + x^149 + x^148 + x^147 + x^146 + x^144 + x^142 + x^141 + x^140 + x^138 + x^132 + x^131 + x^129 + x^128 + x^126 + x^125 + x^123 + x^122 + x^120 + x^119 + x^117 + x^116 + x^115 + x^112 + x^109 + x^107 + x^105 + x^101 + x^97 + x^95 + x^94 + x^92 + x^91 + x^90 + x^87 + x^86 + x^85 + x^83 + x^82 + x^81 + x^79 + x^78 + x^77 + x^73 + x^72 + x^71 + x^68 + x^67 + x^65 + x^64 + x^63 + x^60 + x^58 + x^54 + x^50 + x^45 + x^44 + x^43 + x^42 + x^39 + x^37 + x^26 + x^24 + x^20 + x^16 + x^14 + x^10 + 1 + +17-2-46 405 x^928 + x^898 + x^870 + x^860 + x^848 + x^831 + x^830 + x^818 + x^812 + x^808 + x^804 + x^801 + x^800 + x^792 + x^791 + x^790 + x^784 + x^780 + x^778 + x^775 + x^774 + x^770 + x^763 + x^762 + x^761 + x^760 + x^756 + x^754 + x^750 + x^748 + x^746 + x^745 + x^736 + x^733 + x^732 + x^731 + x^729 + x^728 + x^726 + x^720 + x^719 + x^712 + x^710 + x^706 + x^703 + x^702 + x^701 + x^700 + x^690 + x^686 + x^682 + x^679 + x^676 + x^675 + x^673 + x^672 + x^671 + x^669 + x^668 + x^666 + x^664 + x^663 + x^658 + x^652 + x^651 + x^648 + x^647 + x^646 + x^645 + x^644 + x^643 + x^642 + x^638 + x^633 + x^628 + x^626 + x^624 + x^623 + x^621 + x^617 + x^616 + x^614 + x^613 + x^611 + x^610 + x^609 + x^607 + x^606 + x^604 + x^599 + x^595 + x^594 + x^593 + x^592 + x^590 + x^589 + x^588 + x^584 + x^580 + x^579 + x^578 + x^577 + x^576 + x^574 + x^573 + x^569 + x^568 + x^567 + x^566 + x^564 + x^563 + x^561 + x^560 + x^559 + x^558 + x^557 + x^555 + x^549 + x^548 + x^547 + x^544 + x^543 + x^542 + x^540 + x^535 + x^534 + x^532 + x^531 + x^530 + x^528 + x^527 + x^526 + x^525 + x^520 + x^519 + x^516 + x^513 + x^510 + x^509 + x^500 + x^499 + x^498 + x^497 + x^496 + x^493 + x^492 + x^490 + x^489 + x^487 + x^484 + x^482 + x^480 + x^479 + x^476 + x^473 + x^471 + x^470 + x^468 + x^467 + x^465 + x^464 + x^463 + x^462 + x^461 + x^460 + x^457 + x^453 + x^452 + x^450 + x^448 + x^447 + x^443 + x^440 + x^439 + x^438 + x^437 + x^431 + x^430 + x^429 + x^428 + x^427 + x^426 + x^424 + x^423 + x^420 + x^418 + x^416 + x^415 + x^408 + x^407 + x^406 + x^405 + x^404 + x^403 + x^397 + x^395 + x^391 + x^389 + x^388 + x^386 + x^384 + x^382 + x^381 + x^380 + x^379 + x^378 + x^375 + x^374 + x^373 + x^372 + x^371 + x^369 + x^368 + x^365 + x^364 + x^363 + x^362 + x^360 + x^359 + x^358 + x^357 + x^353 + x^351 + x^350 + x^345 + x^341 + x^338 + x^336 + x^335 + x^334 + x^332 + x^331 + x^329 + x^328 + x^327 + x^323 + x^321 + x^320 + x^319 + x^317 + x^316 + x^314 + x^313 + x^312 + x^310 + x^308 + x^307 + x^304 + x^303 + x^300 + x^297 + x^295 + x^293 + x^291 + x^289 + x^288 + x^287 + x^285 + x^284 + x^282 + x^280 + x^279 + x^277 + x^273 + x^272 + x^269 + x^268 + x^267 + x^266 + x^265 + x^264 + x^262 + x^260 + x^259 + x^256 + x^255 + x^254 + x^253 + x^249 + x^247 + x^245 + x^243 + x^241 + x^240 + x^239 + x^238 + x^237 + x^236 + x^235 + x^234 + x^232 + x^228 + x^226 + x^225 + x^224 + x^223 + x^220 + x^219 + x^217 + x^209 + x^207 + x^206 + x^204 + x^203 + x^202 + x^201 + x^200 + x^199 + x^195 + x^194 + x^189 + x^187 + x^186 + x^185 + x^184 + x^182 + x^180 + x^178 + x^174 + x^173 + x^172 + x^170 + x^167 + x^166 + x^165 + x^164 + x^162 + x^161 + x^160 + x^159 + x^158 + x^157 + x^156 + x^155 + x^154 + x^153 + x^152 + x^151 + x^147 + x^143 + x^141 + x^138 + x^131 + x^130 + x^129 + x^127 + x^122 + x^121 + x^120 + x^118 + x^117 + x^114 + x^111 + x^109 + x^106 + x^105 + x^104 + x^103 + x^102 + x^101 + x^100 + x^99 + x^98 + x^97 + x^93 + x^92 + x^91 + x^90 + x^88 + x^87 + x^86 + x^82 + x^81 + x^80 + x^79 + x^78 + x^77 + x^75 + x^74 + x^73 + x^72 + x^71 + x^68 + x^59 + x^58 + x^51 + x^50 + x^43 + x^42 + x^40 + x^39 + x^34 + x^27 + x^25 + x^24 + x^23 + x^19 + x^18 + x^17 + x^15 + x^14 + x^13 + x^11 + x^10 + 1 + +55-23-24 407 x^928 + x^898 + x^894 + x^870 + x^865 + x^849 + x^846 + x^842 + x^834 + x^833 + x^830 + x^826 + x^820 + x^819 + x^816 + x^813 + x^808 + x^804 + x^803 + x^794 + x^782 + x^778 + x^772 + x^769 + x^767 + x^765 + x^764 + x^762 + x^760 + x^759 + x^756 + x^751 + x^745 + x^743 + x^740 + x^739 + x^737 + x^733 + x^730 + x^729 + x^728 + x^726 + x^722 + x^720 + x^718 + x^717 + x^716 + x^713 + x^710 + x^709 + x^708 + x^707 + x^702 + x^699 + x^697 + x^694 + x^692 + x^691 + x^688 + x^687 + x^682 + x^680 + x^679 + x^678 + x^676 + x^675 + x^674 + x^670 + x^668 + x^666 + x^662 + x^661 + x^657 + x^652 + x^651 + x^649 + x^647 + x^646 + x^643 + x^640 + x^639 + x^637 + x^635 + x^634 + x^633 + x^630 + x^618 + x^616 + x^614 + x^613 + x^611 + x^610 + x^609 + x^608 + x^604 + x^603 + x^602 + x^597 + x^593 + x^591 + x^590 + x^585 + x^582 + x^579 + x^575 + x^574 + x^573 + x^572 + x^569 + x^567 + x^566 + x^564 + x^562 + x^561 + x^560 + x^557 + x^556 + x^555 + x^554 + x^551 + x^550 + x^547 + x^545 + x^542 + x^541 + x^539 + x^537 + x^534 + x^531 + x^530 + x^529 + x^528 + x^526 + x^525 + x^523 + x^522 + x^521 + x^520 + x^519 + x^518 + x^517 + x^515 + x^514 + x^513 + x^511 + x^510 + x^509 + x^508 + x^506 + x^505 + x^500 + x^498 + x^492 + x^485 + x^484 + x^480 + x^479 + x^478 + x^477 + x^476 + x^471 + x^470 + x^469 + x^467 + x^465 + x^464 + x^462 + x^458 + x^457 + x^454 + x^453 + x^452 + x^451 + x^450 + x^445 + x^444 + x^443 + x^441 + x^439 + x^437 + x^436 + x^435 + x^433 + x^431 + x^430 + x^429 + x^425 + x^424 + x^421 + x^419 + x^414 + x^411 + x^410 + x^409 + x^408 + x^407 + x^406 + x^405 + x^396 + x^395 + x^394 + x^390 + x^388 + x^385 + x^383 + x^381 + x^379 + x^378 + x^377 + x^376 + x^375 + x^373 + x^371 + x^369 + x^368 + x^367 + x^360 + x^358 + x^353 + x^351 + x^350 + x^348 + x^347 + x^346 + x^344 + x^343 + x^341 + x^338 + x^336 + x^335 + x^334 + x^331 + x^330 + x^329 + x^327 + x^326 + x^324 + x^322 + x^318 + x^314 + x^313 + x^311 + x^307 + x^305 + x^304 + x^302 + x^299 + x^295 + x^294 + x^293 + x^289 + x^284 + x^282 + x^281 + x^277 + x^272 + x^271 + x^270 + x^268 + x^265 + x^262 + x^259 + x^257 + x^254 + x^252 + x^251 + x^250 + x^249 + x^244 + x^243 + x^240 + x^239 + x^237 + x^235 + x^234 + x^233 + x^231 + x^230 + x^227 + x^224 + x^223 + x^220 + x^219 + x^218 + x^217 + x^215 + x^214 + x^213 + x^212 + x^211 + x^210 + x^209 + x^208 + x^207 + x^205 + x^203 + x^201 + x^200 + x^197 + x^193 + x^190 + x^187 + x^185 + x^184 + x^181 + x^180 + x^179 + x^178 + x^177 + x^176 + x^175 + x^173 + x^172 + x^169 + x^168 + x^166 + x^165 + x^162 + x^159 + x^157 + x^156 + x^155 + x^154 + x^153 + x^152 + x^150 + x^146 + x^145 + x^143 + x^142 + x^141 + x^139 + x^138 + x^137 + x^136 + x^135 + x^134 + x^132 + x^129 + x^123 + x^119 + x^115 + x^114 + x^112 + x^108 + x^107 + x^102 + x^100 + x^98 + x^97 + x^95 + x^93 + x^91 + x^90 + x^87 + x^86 + x^85 + x^83 + x^80 + x^79 + x^78 + x^77 + x^75 + x^74 + x^72 + x^69 + x^68 + x^65 + x^64 + x^61 + x^60 + x^59 + x^58 + x^57 + x^56 + x^55 + x^52 + x^51 + x^50 + x^45 + x^44 + x^43 + x^42 + x^41 + x^40 + x^39 + x^38 + x^37 + x^34 + x^33 + x^31 + x^30 + x^29 + x^26 + x^25 + x^23 + x^21 + x^20 + x^19 + x^18 + x^17 + x^13 + x^12 + x^10 + x^8 + x^7 + 1 + +15-3-26 409 x^928 + x^898 + x^892 + x^870 + x^863 + x^862 + x^858 + x^856 + x^852 + x^833 + x^828 + x^827 + x^826 + x^824 + x^823 + x^820 + x^818 + x^816 + x^812 + x^808 + x^803 + x^801 + x^799 + x^798 + x^797 + x^795 + x^794 + x^792 + x^791 + x^790 + x^788 + x^787 + x^784 + x^783 + x^782 + x^773 + x^772 + x^771 + x^768 + x^767 + x^763 + x^760 + x^757 + x^753 + x^750 + x^746 + x^744 + x^737 + x^732 + x^731 + x^730 + x^729 + x^728 + x^726 + x^725 + x^724 + x^721 + x^717 + x^716 + x^713 + x^711 + x^710 + x^707 + x^706 + x^705 + x^703 + x^702 + x^699 + x^698 + x^697 + x^696 + x^694 + x^692 + x^691 + x^690 + x^689 + x^688 + x^681 + x^678 + x^673 + x^668 + x^666 + x^662 + x^658 + x^657 + x^656 + x^653 + x^651 + x^650 + x^648 + x^644 + x^643 + x^642 + x^641 + x^640 + x^638 + x^636 + x^635 + x^634 + x^633 + x^632 + x^630 + x^628 + x^626 + x^625 + x^623 + x^622 + x^620 + x^618 + x^617 + x^616 + x^614 + x^611 + x^609 + x^607 + x^605 + x^603 + x^596 + x^595 + x^593 + x^592 + x^590 + x^587 + x^586 + x^585 + x^584 + x^583 + x^582 + x^580 + x^577 + x^576 + x^571 + x^569 + x^568 + x^567 + x^565 + x^564 + x^562 + x^561 + x^560 + x^559 + x^558 + x^557 + x^556 + x^555 + x^554 + x^553 + x^552 + x^548 + x^547 + x^546 + x^545 + x^544 + x^542 + x^538 + x^537 + x^534 + x^533 + x^532 + x^529 + x^527 + x^526 + x^524 + x^523 + x^518 + x^517 + x^509 + x^505 + x^503 + x^501 + x^494 + x^493 + x^492 + x^488 + x^485 + x^484 + x^483 + x^480 + x^479 + x^477 + x^475 + x^474 + x^473 + x^472 + x^471 + x^469 + x^467 + x^466 + x^465 + x^464 + x^461 + x^458 + x^456 + x^453 + x^449 + x^446 + x^445 + x^444 + x^443 + x^442 + x^441 + x^440 + x^436 + x^435 + x^432 + x^431 + x^427 + x^426 + x^423 + x^418 + x^415 + x^413 + x^412 + x^411 + x^410 + x^407 + x^406 + x^403 + x^400 + x^398 + x^396 + x^385 + x^383 + x^382 + x^381 + x^375 + x^374 + x^373 + x^364 + x^363 + x^362 + x^360 + x^358 + x^357 + x^355 + x^353 + x^352 + x^350 + x^349 + x^345 + x^344 + x^343 + x^342 + x^340 + x^337 + x^331 + x^329 + x^328 + x^325 + x^324 + x^323 + x^322 + x^315 + x^314 + x^312 + x^311 + x^308 + x^304 + x^303 + x^299 + x^297 + x^295 + x^289 + x^284 + x^282 + x^281 + x^279 + x^278 + x^275 + x^274 + x^273 + x^272 + x^271 + x^270 + x^264 + x^263 + x^262 + x^260 + x^259 + x^258 + x^256 + x^253 + x^252 + x^251 + x^250 + x^248 + x^247 + x^246 + x^245 + x^243 + x^242 + x^240 + x^238 + x^237 + x^234 + x^233 + x^232 + x^231 + x^230 + x^228 + x^221 + x^220 + x^218 + x^213 + x^212 + x^211 + x^210 + x^204 + x^201 + x^200 + x^196 + x^195 + x^194 + x^191 + x^190 + x^189 + x^185 + x^183 + x^181 + x^180 + x^178 + x^176 + x^175 + x^170 + x^168 + x^167 + x^165 + x^164 + x^163 + x^160 + x^159 + x^158 + x^157 + x^156 + x^153 + x^151 + x^150 + x^149 + x^146 + x^145 + x^141 + x^139 + x^138 + x^137 + x^133 + x^132 + x^131 + x^130 + x^129 + x^128 + x^127 + x^125 + x^124 + x^122 + x^121 + x^120 + x^116 + x^115 + x^114 + x^112 + x^110 + x^106 + x^103 + x^101 + x^100 + x^97 + x^96 + x^94 + x^93 + x^92 + x^89 + x^88 + x^87 + x^86 + x^85 + x^84 + x^82 + x^80 + x^77 + x^73 + x^72 + x^71 + x^70 + x^68 + x^67 + x^65 + x^61 + x^60 + x^59 + x^56 + x^55 + x^48 + x^47 + x^42 + x^41 + x^36 + x^35 + x^33 + x^29 + x^27 + x^26 + x^25 + x^18 + x^14 + x^12 + x^10 + x^8 + x^4 + 1 + +33-51-22 409 x^928 + x^898 + x^886 + x^870 + x^859 + x^858 + x^857 + x^844 + x^842 + x^832 + x^831 + x^830 + x^826 + x^815 + x^812 + x^808 + x^805 + x^804 + x^803 + x^799 + x^798 + x^789 + x^787 + x^786 + x^785 + x^782 + x^777 + x^776 + x^775 + x^771 + x^769 + x^766 + x^763 + x^758 + x^756 + x^752 + x^749 + x^745 + x^744 + x^743 + x^741 + x^739 + x^738 + x^735 + x^734 + x^731 + x^730 + x^721 + x^720 + x^715 + x^714 + x^711 + x^708 + x^705 + x^704 + x^703 + x^700 + x^698 + x^695 + x^693 + x^692 + x^690 + x^689 + x^684 + x^682 + x^681 + x^680 + x^677 + x^676 + x^673 + x^672 + x^671 + x^667 + x^666 + x^662 + x^661 + x^658 + x^656 + x^653 + x^652 + x^650 + x^649 + x^647 + x^646 + x^645 + x^644 + x^642 + x^641 + x^640 + x^638 + x^636 + x^634 + x^633 + x^632 + x^631 + x^629 + x^625 + x^624 + x^623 + x^621 + x^620 + x^619 + x^616 + x^615 + x^614 + x^612 + x^611 + x^609 + x^606 + x^605 + x^602 + x^601 + x^599 + x^596 + x^595 + x^592 + x^591 + x^588 + x^587 + x^586 + x^583 + x^582 + x^578 + x^577 + x^576 + x^574 + x^572 + x^570 + x^566 + x^562 + x^560 + x^555 + x^552 + x^550 + x^546 + x^545 + x^544 + x^543 + x^542 + x^539 + x^537 + x^535 + x^534 + x^533 + x^532 + x^531 + x^530 + x^528 + x^525 + x^524 + x^523 + x^519 + x^516 + x^513 + x^511 + x^509 + x^508 + x^507 + x^506 + x^504 + x^500 + x^499 + x^497 + x^495 + x^494 + x^491 + x^488 + x^487 + x^484 + x^480 + x^479 + x^476 + x^474 + x^472 + x^471 + x^470 + x^469 + x^468 + x^464 + x^463 + x^462 + x^460 + x^459 + x^458 + x^457 + x^456 + x^455 + x^454 + x^451 + x^449 + x^448 + x^447 + x^446 + x^445 + x^441 + x^440 + x^439 + x^437 + x^436 + x^435 + x^431 + x^430 + x^427 + x^423 + x^421 + x^419 + x^412 + x^410 + x^407 + x^406 + x^405 + x^404 + x^402 + x^400 + x^398 + x^396 + x^395 + x^394 + x^392 + x^389 + x^387 + x^386 + x^384 + x^382 + x^381 + x^379 + x^378 + x^377 + x^375 + x^372 + x^370 + x^369 + x^363 + x^361 + x^360 + x^351 + x^350 + x^347 + x^343 + x^342 + x^340 + x^336 + x^328 + x^327 + x^326 + x^325 + x^323 + x^322 + x^321 + x^319 + x^318 + x^315 + x^314 + x^313 + x^312 + x^311 + x^308 + x^307 + x^305 + x^301 + x^299 + x^298 + x^295 + x^293 + x^292 + x^290 + x^289 + x^288 + x^286 + x^284 + x^283 + x^282 + x^280 + x^279 + x^275 + x^273 + x^272 + x^269 + x^266 + x^265 + x^264 + x^255 + x^251 + x^249 + x^248 + x^246 + x^244 + x^243 + x^242 + x^238 + x^237 + x^235 + x^234 + x^232 + x^230 + x^229 + x^228 + x^226 + x^224 + x^222 + x^221 + x^218 + x^213 + x^210 + x^209 + x^208 + x^206 + x^203 + x^200 + x^195 + x^193 + x^189 + x^187 + x^186 + x^185 + x^182 + x^181 + x^179 + x^177 + x^174 + x^173 + x^170 + x^163 + x^160 + x^159 + x^155 + x^154 + x^153 + x^152 + x^151 + x^147 + x^146 + x^143 + x^141 + x^133 + x^130 + x^128 + x^125 + x^123 + x^118 + x^116 + x^115 + x^114 + x^110 + x^108 + x^106 + x^105 + x^104 + x^102 + x^100 + x^98 + x^97 + x^96 + x^95 + x^94 + x^93 + x^92 + x^91 + x^89 + x^85 + x^84 + x^83 + x^82 + x^81 + x^80 + x^78 + x^77 + x^74 + x^72 + x^71 + x^69 + x^67 + x^65 + x^64 + x^63 + x^61 + x^58 + x^56 + x^55 + x^53 + x^52 + x^51 + x^50 + x^48 + x^47 + x^46 + x^45 + x^44 + x^43 + x^40 + x^39 + x^37 + x^36 + x^35 + x^33 + x^32 + x^30 + x^29 + x^28 + x^27 + x^25 + x^23 + x^21 + x^16 + x^13 + x^11 + x^10 + x^8 + 1 + +23-25-12 411 x^928 + x^898 + x^873 + x^870 + x^852 + x^847 + x^844 + x^842 + x^838 + x^836 + x^831 + x^828 + x^822 + x^821 + x^818 + x^817 + x^816 + x^814 + x^813 + x^812 + x^811 + x^801 + x^800 + x^798 + x^797 + x^796 + x^792 + x^791 + x^790 + x^787 + x^780 + x^776 + x^774 + x^770 + x^768 + x^766 + x^765 + x^762 + x^761 + x^759 + x^758 + x^757 + x^756 + x^755 + x^752 + x^749 + x^748 + x^747 + x^744 + x^741 + x^740 + x^739 + x^737 + x^735 + x^731 + x^730 + x^729 + x^728 + x^726 + x^725 + x^722 + x^720 + x^718 + x^714 + x^712 + x^711 + x^709 + x^708 + x^707 + x^706 + x^705 + x^704 + x^698 + x^694 + x^693 + x^691 + x^689 + x^686 + x^685 + x^682 + x^676 + x^674 + x^670 + x^669 + x^668 + x^667 + x^666 + x^665 + x^662 + x^661 + x^658 + x^655 + x^653 + x^652 + x^650 + x^647 + x^644 + x^643 + x^640 + x^638 + x^637 + x^635 + x^634 + x^632 + x^629 + x^627 + x^626 + x^625 + x^624 + x^619 + x^618 + x^614 + x^612 + x^611 + x^606 + x^604 + x^602 + x^601 + x^595 + x^592 + x^591 + x^590 + x^589 + x^586 + x^585 + x^581 + x^579 + x^578 + x^575 + x^574 + x^573 + x^568 + x^567 + x^565 + x^564 + x^562 + x^560 + x^559 + x^558 + x^555 + x^554 + x^553 + x^552 + x^549 + x^548 + x^546 + x^544 + x^543 + x^542 + x^541 + x^538 + x^533 + x^532 + x^529 + x^527 + x^526 + x^523 + x^522 + x^521 + x^517 + x^511 + x^508 + x^507 + x^506 + x^505 + x^504 + x^502 + x^497 + x^496 + x^494 + x^490 + x^489 + x^486 + x^485 + x^483 + x^482 + x^481 + x^477 + x^475 + x^473 + x^472 + x^471 + x^468 + x^464 + x^462 + x^461 + x^460 + x^459 + x^455 + x^454 + x^453 + x^449 + x^448 + x^447 + x^446 + x^438 + x^437 + x^435 + x^432 + x^429 + x^427 + x^426 + x^424 + x^423 + x^421 + x^419 + x^418 + x^416 + x^415 + x^413 + x^412 + x^410 + x^409 + x^407 + x^406 + x^405 + x^402 + x^401 + x^399 + x^397 + x^396 + x^394 + x^393 + x^390 + x^385 + x^384 + x^383 + x^380 + x^379 + x^378 + x^376 + x^375 + x^374 + x^372 + x^370 + x^368 + x^367 + x^366 + x^365 + x^363 + x^359 + x^357 + x^356 + x^355 + x^354 + x^353 + x^352 + x^351 + x^350 + x^348 + x^345 + x^341 + x^337 + x^335 + x^334 + x^333 + x^325 + x^319 + x^317 + x^316 + x^314 + x^305 + x^301 + x^298 + x^294 + x^293 + x^292 + x^289 + x^284 + x^281 + x^279 + x^272 + x^271 + x^269 + x^268 + x^267 + x^266 + x^264 + x^263 + x^262 + x^261 + x^256 + x^255 + x^254 + x^252 + x^251 + x^249 + x^248 + x^244 + x^242 + x^239 + x^235 + x^234 + x^233 + x^227 + x^226 + x^223 + x^221 + x^220 + x^216 + x^214 + x^211 + x^209 + x^207 + x^205 + x^204 + x^200 + x^198 + x^197 + x^196 + x^193 + x^188 + x^187 + x^185 + x^183 + x^182 + x^181 + x^179 + x^177 + x^175 + x^173 + x^172 + x^170 + x^169 + x^167 + x^163 + x^162 + x^158 + x^157 + x^156 + x^155 + x^154 + x^152 + x^151 + x^150 + x^149 + x^145 + x^144 + x^143 + x^142 + x^140 + x^139 + x^138 + x^136 + x^135 + x^133 + x^131 + x^130 + x^127 + x^126 + x^124 + x^122 + x^120 + x^118 + x^116 + x^114 + x^111 + x^110 + x^106 + x^105 + x^103 + x^102 + x^101 + x^100 + x^99 + x^98 + x^97 + x^95 + x^94 + x^93 + x^92 + x^91 + x^90 + x^86 + x^85 + x^82 + x^81 + x^78 + x^75 + x^70 + x^66 + x^64 + x^63 + x^61 + x^60 + x^55 + x^54 + x^53 + x^52 + x^50 + x^49 + x^46 + x^45 + x^44 + x^41 + x^38 + x^37 + x^36 + x^32 + x^30 + x^29 + x^27 + x^22 + x^18 + x^16 + x^13 + x^9 + x^8 + x^7 + x^5 + 1 + +36-2-35 411 x^928 + x^898 + x^870 + x^860 + x^850 + x^842 + x^830 + x^824 + x^822 + x^812 + x^808 + x^806 + x^804 + x^794 + x^792 + x^790 + x^786 + x^782 + x^778 + x^776 + x^772 + x^768 + x^764 + x^754 + x^752 + x^747 + x^746 + x^745 + x^742 + x^738 + x^737 + x^736 + x^729 + x^728 + x^725 + x^724 + x^722 + x^721 + x^720 + x^718 + x^716 + x^715 + x^714 + x^712 + x^707 + x^706 + x^704 + x^703 + x^701 + x^699 + x^693 + x^691 + x^689 + x^688 + x^687 + x^684 + x^682 + x^680 + x^674 + x^672 + x^671 + x^669 + x^668 + x^663 + x^662 + x^661 + x^660 + x^659 + x^658 + x^654 + x^652 + x^651 + x^650 + x^649 + x^648 + x^644 + x^642 + x^640 + x^639 + x^638 + x^635 + x^633 + x^632 + x^624 + x^623 + x^620 + x^619 + x^618 + x^617 + x^611 + x^607 + x^606 + x^604 + x^602 + x^601 + x^600 + x^598 + x^596 + x^594 + x^593 + x^589 + x^585 + x^583 + x^580 + x^579 + x^577 + x^576 + x^575 + x^569 + x^568 + x^567 + x^565 + x^564 + x^563 + x^560 + x^558 + x^556 + x^555 + x^553 + x^552 + x^551 + x^549 + x^548 + x^545 + x^542 + x^541 + x^539 + x^536 + x^534 + x^533 + x^530 + x^528 + x^526 + x^524 + x^521 + x^519 + x^517 + x^515 + x^511 + x^510 + x^508 + x^506 + x^505 + x^504 + x^503 + x^502 + x^501 + x^500 + x^499 + x^498 + x^496 + x^494 + x^493 + x^492 + x^491 + x^490 + x^488 + x^487 + x^484 + x^480 + x^478 + x^474 + x^472 + x^471 + x^466 + x^465 + x^464 + x^463 + x^461 + x^459 + x^458 + x^457 + x^454 + x^453 + x^446 + x^445 + x^441 + x^440 + x^439 + x^438 + x^437 + x^436 + x^435 + x^433 + x^428 + x^427 + x^426 + x^425 + x^414 + x^407 + x^406 + x^405 + x^403 + x^402 + x^401 + x^399 + x^398 + x^397 + x^395 + x^393 + x^392 + x^391 + x^390 + x^387 + x^385 + x^384 + x^383 + x^382 + x^379 + x^378 + x^376 + x^374 + x^373 + x^371 + x^370 + x^369 + x^367 + x^365 + x^364 + x^363 + x^359 + x^357 + x^355 + x^354 + x^353 + x^350 + x^346 + x^345 + x^344 + x^343 + x^340 + x^338 + x^337 + x^336 + x^335 + x^330 + x^322 + x^321 + x^320 + x^317 + x^316 + x^311 + x^310 + x^309 + x^308 + x^307 + x^306 + x^305 + x^303 + x^299 + x^296 + x^295 + x^294 + x^293 + x^292 + x^289 + x^288 + x^284 + x^283 + x^282 + x^281 + x^280 + x^278 + x^277 + x^275 + x^269 + x^265 + x^263 + x^260 + x^258 + x^256 + x^255 + x^254 + x^253 + x^252 + x^250 + x^247 + x^246 + x^245 + x^242 + x^241 + x^240 + x^239 + x^234 + x^233 + x^232 + x^230 + x^226 + x^224 + x^222 + x^218 + x^217 + x^216 + x^215 + x^214 + x^213 + x^212 + x^211 + x^208 + x^204 + x^203 + x^202 + x^201 + x^200 + x^199 + x^198 + x^197 + x^194 + x^192 + x^191 + x^184 + x^181 + x^180 + x^178 + x^177 + x^176 + x^175 + x^174 + x^173 + x^171 + x^170 + x^168 + x^166 + x^161 + x^160 + x^157 + x^156 + x^153 + x^149 + x^148 + x^147 + x^145 + x^144 + x^143 + x^139 + x^137 + x^136 + x^134 + x^132 + x^131 + x^130 + x^129 + x^128 + x^127 + x^126 + x^125 + x^118 + x^117 + x^116 + x^115 + x^114 + x^113 + x^111 + x^107 + x^105 + x^104 + x^103 + x^101 + x^99 + x^98 + x^97 + x^96 + x^92 + x^91 + x^90 + x^89 + x^87 + x^86 + x^85 + x^84 + x^83 + x^82 + x^81 + x^79 + x^78 + x^76 + x^72 + x^70 + x^69 + x^68 + x^67 + x^65 + x^64 + x^62 + x^61 + x^58 + x^54 + x^53 + x^52 + x^50 + x^49 + x^48 + x^46 + x^44 + x^43 + x^40 + x^39 + x^38 + x^35 + x^32 + x^31 + x^30 + x^29 + x^28 + x^27 + x^25 + x^24 + x^18 + x^14 + x^10 + 1 + +55-5-54 411 x^928 + x^898 + x^883 + x^870 + x^867 + x^854 + x^853 + x^840 + x^838 + x^837 + x^836 + x^834 + x^824 + x^822 + x^820 + x^812 + x^811 + x^810 + x^808 + x^807 + x^800 + x^797 + x^796 + x^795 + x^793 + x^792 + x^783 + x^779 + x^777 + x^776 + x^774 + x^770 + x^769 + x^768 + x^764 + x^762 + x^760 + x^754 + x^753 + x^752 + x^751 + x^749 + x^747 + x^746 + x^741 + x^740 + x^739 + x^738 + x^735 + x^734 + x^733 + x^732 + x^730 + x^728 + x^723 + x^721 + x^717 + x^715 + x^712 + x^711 + x^710 + x^709 + x^708 + x^707 + x^706 + x^704 + x^702 + x^701 + x^698 + x^697 + x^696 + x^694 + x^692 + x^690 + x^687 + x^683 + x^682 + x^681 + x^675 + x^674 + x^670 + x^669 + x^667 + x^666 + x^664 + x^663 + x^662 + x^657 + x^655 + x^654 + x^653 + x^652 + x^650 + x^646 + x^645 + x^644 + x^642 + x^637 + x^636 + x^635 + x^630 + x^629 + x^627 + x^626 + x^620 + x^615 + x^614 + x^612 + x^611 + x^610 + x^607 + x^606 + x^605 + x^604 + x^603 + x^601 + x^597 + x^594 + x^592 + x^591 + x^586 + x^584 + x^583 + x^582 + x^579 + x^577 + x^574 + x^573 + x^568 + x^565 + x^564 + x^563 + x^562 + x^557 + x^554 + x^553 + x^552 + x^550 + x^548 + x^546 + x^544 + x^543 + x^542 + x^540 + x^537 + x^536 + x^532 + x^531 + x^530 + x^524 + x^521 + x^520 + x^519 + x^517 + x^516 + x^515 + x^513 + x^512 + x^510 + x^509 + x^507 + x^505 + x^503 + x^502 + x^501 + x^500 + x^497 + x^496 + x^488 + x^486 + x^484 + x^481 + x^480 + x^478 + x^476 + x^475 + x^467 + x^465 + x^464 + x^463 + x^459 + x^458 + x^457 + x^455 + x^453 + x^452 + x^446 + x^445 + x^444 + x^441 + x^440 + x^439 + x^438 + x^434 + x^433 + x^432 + x^431 + x^427 + x^425 + x^421 + x^414 + x^412 + x^411 + x^409 + x^407 + x^406 + x^405 + x^403 + x^401 + x^400 + x^399 + x^398 + x^397 + x^396 + x^395 + x^394 + x^389 + x^385 + x^383 + x^380 + x^377 + x^376 + x^375 + x^374 + x^373 + x^372 + x^370 + x^367 + x^366 + x^365 + x^364 + x^363 + x^362 + x^360 + x^358 + x^356 + x^352 + x^351 + x^350 + x^349 + x^346 + x^342 + x^340 + x^338 + x^336 + x^335 + x^330 + x^329 + x^325 + x^324 + x^323 + x^321 + x^317 + x^314 + x^312 + x^307 + x^304 + x^303 + x^301 + x^298 + x^296 + x^295 + x^294 + x^293 + x^290 + x^289 + x^288 + x^287 + x^286 + x^285 + x^278 + x^277 + x^274 + x^273 + x^272 + x^270 + x^267 + x^265 + x^261 + x^260 + x^259 + x^258 + x^257 + x^256 + x^255 + x^254 + x^252 + x^249 + x^243 + x^241 + x^240 + x^239 + x^238 + x^235 + x^232 + x^231 + x^230 + x^227 + x^226 + x^224 + x^222 + x^221 + x^220 + x^219 + x^215 + x^214 + x^213 + x^211 + x^210 + x^209 + x^207 + x^206 + x^205 + x^200 + x^198 + x^195 + x^193 + x^192 + x^190 + x^186 + x^185 + x^181 + x^178 + x^176 + x^175 + x^172 + x^170 + x^169 + x^168 + x^166 + x^164 + x^158 + x^153 + x^152 + x^150 + x^149 + x^148 + x^147 + x^146 + x^143 + x^142 + x^140 + x^139 + x^136 + x^134 + x^133 + x^128 + x^125 + x^122 + x^121 + x^120 + x^116 + x^115 + x^114 + x^112 + x^111 + x^107 + x^106 + x^103 + x^101 + x^100 + x^99 + x^98 + x^96 + x^95 + x^94 + x^93 + x^92 + x^89 + x^88 + x^87 + x^86 + x^85 + x^84 + x^81 + x^79 + x^77 + x^76 + x^74 + x^72 + x^71 + x^70 + x^69 + x^65 + x^64 + x^62 + x^55 + x^53 + x^49 + x^48 + x^46 + x^44 + x^43 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 + x^26 + x^24 + x^23 + x^22 + x^20 + x^18 + x^17 + x^8 + 1 + +14-19-11 415 x^928 + x^898 + x^882 + x^870 + x^863 + x^862 + x^860 + x^854 + x^852 + x^844 + x^843 + x^833 + x^832 + x^825 + x^822 + x^819 + x^817 + x^816 + x^813 + x^811 + x^802 + x^798 + x^797 + x^795 + x^794 + x^790 + x^789 + x^787 + x^786 + x^781 + x^778 + x^776 + x^773 + x^772 + x^771 + x^768 + x^759 + x^754 + x^749 + x^746 + x^744 + x^743 + x^741 + x^740 + x^735 + x^734 + x^733 + x^730 + x^729 + x^725 + x^722 + x^717 + x^714 + x^711 + x^706 + x^705 + x^698 + x^697 + x^695 + x^691 + x^687 + x^686 + x^683 + x^681 + x^675 + x^674 + x^673 + x^672 + x^671 + x^670 + x^667 + x^665 + x^662 + x^661 + x^660 + x^658 + x^657 + x^656 + x^655 + x^654 + x^652 + x^649 + x^648 + x^645 + x^643 + x^642 + x^638 + x^636 + x^635 + x^634 + x^633 + x^632 + x^630 + x^629 + x^627 + x^626 + x^625 + x^624 + x^623 + x^622 + x^620 + x^617 + x^616 + x^615 + x^613 + x^612 + x^611 + x^608 + x^604 + x^603 + x^602 + x^599 + x^598 + x^597 + x^595 + x^594 + x^593 + x^592 + x^591 + x^590 + x^587 + x^586 + x^585 + x^584 + x^582 + x^580 + x^579 + x^578 + x^577 + x^570 + x^569 + x^568 + x^567 + x^566 + x^563 + x^562 + x^560 + x^558 + x^552 + x^550 + x^549 + x^546 + x^544 + x^543 + x^540 + x^537 + x^536 + x^535 + x^533 + x^532 + x^531 + x^530 + x^529 + x^527 + x^526 + x^522 + x^521 + x^519 + x^517 + x^514 + x^513 + x^512 + x^511 + x^510 + x^509 + x^507 + x^506 + x^504 + x^503 + x^502 + x^501 + x^500 + x^499 + x^498 + x^496 + x^494 + x^492 + x^491 + x^489 + x^488 + x^487 + x^485 + x^480 + x^479 + x^478 + x^475 + x^472 + x^463 + x^458 + x^455 + x^453 + x^452 + x^448 + x^443 + x^438 + x^436 + x^435 + x^432 + x^429 + x^428 + x^427 + x^421 + x^420 + x^419 + x^418 + x^417 + x^416 + x^414 + x^412 + x^411 + x^410 + x^406 + x^405 + x^404 + x^402 + x^401 + x^400 + x^399 + x^393 + x^388 + x^387 + x^386 + x^385 + x^384 + x^380 + x^379 + x^378 + x^376 + x^375 + x^373 + x^370 + x^368 + x^367 + x^365 + x^363 + x^362 + x^359 + x^358 + x^357 + x^356 + x^353 + x^350 + x^347 + x^343 + x^342 + x^341 + x^339 + x^338 + x^334 + x^333 + x^327 + x^325 + x^323 + x^321 + x^318 + x^317 + x^314 + x^310 + x^309 + x^308 + x^307 + x^306 + x^305 + x^304 + x^301 + x^300 + x^298 + x^295 + x^292 + x^288 + x^286 + x^285 + x^283 + x^282 + x^281 + x^278 + x^277 + x^275 + x^270 + x^269 + x^263 + x^260 + x^256 + x^254 + x^253 + x^250 + x^249 + x^248 + x^245 + x^244 + x^241 + x^239 + x^237 + x^233 + x^227 + x^225 + x^223 + x^222 + x^220 + x^219 + x^218 + x^216 + x^214 + x^212 + x^207 + x^203 + x^201 + x^200 + x^198 + x^197 + x^196 + x^195 + x^191 + x^189 + x^188 + x^184 + x^183 + x^181 + x^180 + x^179 + x^176 + x^174 + x^173 + x^170 + x^169 + x^168 + x^167 + x^165 + x^164 + x^163 + x^162 + x^161 + x^159 + x^158 + x^157 + x^155 + x^154 + x^152 + x^151 + x^150 + x^147 + x^144 + x^143 + x^142 + x^140 + x^139 + x^138 + x^134 + x^131 + x^130 + x^129 + x^124 + x^123 + x^122 + x^121 + x^120 + x^116 + x^115 + x^114 + x^111 + x^110 + x^108 + x^106 + x^104 + x^103 + x^99 + x^98 + x^95 + x^93 + x^91 + x^87 + x^84 + x^83 + x^80 + x^77 + x^76 + x^75 + x^74 + x^73 + x^70 + x^65 + x^64 + x^63 + x^62 + x^61 + x^57 + x^56 + x^55 + x^53 + x^51 + x^50 + x^49 + x^47 + x^46 + x^45 + x^43 + x^42 + x^41 + x^39 + x^37 + x^36 + x^33 + x^31 + x^29 + x^28 + x^24 + x^23 + x^22 + x^21 + x^18 + x^17 + x^16 + x^11 + x^10 + 1 + +34-37-5 415 x^928 + x^898 + x^889 + x^874 + x^870 + x^861 + x^846 + x^844 + x^839 + x^835 + x^833 + x^831 + x^818 + x^816 + x^812 + x^809 + x^808 + x^805 + x^804 + x^794 + x^792 + x^790 + x^788 + x^787 + x^784 + x^783 + x^778 + x^776 + x^775 + x^773 + x^772 + x^770 + x^769 + x^768 + x^764 + x^760 + x^759 + x^757 + x^756 + x^755 + x^754 + x^749 + x^748 + x^744 + x^740 + x^737 + x^735 + x^730 + x^729 + x^728 + x^727 + x^726 + x^725 + x^721 + x^720 + x^719 + x^718 + x^716 + x^715 + x^713 + x^712 + x^710 + x^709 + x^706 + x^699 + x^695 + x^689 + x^686 + x^682 + x^681 + x^680 + x^677 + x^674 + x^673 + x^672 + x^671 + x^665 + x^664 + x^663 + x^662 + x^661 + x^660 + x^657 + x^656 + x^651 + x^649 + x^646 + x^640 + x^639 + x^635 + x^634 + x^633 + x^632 + x^629 + x^628 + x^624 + x^623 + x^621 + x^620 + x^616 + x^615 + x^614 + x^613 + x^611 + x^610 + x^609 + x^607 + x^606 + x^605 + x^604 + x^601 + x^599 + x^597 + x^596 + x^595 + x^593 + x^591 + x^590 + x^585 + x^584 + x^583 + x^582 + x^580 + x^577 + x^576 + x^573 + x^569 + x^567 + x^564 + x^560 + x^559 + x^558 + x^557 + x^556 + x^555 + x^552 + x^551 + x^550 + x^547 + x^545 + x^544 + x^542 + x^541 + x^540 + x^539 + x^538 + x^535 + x^534 + x^533 + x^526 + x^523 + x^520 + x^518 + x^517 + x^515 + x^514 + x^513 + x^511 + x^510 + x^509 + x^504 + x^503 + x^502 + x^501 + x^496 + x^495 + x^494 + x^493 + x^492 + x^489 + x^485 + x^484 + x^482 + x^481 + x^478 + x^475 + x^472 + x^469 + x^468 + x^467 + x^465 + x^464 + x^462 + x^460 + x^455 + x^453 + x^452 + x^451 + x^449 + x^448 + x^447 + x^446 + x^445 + x^443 + x^442 + x^441 + x^438 + x^435 + x^434 + x^432 + x^431 + x^430 + x^427 + x^424 + x^423 + x^422 + x^419 + x^412 + x^411 + x^409 + x^408 + x^406 + x^405 + x^404 + x^403 + x^401 + x^400 + x^399 + x^393 + x^390 + x^389 + x^387 + x^385 + x^384 + x^380 + x^378 + x^376 + x^374 + x^372 + x^371 + x^370 + x^369 + x^366 + x^365 + x^360 + x^359 + x^357 + x^350 + x^349 + x^347 + x^346 + x^345 + x^344 + x^342 + x^341 + x^340 + x^337 + x^335 + x^334 + x^332 + x^331 + x^330 + x^328 + x^327 + x^325 + x^323 + x^322 + x^321 + x^320 + x^316 + x^314 + x^310 + x^308 + x^306 + x^305 + x^304 + x^301 + x^300 + x^295 + x^291 + x^290 + x^289 + x^288 + x^287 + x^283 + x^281 + x^280 + x^279 + x^277 + x^274 + x^272 + x^271 + x^268 + x^266 + x^264 + x^263 + x^261 + x^259 + x^255 + x^254 + x^253 + x^250 + x^249 + x^246 + x^244 + x^242 + x^241 + x^237 + x^233 + x^230 + x^229 + x^227 + x^223 + x^222 + x^217 + x^216 + x^208 + x^207 + x^206 + x^205 + x^204 + x^203 + x^198 + x^196 + x^192 + x^191 + x^190 + x^189 + x^188 + x^187 + x^186 + x^185 + x^182 + x^181 + x^180 + x^178 + x^175 + x^171 + x^166 + x^165 + x^164 + x^163 + x^162 + x^161 + x^159 + x^157 + x^156 + x^149 + x^148 + x^146 + x^145 + x^143 + x^142 + x^141 + x^140 + x^139 + x^138 + x^136 + x^135 + x^131 + x^130 + x^129 + x^127 + x^125 + x^124 + x^117 + x^115 + x^113 + x^110 + x^108 + x^107 + x^106 + x^105 + x^103 + x^99 + x^98 + x^95 + x^93 + x^90 + x^87 + x^84 + x^83 + x^81 + x^80 + x^79 + x^78 + x^77 + x^72 + x^71 + x^68 + x^65 + x^64 + x^63 + x^62 + x^59 + x^58 + x^56 + x^55 + x^54 + x^53 + x^52 + x^51 + x^46 + x^45 + x^44 + x^43 + x^40 + x^39 + x^36 + x^35 + x^33 + x^32 + x^31 + x^30 + x^27 + x^25 + x^24 + x^22 + x^18 + x^17 + x^14 + x^13 + x^12 + x^10 + 1 + +37-3-56 415 x^928 + x^898 + x^878 + x^870 + x^862 + x^860 + x^849 + x^840 + x^831 + x^822 + x^819 + x^818 + x^811 + x^802 + x^801 + x^796 + x^795 + x^794 + x^793 + x^790 + x^789 + x^784 + x^781 + x^780 + x^777 + x^775 + x^772 + x^766 + x^760 + x^758 + x^755 + x^753 + x^752 + x^747 + x^745 + x^744 + x^743 + x^740 + x^738 + x^737 + x^736 + x^732 + x^730 + x^729 + x^728 + x^726 + x^721 + x^716 + x^715 + x^713 + x^712 + x^711 + x^710 + x^709 + x^706 + x^705 + x^702 + x^700 + x^699 + x^698 + x^697 + x^696 + x^693 + x^692 + x^690 + x^687 + x^686 + x^684 + x^683 + x^682 + x^681 + x^680 + x^678 + x^675 + x^674 + x^673 + x^672 + x^670 + x^669 + x^667 + x^664 + x^662 + x^661 + x^660 + x^658 + x^657 + x^656 + x^654 + x^653 + x^652 + x^650 + x^649 + x^648 + x^645 + x^644 + x^641 + x^639 + x^635 + x^634 + x^631 + x^630 + x^629 + x^628 + x^622 + x^621 + x^620 + x^617 + x^616 + x^614 + x^610 + x^609 + x^608 + x^606 + x^604 + x^601 + x^600 + x^598 + x^597 + x^596 + x^593 + x^592 + x^590 + x^588 + x^587 + x^586 + x^585 + x^584 + x^583 + x^579 + x^578 + x^575 + x^574 + x^573 + x^569 + x^568 + x^566 + x^564 + x^563 + x^561 + x^559 + x^557 + x^556 + x^553 + x^552 + x^550 + x^548 + x^546 + x^543 + x^542 + x^541 + x^540 + x^538 + x^537 + x^536 + x^533 + x^532 + x^529 + x^528 + x^527 + x^525 + x^522 + x^520 + x^519 + x^518 + x^517 + x^515 + x^514 + x^513 + x^509 + x^507 + x^506 + x^504 + x^503 + x^502 + x^499 + x^495 + x^494 + x^493 + x^492 + x^490 + x^486 + x^482 + x^481 + x^480 + x^479 + x^477 + x^476 + x^475 + x^474 + x^473 + x^472 + x^470 + x^468 + x^467 + x^463 + x^457 + x^453 + x^451 + x^448 + x^446 + x^445 + x^440 + x^438 + x^437 + x^434 + x^433 + x^432 + x^431 + x^430 + x^425 + x^424 + x^423 + x^417 + x^414 + x^409 + x^407 + x^406 + x^405 + x^402 + x^400 + x^399 + x^398 + x^397 + x^396 + x^395 + x^394 + x^391 + x^389 + x^387 + x^386 + x^384 + x^383 + x^379 + x^377 + x^372 + x^371 + x^369 + x^368 + x^366 + x^365 + x^364 + x^361 + x^359 + x^357 + x^354 + x^351 + x^348 + x^347 + x^340 + x^339 + x^337 + x^336 + x^334 + x^329 + x^327 + x^326 + x^325 + x^322 + x^320 + x^318 + x^315 + x^314 + x^313 + x^312 + x^310 + x^309 + x^308 + x^306 + x^305 + x^301 + x^300 + x^298 + x^294 + x^293 + x^292 + x^291 + x^288 + x^285 + x^284 + x^282 + x^279 + x^275 + x^272 + x^271 + x^270 + x^269 + x^268 + x^266 + x^259 + x^257 + x^254 + x^253 + x^251 + x^250 + x^248 + x^246 + x^244 + x^243 + x^242 + x^238 + x^237 + x^234 + x^233 + x^232 + x^231 + x^230 + x^227 + x^223 + x^222 + x^221 + x^219 + x^216 + x^215 + x^212 + x^207 + x^205 + x^203 + x^200 + x^199 + x^198 + x^196 + x^195 + x^193 + x^191 + x^187 + x^185 + x^183 + x^181 + x^179 + x^178 + x^177 + x^174 + x^171 + x^170 + x^168 + x^166 + x^165 + x^163 + x^160 + x^156 + x^155 + x^154 + x^151 + x^150 + x^149 + x^146 + x^145 + x^142 + x^141 + x^136 + x^135 + x^134 + x^132 + x^128 + x^127 + x^125 + x^121 + x^118 + x^117 + x^116 + x^113 + x^112 + x^111 + x^108 + x^106 + x^105 + x^104 + x^102 + x^101 + x^100 + x^99 + x^96 + x^95 + x^94 + x^89 + x^85 + x^83 + x^81 + x^80 + x^79 + x^69 + x^65 + x^63 + x^62 + x^61 + x^59 + x^58 + x^57 + x^56 + x^52 + x^49 + x^46 + x^45 + x^43 + x^42 + x^41 + x^40 + x^39 + x^38 + x^37 + x^36 + x^33 + x^32 + x^28 + x^24 + x^23 + x^20 + x^18 + x^17 + x^14 + x^11 + x^10 + x^9 + x^8 + 1 + +55-11-54 417 x^928 + x^898 + x^896 + x^870 + x^867 + x^866 + x^864 + x^853 + x^846 + x^837 + x^836 + x^835 + x^828 + x^826 + x^824 + x^823 + x^817 + x^814 + x^812 + x^810 + x^808 + x^807 + x^806 + x^805 + x^804 + x^803 + x^802 + x^800 + x^796 + x^791 + x^789 + x^786 + x^784 + x^781 + x^780 + x^777 + x^772 + x^771 + x^770 + x^767 + x^762 + x^761 + x^757 + x^753 + x^747 + x^742 + x^740 + x^739 + x^737 + x^736 + x^735 + x^734 + x^733 + x^731 + x^730 + x^725 + x^723 + x^722 + x^717 + x^715 + x^708 + x^707 + x^705 + x^704 + x^703 + x^699 + x^698 + x^695 + x^690 + x^689 + x^686 + x^685 + x^684 + x^682 + x^681 + x^680 + x^678 + x^677 + x^675 + x^669 + x^667 + x^666 + x^665 + x^664 + x^663 + x^662 + x^661 + x^660 + x^658 + x^657 + x^656 + x^655 + x^654 + x^651 + x^650 + x^646 + x^645 + x^643 + x^641 + x^640 + x^639 + x^637 + x^634 + x^633 + x^632 + x^631 + x^627 + x^624 + x^622 + x^620 + x^617 + x^616 + x^615 + x^614 + x^613 + x^608 + x^601 + x^600 + x^599 + x^595 + x^592 + x^591 + x^590 + x^586 + x^580 + x^579 + x^578 + x^577 + x^576 + x^575 + x^571 + x^570 + x^569 + x^567 + x^566 + x^565 + x^564 + x^563 + x^562 + x^561 + x^560 + x^558 + x^557 + x^555 + x^554 + x^553 + x^552 + x^551 + x^550 + x^549 + x^545 + x^543 + x^542 + x^539 + x^537 + x^532 + x^530 + x^526 + x^525 + x^520 + x^519 + x^517 + x^515 + x^513 + x^511 + x^510 + x^508 + x^504 + x^502 + x^501 + x^498 + x^493 + x^492 + x^490 + x^489 + x^488 + x^487 + x^485 + x^484 + x^483 + x^481 + x^480 + x^479 + x^477 + x^475 + x^473 + x^472 + x^469 + x^468 + x^467 + x^466 + x^463 + x^462 + x^461 + x^459 + x^455 + x^453 + x^451 + x^450 + x^449 + x^445 + x^444 + x^443 + x^442 + x^441 + x^440 + x^437 + x^435 + x^434 + x^431 + x^430 + x^429 + x^427 + x^426 + x^422 + x^421 + x^420 + x^413 + x^410 + x^409 + x^408 + x^407 + x^405 + x^403 + x^402 + x^401 + x^397 + x^396 + x^393 + x^389 + x^386 + x^384 + x^380 + x^379 + x^376 + x^373 + x^372 + x^370 + x^369 + x^367 + x^365 + x^364 + x^363 + x^358 + x^356 + x^355 + x^354 + x^353 + x^349 + x^345 + x^342 + x^334 + x^332 + x^331 + x^329 + x^325 + x^324 + x^322 + x^316 + x^315 + x^314 + x^312 + x^310 + x^307 + x^306 + x^298 + x^297 + x^296 + x^294 + x^293 + x^292 + x^288 + x^285 + x^284 + x^283 + x^282 + x^279 + x^273 + x^271 + x^267 + x^262 + x^257 + x^255 + x^253 + x^252 + x^251 + x^250 + x^248 + x^247 + x^245 + x^244 + x^243 + x^240 + x^238 + x^237 + x^236 + x^234 + x^232 + x^229 + x^224 + x^223 + x^222 + x^221 + x^218 + x^216 + x^215 + x^213 + x^211 + x^210 + x^209 + x^207 + x^205 + x^203 + x^202 + x^201 + x^198 + x^195 + x^194 + x^193 + x^192 + x^190 + x^189 + x^187 + x^186 + x^185 + x^184 + x^183 + x^182 + x^181 + x^179 + x^177 + x^176 + x^175 + x^174 + x^172 + x^170 + x^167 + x^160 + x^157 + x^156 + x^153 + x^151 + x^150 + x^149 + x^143 + x^140 + x^137 + x^136 + x^134 + x^133 + x^129 + x^128 + x^124 + x^123 + x^120 + x^118 + x^117 + x^116 + x^115 + x^114 + x^112 + x^111 + x^110 + x^108 + x^105 + x^104 + x^102 + x^101 + x^100 + x^98 + x^97 + x^93 + x^92 + x^91 + x^90 + x^88 + x^87 + x^85 + x^81 + x^79 + x^77 + x^76 + x^74 + x^73 + x^72 + x^70 + x^65 + x^63 + x^62 + x^61 + x^59 + x^55 + x^53 + x^51 + x^50 + x^47 + x^46 + x^45 + x^44 + x^43 + x^40 + x^38 + x^34 + x^32 + x^30 + x^29 + x^28 + x^27 + x^24 + x^22 + x^21 + x^18 + x^17 + x^14 + x^11 + x^4 + 1 + +30-3-41 419 x^928 + x^906 + x^898 + x^884 + x^879 + x^876 + x^870 + x^862 + x^854 + x^852 + x^849 + x^844 + x^835 + x^832 + x^829 + x^825 + x^822 + x^819 + x^818 + x^813 + x^812 + x^807 + x^805 + x^803 + x^802 + x^798 + x^795 + x^794 + x^792 + x^790 + x^789 + x^788 + x^786 + x^785 + x^778 + x^777 + x^769 + x^768 + x^762 + x^760 + x^753 + x^751 + x^748 + x^747 + x^746 + x^742 + x^738 + x^735 + x^734 + x^733 + x^731 + x^729 + x^726 + x^725 + x^723 + x^719 + x^718 + x^716 + x^713 + x^706 + x^703 + x^702 + x^701 + x^700 + x^699 + x^696 + x^693 + x^690 + x^687 + x^686 + x^685 + x^684 + x^682 + x^679 + x^677 + x^673 + x^672 + x^671 + x^670 + x^664 + x^659 + x^656 + x^653 + x^651 + x^650 + x^647 + x^643 + x^642 + x^639 + x^637 + x^635 + x^633 + x^630 + x^627 + x^625 + x^624 + x^621 + x^616 + x^615 + x^614 + x^613 + x^612 + x^607 + x^604 + x^603 + x^600 + x^597 + x^596 + x^595 + x^593 + x^592 + x^591 + x^588 + x^586 + x^583 + x^581 + x^580 + x^578 + x^575 + x^574 + x^571 + x^569 + x^568 + x^567 + x^566 + x^565 + x^564 + x^562 + x^560 + x^559 + x^558 + x^557 + x^555 + x^554 + x^553 + x^552 + x^550 + x^546 + x^545 + x^543 + x^542 + x^537 + x^535 + x^534 + x^532 + x^530 + x^527 + x^525 + x^523 + x^521 + x^520 + x^512 + x^506 + x^505 + x^504 + x^503 + x^501 + x^500 + x^499 + x^498 + x^497 + x^495 + x^494 + x^493 + x^489 + x^487 + x^485 + x^484 + x^479 + x^478 + x^477 + x^475 + x^474 + x^471 + x^470 + x^469 + x^468 + x^467 + x^466 + x^464 + x^462 + x^459 + x^458 + x^457 + x^456 + x^451 + x^449 + x^448 + x^447 + x^446 + x^443 + x^441 + x^440 + x^439 + x^437 + x^435 + x^434 + x^433 + x^432 + x^431 + x^430 + x^427 + x^425 + x^424 + x^423 + x^420 + x^419 + x^418 + x^417 + x^416 + x^414 + x^412 + x^410 + x^409 + x^405 + x^404 + x^402 + x^401 + x^400 + x^396 + x^394 + x^393 + x^391 + x^390 + x^389 + x^388 + x^387 + x^386 + x^385 + x^380 + x^379 + x^369 + x^368 + x^363 + x^362 + x^361 + x^360 + x^359 + x^355 + x^354 + x^351 + x^347 + x^346 + x^345 + x^342 + x^337 + x^336 + x^332 + x^328 + x^324 + x^322 + x^321 + x^319 + x^318 + x^317 + x^315 + x^312 + x^310 + x^309 + x^307 + x^303 + x^302 + x^300 + x^298 + x^295 + x^289 + x^288 + x^287 + x^286 + x^285 + x^284 + x^283 + x^282 + x^281 + x^276 + x^275 + x^273 + x^272 + x^271 + x^268 + x^265 + x^263 + x^262 + x^260 + x^259 + x^255 + x^254 + x^253 + x^252 + x^249 + x^245 + x^244 + x^243 + x^242 + x^241 + x^240 + x^239 + x^236 + x^235 + x^233 + x^232 + x^231 + x^230 + x^224 + x^223 + x^222 + x^221 + x^218 + x^215 + x^214 + x^213 + x^209 + x^206 + x^204 + x^199 + x^197 + x^196 + x^194 + x^190 + x^186 + x^185 + x^183 + x^182 + x^181 + x^180 + x^179 + x^178 + x^177 + x^176 + x^175 + x^174 + x^173 + x^168 + x^167 + x^166 + x^165 + x^164 + x^163 + x^162 + x^161 + x^159 + x^158 + x^156 + x^155 + x^153 + x^151 + x^148 + x^144 + x^143 + x^142 + x^141 + x^139 + x^136 + x^135 + x^131 + x^129 + x^128 + x^127 + x^126 + x^125 + x^124 + x^123 + x^121 + x^119 + x^118 + x^116 + x^113 + x^111 + x^109 + x^107 + x^106 + x^103 + x^102 + x^100 + x^99 + x^98 + x^96 + x^93 + x^92 + x^90 + x^86 + x^85 + x^83 + x^82 + x^80 + x^76 + x^73 + x^72 + x^71 + x^70 + x^67 + x^66 + x^64 + x^61 + x^60 + x^59 + x^56 + x^53 + x^50 + x^49 + x^45 + x^44 + x^42 + x^39 + x^36 + x^35 + x^30 + x^28 + x^25 + x^22 + x^19 + x^18 + x^17 + x^15 + x^14 + x^11 + x^10 + x^5 + 1 + +11-45-50 423 x^928 + x^898 + x^891 + x^870 + x^862 + x^861 + x^858 + x^857 + x^854 + x^851 + x^828 + x^825 + x^824 + x^822 + x^818 + x^814 + x^811 + x^808 + x^802 + x^800 + x^798 + x^797 + x^787 + x^786 + x^785 + x^783 + x^782 + x^781 + x^776 + x^772 + x^771 + x^770 + x^768 + x^765 + x^760 + x^758 + x^756 + x^754 + x^750 + x^749 + x^746 + x^742 + x^738 + x^737 + x^736 + x^735 + x^734 + x^732 + x^729 + x^723 + x^719 + x^718 + x^716 + x^715 + x^714 + x^713 + x^708 + x^707 + x^705 + x^704 + x^703 + x^702 + x^701 + x^700 + x^698 + x^697 + x^696 + x^694 + x^692 + x^688 + x^684 + x^683 + x^681 + x^680 + x^678 + x^666 + x^664 + x^663 + x^662 + x^660 + x^659 + x^658 + x^656 + x^654 + x^653 + x^652 + x^651 + x^650 + x^649 + x^647 + x^645 + x^642 + x^641 + x^639 + x^638 + x^637 + x^636 + x^634 + x^632 + x^631 + x^626 + x^625 + x^623 + x^620 + x^617 + x^616 + x^614 + x^612 + x^610 + x^609 + x^607 + x^605 + x^603 + x^601 + x^600 + x^598 + x^597 + x^596 + x^595 + x^594 + x^591 + x^589 + x^588 + x^586 + x^585 + x^584 + x^581 + x^579 + x^578 + x^577 + x^576 + x^575 + x^573 + x^572 + x^570 + x^569 + x^568 + x^566 + x^561 + x^559 + x^558 + x^557 + x^554 + x^552 + x^551 + x^550 + x^549 + x^548 + x^546 + x^544 + x^543 + x^541 + x^540 + x^538 + x^533 + x^531 + x^529 + x^528 + x^523 + x^519 + x^515 + x^512 + x^511 + x^510 + x^508 + x^507 + x^506 + x^501 + x^498 + x^497 + x^496 + x^495 + x^494 + x^492 + x^490 + x^489 + x^488 + x^487 + x^486 + x^485 + x^483 + x^477 + x^476 + x^474 + x^472 + x^470 + x^469 + x^467 + x^465 + x^462 + x^460 + x^458 + x^452 + x^451 + x^449 + x^447 + x^446 + x^445 + x^442 + x^441 + x^440 + x^437 + x^436 + x^434 + x^433 + x^431 + x^428 + x^427 + x^426 + x^425 + x^423 + x^422 + x^420 + x^419 + x^418 + x^417 + x^415 + x^414 + x^413 + x^412 + x^411 + x^409 + x^407 + x^404 + x^403 + x^402 + x^399 + x^398 + x^396 + x^395 + x^392 + x^391 + x^389 + x^388 + x^387 + x^386 + x^385 + x^384 + x^381 + x^379 + x^378 + x^377 + x^375 + x^373 + x^369 + x^367 + x^365 + x^364 + x^362 + x^359 + x^358 + x^357 + x^355 + x^351 + x^348 + x^347 + x^346 + x^345 + x^344 + x^343 + x^335 + x^334 + x^332 + x^328 + x^325 + x^324 + x^321 + x^319 + x^318 + x^316 + x^314 + x^312 + x^311 + x^310 + x^304 + x^300 + x^297 + x^296 + x^294 + x^293 + x^292 + x^291 + x^289 + x^288 + x^285 + x^283 + x^277 + x^276 + x^274 + x^271 + x^270 + x^269 + x^268 + x^265 + x^264 + x^260 + x^259 + x^258 + x^257 + x^256 + x^252 + x^251 + x^249 + x^248 + x^247 + x^246 + x^244 + x^242 + x^238 + x^236 + x^233 + x^232 + x^231 + x^230 + x^229 + x^228 + x^227 + x^222 + x^220 + x^218 + x^214 + x^212 + x^209 + x^208 + x^207 + x^204 + x^203 + x^202 + x^200 + x^193 + x^190 + x^189 + x^188 + x^186 + x^185 + x^183 + x^178 + x^176 + x^174 + x^171 + x^169 + x^168 + x^166 + x^165 + x^163 + x^162 + x^161 + x^158 + x^156 + x^155 + x^154 + x^153 + x^150 + x^147 + x^146 + x^145 + x^141 + x^136 + x^134 + x^132 + x^128 + x^127 + x^126 + x^123 + x^120 + x^119 + x^114 + x^112 + x^111 + x^105 + x^104 + x^101 + x^100 + x^98 + x^97 + x^96 + x^95 + x^94 + x^92 + x^91 + x^88 + x^87 + x^85 + x^84 + x^83 + x^80 + x^78 + x^77 + x^76 + x^74 + x^73 + x^72 + x^68 + x^64 + x^62 + x^61 + x^60 + x^56 + x^55 + x^54 + x^53 + x^50 + x^46 + x^42 + x^41 + x^40 + x^39 + x^36 + x^32 + x^31 + x^30 + x^28 + x^25 + x^24 + x^16 + x^15 + x^14 + x^13 + x^10 + x^9 + x^6 + x^3 + 1 + +50-19-47 423 x^928 + x^906 + x^898 + x^897 + x^884 + x^876 + x^870 + x^862 + x^854 + x^840 + x^836 + x^835 + x^832 + x^831 + x^830 + x^821 + x^814 + x^813 + x^812 + x^810 + x^808 + x^805 + x^803 + x^796 + x^794 + x^792 + x^791 + x^786 + x^785 + x^783 + x^777 + x^774 + x^773 + x^767 + x^766 + x^764 + x^762 + x^761 + x^760 + x^759 + x^758 + x^754 + x^750 + x^748 + x^747 + x^746 + x^743 + x^742 + x^739 + x^734 + x^732 + x^728 + x^727 + x^724 + x^722 + x^720 + x^718 + x^716 + x^714 + x^713 + x^712 + x^711 + x^709 + x^708 + x^705 + x^704 + x^702 + x^699 + x^698 + x^695 + x^694 + x^693 + x^692 + x^691 + x^688 + x^686 + x^683 + x^682 + x^681 + x^679 + x^676 + x^674 + x^672 + x^671 + x^668 + x^664 + x^663 + x^661 + x^659 + x^657 + x^656 + x^654 + x^652 + x^646 + x^645 + x^641 + x^638 + x^635 + x^634 + x^632 + x^629 + x^628 + x^619 + x^618 + x^617 + x^615 + x^613 + x^612 + x^610 + x^609 + x^607 + x^606 + x^605 + x^604 + x^603 + x^602 + x^598 + x^597 + x^596 + x^595 + x^593 + x^591 + x^590 + x^589 + x^584 + x^583 + x^582 + x^581 + x^580 + x^579 + x^576 + x^575 + x^573 + x^569 + x^568 + x^567 + x^565 + x^564 + x^562 + x^556 + x^554 + x^552 + x^551 + x^550 + x^548 + x^547 + x^546 + x^545 + x^542 + x^541 + x^540 + x^539 + x^537 + x^535 + x^534 + x^532 + x^528 + x^526 + x^525 + x^524 + x^522 + x^521 + x^519 + x^515 + x^513 + x^510 + x^508 + x^507 + x^506 + x^505 + x^504 + x^502 + x^501 + x^500 + x^499 + x^496 + x^495 + x^494 + x^492 + x^490 + x^489 + x^486 + x^485 + x^484 + x^483 + x^481 + x^479 + x^478 + x^476 + x^474 + x^471 + x^469 + x^467 + x^465 + x^463 + x^458 + x^456 + x^455 + x^454 + x^451 + x^449 + x^448 + x^443 + x^442 + x^440 + x^439 + x^437 + x^436 + x^433 + x^432 + x^431 + x^430 + x^428 + x^427 + x^426 + x^425 + x^424 + x^421 + x^420 + x^416 + x^415 + x^413 + x^412 + x^411 + x^410 + x^409 + x^408 + x^406 + x^405 + x^403 + x^400 + x^399 + x^398 + x^396 + x^395 + x^393 + x^391 + x^388 + x^387 + x^386 + x^381 + x^380 + x^377 + x^376 + x^375 + x^374 + x^373 + x^371 + x^370 + x^369 + x^368 + x^361 + x^359 + x^358 + x^357 + x^355 + x^352 + x^350 + x^348 + x^347 + x^346 + x^343 + x^342 + x^339 + x^337 + x^335 + x^334 + x^332 + x^329 + x^325 + x^324 + x^323 + x^322 + x^318 + x^317 + x^315 + x^314 + x^311 + x^310 + x^306 + x^304 + x^300 + x^299 + x^297 + x^295 + x^294 + x^289 + x^286 + x^283 + x^279 + x^278 + x^276 + x^275 + x^273 + x^269 + x^268 + x^266 + x^264 + x^260 + x^258 + x^257 + x^254 + x^252 + x^250 + x^248 + x^247 + x^246 + x^245 + x^241 + x^239 + x^237 + x^236 + x^235 + x^234 + x^232 + x^228 + x^227 + x^226 + x^225 + x^218 + x^215 + x^214 + x^212 + x^211 + x^210 + x^208 + x^206 + x^205 + x^204 + x^203 + x^201 + x^198 + x^196 + x^193 + x^192 + x^190 + x^189 + x^188 + x^184 + x^181 + x^180 + x^178 + x^177 + x^176 + x^175 + x^174 + x^173 + x^171 + x^169 + x^168 + x^166 + x^165 + x^164 + x^162 + x^160 + x^159 + x^158 + x^153 + x^150 + x^148 + x^147 + x^146 + x^143 + x^142 + x^139 + x^136 + x^135 + x^134 + x^133 + x^132 + x^126 + x^124 + x^121 + x^118 + x^116 + x^112 + x^109 + x^108 + x^107 + x^104 + x^101 + x^98 + x^97 + x^95 + x^93 + x^88 + x^86 + x^85 + x^84 + x^83 + x^78 + x^77 + x^73 + x^70 + x^69 + x^68 + x^66 + x^64 + x^63 + x^62 + x^58 + x^57 + x^53 + x^52 + x^50 + x^48 + x^45 + x^43 + x^42 + x^41 + x^40 + x^39 + x^38 + x^33 + x^26 + x^22 + x^21 + x^20 + x^18 + x^17 + x^10 + x^8 + 1 + +52-27-13 427 x^928 + x^898 + x^870 + x^866 + x^858 + x^855 + x^854 + x^847 + x^844 + x^836 + x^833 + x^832 + x^829 + x^828 + x^825 + x^822 + x^818 + x^817 + x^814 + x^811 + x^808 + x^807 + x^800 + x^799 + x^798 + x^796 + x^795 + x^792 + x^789 + x^781 + x^777 + x^774 + x^770 + x^769 + x^768 + x^767 + x^765 + x^762 + x^759 + x^756 + x^755 + x^754 + x^752 + x^751 + x^748 + x^746 + x^745 + x^744 + x^740 + x^739 + x^738 + x^736 + x^735 + x^734 + x^732 + x^730 + x^729 + x^728 + x^724 + x^721 + x^719 + x^718 + x^717 + x^716 + x^713 + x^712 + x^710 + x^709 + x^706 + x^705 + x^704 + x^702 + x^701 + x^699 + x^698 + x^697 + x^695 + x^693 + x^690 + x^689 + x^687 + x^686 + x^685 + x^683 + x^682 + x^681 + x^680 + x^677 + x^676 + x^675 + x^674 + x^671 + x^669 + x^668 + x^667 + x^666 + x^665 + x^663 + x^660 + x^659 + x^655 + x^654 + x^653 + x^652 + x^650 + x^648 + x^647 + x^643 + x^642 + x^640 + x^639 + x^638 + x^635 + x^634 + x^633 + x^632 + x^630 + x^629 + x^628 + x^626 + x^624 + x^620 + x^618 + x^617 + x^616 + x^615 + x^612 + x^610 + x^607 + x^606 + x^604 + x^603 + x^602 + x^598 + x^597 + x^593 + x^589 + x^588 + x^587 + x^585 + x^583 + x^581 + x^580 + x^579 + x^578 + x^570 + x^569 + x^566 + x^565 + x^564 + x^563 + x^561 + x^559 + x^558 + x^557 + x^556 + x^555 + x^551 + x^550 + x^548 + x^547 + x^545 + x^544 + x^540 + x^539 + x^538 + x^537 + x^536 + x^535 + x^534 + x^532 + x^529 + x^528 + x^527 + x^526 + x^525 + x^524 + x^523 + x^522 + x^519 + x^517 + x^516 + x^515 + x^513 + x^512 + x^511 + x^507 + x^504 + x^502 + x^495 + x^493 + x^492 + x^491 + x^490 + x^489 + x^484 + x^483 + x^481 + x^480 + x^478 + x^466 + x^464 + x^463 + x^460 + x^459 + x^457 + x^455 + x^454 + x^451 + x^450 + x^448 + x^447 + x^435 + x^434 + x^433 + x^432 + x^431 + x^428 + x^424 + x^423 + x^422 + x^421 + x^420 + x^419 + x^414 + x^413 + x^412 + x^411 + x^410 + x^409 + x^407 + x^404 + x^401 + x^396 + x^395 + x^394 + x^391 + x^390 + x^389 + x^388 + x^385 + x^382 + x^379 + x^377 + x^372 + x^370 + x^367 + x^365 + x^362 + x^361 + x^357 + x^355 + x^354 + x^351 + x^349 + x^348 + x^345 + x^343 + x^342 + x^340 + x^338 + x^334 + x^330 + x^328 + x^324 + x^323 + x^320 + x^319 + x^318 + x^317 + x^315 + x^313 + x^310 + x^309 + x^307 + x^303 + x^302 + x^301 + x^300 + x^299 + x^298 + x^297 + x^296 + x^295 + x^292 + x^289 + x^288 + x^287 + x^285 + x^283 + x^280 + x^274 + x^273 + x^271 + x^267 + x^265 + x^264 + x^261 + x^259 + x^258 + x^256 + x^254 + x^253 + x^248 + x^247 + x^244 + x^239 + x^238 + x^237 + x^233 + x^232 + x^230 + x^229 + x^228 + x^227 + x^224 + x^221 + x^217 + x^216 + x^215 + x^214 + x^209 + x^207 + x^206 + x^205 + x^203 + x^201 + x^200 + x^199 + x^196 + x^195 + x^194 + x^192 + x^191 + x^190 + x^189 + x^187 + x^186 + x^182 + x^180 + x^177 + x^176 + x^174 + x^173 + x^170 + x^169 + x^168 + x^167 + x^164 + x^160 + x^154 + x^153 + x^152 + x^148 + x^144 + x^143 + x^142 + x^141 + x^140 + x^139 + x^135 + x^134 + x^133 + x^132 + x^129 + x^125 + x^124 + x^122 + x^121 + x^120 + x^118 + x^117 + x^116 + x^115 + x^112 + x^111 + x^110 + x^109 + x^108 + x^107 + x^106 + x^104 + x^103 + x^101 + x^96 + x^95 + x^93 + x^90 + x^87 + x^86 + x^82 + x^79 + x^77 + x^75 + x^70 + x^68 + x^67 + x^66 + x^62 + x^60 + x^57 + x^54 + x^53 + x^52 + x^51 + x^49 + x^46 + x^45 + x^43 + x^42 + x^41 + x^40 + x^39 + x^38 + x^36 + x^33 + x^31 + x^30 + x^29 + x^27 + x^26 + x^24 + x^22 + x^18 + x^7 + x^6 + 1 + +54-9-25 433 x^928 + x^898 + x^875 + x^870 + x^860 + x^850 + x^847 + x^840 + x^835 + x^832 + x^822 + x^819 + x^815 + x^812 + x^808 + x^807 + x^805 + x^804 + x^800 + x^797 + x^794 + x^792 + x^791 + x^790 + x^787 + x^785 + x^782 + x^778 + x^777 + x^776 + x^771 + x^770 + x^767 + x^766 + x^764 + x^763 + x^760 + x^759 + x^757 + x^755 + x^754 + x^753 + x^752 + x^748 + x^743 + x^742 + x^739 + x^738 + x^737 + x^735 + x^734 + x^733 + x^731 + x^730 + x^729 + x^725 + x^724 + x^723 + x^720 + x^719 + x^717 + x^716 + x^714 + x^713 + x^711 + x^710 + x^709 + x^706 + x^705 + x^700 + x^699 + x^697 + x^696 + x^695 + x^693 + x^692 + x^691 + x^688 + x^687 + x^686 + x^685 + x^683 + x^682 + x^680 + x^676 + x^671 + x^670 + x^664 + x^663 + x^662 + x^661 + x^660 + x^658 + x^657 + x^656 + x^654 + x^653 + x^649 + x^648 + x^646 + x^645 + x^642 + x^637 + x^635 + x^633 + x^632 + x^630 + x^627 + x^624 + x^623 + x^621 + x^620 + x^619 + x^617 + x^615 + x^614 + x^613 + x^610 + x^609 + x^608 + x^607 + x^604 + x^602 + x^601 + x^600 + x^599 + x^593 + x^592 + x^591 + x^589 + x^585 + x^584 + x^580 + x^576 + x^575 + x^574 + x^570 + x^569 + x^566 + x^565 + x^564 + x^563 + x^562 + x^561 + x^559 + x^558 + x^554 + x^552 + x^551 + x^547 + x^546 + x^545 + x^543 + x^541 + x^540 + x^535 + x^532 + x^531 + x^530 + x^529 + x^525 + x^524 + x^523 + x^521 + x^520 + x^519 + x^518 + x^517 + x^516 + x^515 + x^513 + x^512 + x^510 + x^509 + x^504 + x^503 + x^502 + x^501 + x^500 + x^497 + x^496 + x^494 + x^493 + x^488 + x^487 + x^486 + x^484 + x^482 + x^481 + x^480 + x^479 + x^475 + x^472 + x^465 + x^464 + x^462 + x^461 + x^460 + x^457 + x^456 + x^455 + x^454 + x^452 + x^451 + x^450 + x^449 + x^448 + x^446 + x^445 + x^444 + x^443 + x^441 + x^438 + x^437 + x^436 + x^434 + x^432 + x^430 + x^427 + x^425 + x^424 + x^421 + x^420 + x^419 + x^418 + x^417 + x^414 + x^413 + x^410 + x^409 + x^407 + x^404 + x^403 + x^402 + x^401 + x^400 + x^399 + x^398 + x^396 + x^393 + x^392 + x^391 + x^390 + x^389 + x^385 + x^382 + x^381 + x^380 + x^379 + x^378 + x^375 + x^373 + x^372 + x^367 + x^366 + x^362 + x^358 + x^356 + x^355 + x^354 + x^353 + x^349 + x^348 + x^347 + x^342 + x^340 + x^339 + x^336 + x^334 + x^331 + x^329 + x^328 + x^327 + x^326 + x^325 + x^323 + x^319 + x^317 + x^316 + x^314 + x^313 + x^312 + x^310 + x^309 + x^308 + x^307 + x^302 + x^299 + x^298 + x^297 + x^296 + x^293 + x^291 + x^289 + x^288 + x^286 + x^285 + x^284 + x^282 + x^279 + x^274 + x^272 + x^271 + x^268 + x^266 + x^263 + x^261 + x^258 + x^255 + x^252 + x^251 + x^249 + x^248 + x^247 + x^246 + x^243 + x^242 + x^240 + x^237 + x^236 + x^234 + x^233 + x^232 + x^231 + x^228 + x^226 + x^223 + x^222 + x^221 + x^220 + x^219 + x^217 + x^216 + x^215 + x^213 + x^212 + x^210 + x^205 + x^203 + x^202 + x^199 + x^197 + x^195 + x^194 + x^193 + x^191 + x^189 + x^188 + x^187 + x^184 + x^182 + x^178 + x^173 + x^172 + x^170 + x^169 + x^167 + x^165 + x^162 + x^159 + x^158 + x^157 + x^156 + x^154 + x^149 + x^148 + x^145 + x^144 + x^141 + x^140 + x^135 + x^134 + x^133 + x^131 + x^130 + x^129 + x^127 + x^126 + x^123 + x^120 + x^119 + x^118 + x^117 + x^115 + x^110 + x^108 + x^106 + x^105 + x^104 + x^99 + x^97 + x^95 + x^89 + x^86 + x^84 + x^82 + x^78 + x^75 + x^71 + x^70 + x^69 + x^68 + x^65 + x^63 + x^62 + x^61 + x^59 + x^58 + x^51 + x^50 + x^49 + x^48 + x^47 + x^44 + x^43 + x^42 + x^39 + x^38 + x^37 + x^35 + x^30 + x^27 + x^25 + x^21 + x^19 + x^17 + x^16 + x^13 + x^12 + x^10 + x^8 + x^6 + 1 + +56-43-35 433 x^928 + x^898 + x^896 + x^871 + x^870 + x^864 + x^848 + x^847 + x^846 + x^841 + x^840 + x^834 + x^833 + x^832 + x^829 + x^821 + x^814 + x^811 + x^808 + x^806 + x^803 + x^798 + x^797 + x^796 + x^792 + x^788 + x^786 + x^784 + x^783 + x^781 + x^779 + x^777 + x^776 + x^774 + x^773 + x^770 + x^769 + x^766 + x^762 + x^760 + x^756 + x^754 + x^752 + x^751 + x^749 + x^746 + x^745 + x^740 + x^737 + x^736 + x^734 + x^730 + x^727 + x^723 + x^721 + x^720 + x^718 + x^716 + x^714 + x^712 + x^709 + x^707 + x^706 + x^705 + x^704 + x^703 + x^702 + x^701 + x^700 + x^699 + x^697 + x^696 + x^695 + x^690 + x^689 + x^687 + x^685 + x^684 + x^682 + x^678 + x^675 + x^672 + x^670 + x^669 + x^668 + x^666 + x^665 + x^663 + x^662 + x^661 + x^659 + x^657 + x^655 + x^651 + x^650 + x^648 + x^646 + x^641 + x^639 + x^634 + x^632 + x^631 + x^630 + x^629 + x^626 + x^625 + x^623 + x^620 + x^619 + x^614 + x^612 + x^610 + x^609 + x^608 + x^607 + x^606 + x^604 + x^603 + x^602 + x^601 + x^600 + x^597 + x^596 + x^595 + x^594 + x^593 + x^591 + x^588 + x^585 + x^584 + x^583 + x^582 + x^581 + x^580 + x^578 + x^576 + x^574 + x^573 + x^572 + x^571 + x^570 + x^569 + x^563 + x^561 + x^560 + x^558 + x^556 + x^553 + x^552 + x^551 + x^550 + x^549 + x^546 + x^545 + x^544 + x^543 + x^542 + x^539 + x^538 + x^537 + x^536 + x^534 + x^532 + x^531 + x^529 + x^523 + x^519 + x^518 + x^517 + x^515 + x^514 + x^513 + x^512 + x^510 + x^509 + x^508 + x^503 + x^502 + x^501 + x^500 + x^499 + x^498 + x^496 + x^495 + x^494 + x^491 + x^490 + x^489 + x^487 + x^486 + x^484 + x^482 + x^481 + x^473 + x^471 + x^468 + x^466 + x^465 + x^464 + x^461 + x^460 + x^457 + x^456 + x^455 + x^453 + x^450 + x^448 + x^444 + x^442 + x^436 + x^435 + x^434 + x^433 + x^432 + x^431 + x^430 + x^429 + x^422 + x^421 + x^420 + x^419 + x^418 + x^417 + x^416 + x^414 + x^413 + x^412 + x^411 + x^409 + x^408 + x^407 + x^406 + x^404 + x^403 + x^402 + x^401 + x^398 + x^395 + x^392 + x^391 + x^389 + x^387 + x^385 + x^380 + x^377 + x^374 + x^372 + x^371 + x^369 + x^366 + x^363 + x^362 + x^361 + x^359 + x^358 + x^356 + x^355 + x^354 + x^353 + x^352 + x^350 + x^349 + x^348 + x^347 + x^345 + x^344 + x^343 + x^336 + x^334 + x^331 + x^330 + x^327 + x^325 + x^324 + x^322 + x^317 + x^310 + x^308 + x^307 + x^304 + x^301 + x^300 + x^299 + x^298 + x^297 + x^293 + x^290 + x^286 + x^284 + x^283 + x^282 + x^281 + x^279 + x^277 + x^275 + x^274 + x^270 + x^269 + x^267 + x^264 + x^262 + x^259 + x^254 + x^253 + x^248 + x^247 + x^245 + x^243 + x^242 + x^241 + x^237 + x^236 + x^232 + x^231 + x^230 + x^229 + x^228 + x^226 + x^225 + x^224 + x^223 + x^221 + x^218 + x^215 + x^214 + x^213 + x^212 + x^210 + x^208 + x^204 + x^202 + x^201 + x^199 + x^196 + x^195 + x^193 + x^192 + x^191 + x^190 + x^186 + x^184 + x^179 + x^178 + x^177 + x^176 + x^175 + x^174 + x^173 + x^171 + x^169 + x^167 + x^166 + x^165 + x^160 + x^158 + x^155 + x^154 + x^153 + x^152 + x^151 + x^150 + x^149 + x^148 + x^147 + x^145 + x^143 + x^140 + x^139 + x^138 + x^134 + x^132 + x^130 + x^129 + x^128 + x^127 + x^120 + x^118 + x^117 + x^115 + x^113 + x^112 + x^109 + x^108 + x^107 + x^106 + x^105 + x^101 + x^100 + x^96 + x^95 + x^94 + x^93 + x^91 + x^89 + x^88 + x^87 + x^85 + x^84 + x^83 + x^82 + x^81 + x^79 + x^77 + x^75 + x^74 + x^73 + x^72 + x^70 + x^68 + x^67 + x^65 + x^64 + x^63 + x^60 + x^58 + x^57 + x^56 + x^54 + x^53 + x^52 + x^50 + x^45 + x^44 + x^43 + x^41 + x^35 + x^30 + x^23 + x^20 + x^18 + x^15 + x^6 + 1 + +44-9-45 441 x^928 + x^898 + x^890 + x^885 + x^880 + x^872 + x^870 + x^867 + x^860 + x^850 + x^847 + x^842 + x^837 + x^834 + x^830 + x^825 + x^824 + x^817 + x^816 + x^811 + x^808 + x^806 + x^802 + x^800 + x^798 + x^797 + x^794 + x^792 + x^790 + x^787 + x^786 + x^782 + x^778 + x^777 + x^776 + x^774 + x^773 + x^772 + x^771 + x^769 + x^768 + x^767 + x^765 + x^760 + x^759 + x^758 + x^757 + x^754 + x^750 + x^749 + x^747 + x^746 + x^745 + x^743 + x^741 + x^738 + x^737 + x^736 + x^734 + x^732 + x^725 + x^724 + x^722 + x^721 + x^720 + x^717 + x^716 + x^714 + x^713 + x^710 + x^708 + x^707 + x^705 + x^704 + x^702 + x^701 + x^699 + x^698 + x^696 + x^694 + x^692 + x^691 + x^690 + x^686 + x^683 + x^681 + x^679 + x^676 + x^674 + x^673 + x^672 + x^671 + x^668 + x^667 + x^665 + x^664 + x^663 + x^662 + x^661 + x^659 + x^658 + x^657 + x^655 + x^654 + x^652 + x^651 + x^648 + x^644 + x^643 + x^641 + x^640 + x^635 + x^634 + x^633 + x^632 + x^631 + x^630 + x^629 + x^628 + x^624 + x^621 + x^619 + x^618 + x^617 + x^616 + x^614 + x^611 + x^610 + x^608 + x^607 + x^599 + x^598 + x^596 + x^594 + x^590 + x^589 + x^588 + x^587 + x^586 + x^584 + x^581 + x^579 + x^578 + x^577 + x^575 + x^573 + x^572 + x^570 + x^569 + x^568 + x^567 + x^565 + x^564 + x^563 + x^562 + x^561 + x^559 + x^558 + x^554 + x^553 + x^552 + x^551 + x^549 + x^546 + x^544 + x^541 + x^537 + x^536 + x^535 + x^532 + x^531 + x^529 + x^526 + x^517 + x^516 + x^514 + x^511 + x^509 + x^507 + x^502 + x^501 + x^498 + x^497 + x^495 + x^494 + x^493 + x^491 + x^487 + x^486 + x^485 + x^484 + x^483 + x^482 + x^481 + x^480 + x^477 + x^474 + x^473 + x^470 + x^469 + x^467 + x^466 + x^465 + x^463 + x^462 + x^461 + x^460 + x^459 + x^457 + x^455 + x^454 + x^453 + x^451 + x^449 + x^448 + x^446 + x^443 + x^442 + x^440 + x^437 + x^436 + x^435 + x^434 + x^433 + x^431 + x^430 + x^427 + x^423 + x^419 + x^418 + x^414 + x^412 + x^411 + x^410 + x^409 + x^407 + x^406 + x^405 + x^402 + x^397 + x^395 + x^394 + x^389 + x^388 + x^384 + x^381 + x^380 + x^379 + x^378 + x^376 + x^373 + x^372 + x^371 + x^370 + x^368 + x^367 + x^366 + x^365 + x^364 + x^362 + x^361 + x^355 + x^354 + x^352 + x^350 + x^347 + x^345 + x^344 + x^343 + x^342 + x^339 + x^337 + x^335 + x^333 + x^332 + x^329 + x^327 + x^323 + x^322 + x^321 + x^320 + x^319 + x^316 + x^312 + x^310 + x^309 + x^308 + x^307 + x^305 + x^303 + x^302 + x^301 + x^298 + x^294 + x^293 + x^283 + x^282 + x^276 + x^275 + x^274 + x^271 + x^267 + x^264 + x^262 + x^261 + x^259 + x^258 + x^257 + x^256 + x^253 + x^252 + x^251 + x^250 + x^246 + x^245 + x^244 + x^243 + x^242 + x^241 + x^240 + x^238 + x^237 + x^236 + x^235 + x^233 + x^232 + x^229 + x^228 + x^225 + x^224 + x^223 + x^220 + x^216 + x^211 + x^210 + x^207 + x^205 + x^204 + x^202 + x^201 + x^200 + x^197 + x^196 + x^195 + x^194 + x^190 + x^189 + x^185 + x^183 + x^182 + x^181 + x^180 + x^179 + x^177 + x^176 + x^174 + x^173 + x^172 + x^171 + x^168 + x^167 + x^164 + x^163 + x^157 + x^156 + x^152 + x^151 + x^150 + x^148 + x^147 + x^146 + x^143 + x^142 + x^141 + x^140 + x^139 + x^138 + x^137 + x^135 + x^134 + x^133 + x^132 + x^129 + x^128 + x^127 + x^125 + x^122 + x^120 + x^119 + x^113 + x^112 + x^110 + x^109 + x^108 + x^107 + x^106 + x^105 + x^104 + x^101 + x^98 + x^97 + x^96 + x^93 + x^92 + x^91 + x^87 + x^83 + x^80 + x^78 + x^77 + x^74 + x^72 + x^66 + x^64 + x^63 + x^60 + x^59 + x^56 + x^51 + x^50 + x^49 + x^46 + x^43 + x^42 + x^41 + x^39 + x^38 + x^32 + x^31 + x^29 + x^28 + x^26 + x^24 + x^23 + x^21 + x^20 + x^19 + x^18 + x^15 + x^10 + 1 diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index 22136d687c..cdb6031b07 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2018. All Rights Reserved. +%% Copyright Ericsson AB 2004-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -2591,7 +2591,7 @@ otp_7184(Config) when is_list(Config) -> otp_7232(Config) when is_list(Config) -> Info = <<"qlc:info(qlc:sort(qlc:q([X || X <- [55296,56296]]), " "{order, fun(A,B)-> A>B end})).">>, - "qlc:sort([55296,56296],\n" + "qlc:sort([55296, 56296],\n" " [{order,\n" " fun(A, B) ->\n" " A > B\n" @@ -2752,7 +2752,7 @@ otp_10302(Config) when is_list(Config) -> h().">>, "ok.\n\"\x{400}\"\nA = \"\x{400}\".\nok.\n" - "1: io:setopts([{encoding,utf8}])\n-> ok.\n" + "1: io:setopts([{encoding, utf8}])\n-> ok.\n" "2: A = [1024] = \"\x{400}\"\n-> \"\x{400}\"\n" "3: b()\n-> ok.\nok.\n" = t({Node,Test4}), diff --git a/lib/stdlib/test/stdlib.spec b/lib/stdlib/test/stdlib.spec index 9c625091a8..bf64eae2c7 100644 --- a/lib/stdlib/test/stdlib.spec +++ b/lib/stdlib/test/stdlib.spec @@ -1,4 +1,7 @@ {suites,"../stdlib_test",all}. {skip_groups,"../stdlib_test",stdlib_bench_SUITE, - [base64,gen_server,gen_statem,unicode], + [binary,base64,gen_server,gen_statem,unicode], + "Benchmark only"}. +{skip_groups,"../stdlib_test",ets_SUITE, + [benchmark], "Benchmark only"}. diff --git a/lib/stdlib/test/stdlib_bench.spec b/lib/stdlib/test/stdlib_bench.spec index 7a0da811a0..6d665f22b6 100644 --- a/lib/stdlib/test/stdlib_bench.spec +++ b/lib/stdlib/test/stdlib_bench.spec @@ -8,3 +8,4 @@ {skip_groups,"../stdlib_test",stdlib_bench_SUITE, [gen_server_comparison,gen_statem_comparison], "Not a benchmark"}. +{groups,"../stdlib_test",ets_SUITE,[benchmark]}. diff --git a/lib/stdlib/test/stdlib_bench_SUITE.erl b/lib/stdlib/test/stdlib_bench_SUITE.erl index 2364e8376f..3456442088 100644 --- a/lib/stdlib/test/stdlib_bench_SUITE.erl +++ b/lib/stdlib/test/stdlib_bench_SUITE.erl @@ -29,7 +29,7 @@ suite() -> [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}]. all() -> - [{group,unicode},{group,base64}, + [{group,unicode},{group,base64},{group,binary}, {group,gen_server},{group,gen_statem}, {group,gen_server_comparison},{group,gen_statem_comparison}]. @@ -38,6 +38,13 @@ groups() -> [norm_nfc_list, norm_nfc_deep_l, norm_nfc_binary, string_lexemes_list, string_lexemes_binary ]}, + %% Only run 1 binary match repeat as it is very slow pre OTP-22. + %% The results seem to be stable enough anyway + {binary, [{repeat, 1}], + [match_single_pattern_no_match, + matches_single_pattern_no_match, + matches_single_pattern_eventual_match, + matches_single_pattern_frequent_match]}, {base64,[{repeat,5}], [decode_binary, decode_binary_to_string, decode_list, decode_list_to_string, @@ -60,9 +67,9 @@ cases(gen_server) -> [simple, simple_timer, simple_mon, simple_timer_mon, generic, generic_timer]; cases(gen_statem) -> - [generic, generic_fsm, generic_fsm_transit, - generic_statem, generic_statem_transit, - generic_statem_complex]. + [generic, generic_log, generic_log100, generic_fsm, generic_fsm_transit, + generic_statem, generic_statem_log, generic_statem_log100, + generic_statem_transit, generic_statem_complex]. init_per_group(gen_server, Config) -> compile_servers(Config), @@ -157,41 +164,59 @@ norm_data(Config) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +match_single_pattern_no_match(_Config) -> + Binary = binary:copy(<<"ugbcfuysabfuqyfikgfsdalpaskfhgjsdgfjwsalp">>, 1000000), + comment(test(100, binary, match, [Binary, <<"o">>])). + +matches_single_pattern_no_match(_Config) -> + Binary = binary:copy(<<"ugbcfuysabfuqyfikgfsdalpaskfhgjsdgfjwsalp">>, 1000000), + comment(test(100, binary, matches, [Binary, <<"o">>])). + +matches_single_pattern_eventual_match(_Config) -> + Binary = binary:copy(<<"ugbcfuysabfuqyfikgfsdalpaskfhgjsdgfjwsal\n">>, 1000000), + comment(test(100, binary, matches, [Binary, <<"\n">>])). + +matches_single_pattern_frequent_match(_Config) -> + Binary = binary:copy(<<"abc\n">>, 1000000), + comment(test(100, binary, matches, [Binary, <<"abc">>])). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + decode_binary(_Config) -> - comment(test(decode, encoded_binary())). + comment(test(base64, decode, [encoded_binary()])). decode_binary_to_string(_Config) -> - comment(test(decode_to_string, encoded_binary())). + comment(test(base64, decode_to_string, [encoded_binary()])). decode_list(_Config) -> - comment(test(decode, encoded_list())). + comment(test(base64, decode, [encoded_list()])). decode_list_to_string(_Config) -> - comment(test(decode_to_string, encoded_list())). + comment(test(base64, decode_to_string, [encoded_list()])). encode_binary(_Config) -> - comment(test(encode, binary())). + comment(test(base64, encode, [binary()])). encode_binary_to_string(_Config) -> - comment(test(encode_to_string, binary())). + comment(test(base64, encode_to_string, [binary()])). encode_list(_Config) -> - comment(test(encode, list())). + comment(test(base64, encode, [list()])). encode_list_to_string(_Config) -> - comment(test(encode_to_string, list())). + comment(test(base64, encode_to_string, [list()])). mime_binary_decode(_Config) -> - comment(test(mime_decode, encoded_binary())). + comment(test(base64, mime_decode, [encoded_binary()])). mime_binary_decode_to_string(_Config) -> - comment(test(mime_decode_to_string, encoded_binary())). + comment(test(base64, mime_decode_to_string, [encoded_binary()])). mime_list_decode(_Config) -> - comment(test(mime_decode, encoded_list())). + comment(test(base64, mime_decode, [encoded_list()])). mime_list_decode_to_string(_Config) -> - comment(test(mime_decode_to_string, encoded_list())). + comment(test(base64, mime_decode_to_string, [encoded_list()])). -define(SIZE, 10000). -define(N, 1000). @@ -209,15 +234,17 @@ binary() -> list() -> random_byte_list(?SIZE). -test(Func, Data) -> - F = fun() -> loop(?N, Func, Data) end, +test(Mod, Fun, Args) -> + test(?N, Mod, Fun, Args). +test(Iter, Mod, Fun, Args) -> + F = fun() -> loop(Iter, Mod, Fun, Args) end, {Time, ok} = timer:tc(fun() -> lspawn(F) end), - report_base64(Time). + report_mfa(Iter, Time, Mod). -loop(0, _F, _D) -> garbage_collect(), ok; -loop(N, F, D) -> - _ = base64:F(D), - loop(N - 1, F, D). +loop(0, _M, _F, _A) -> garbage_collect(), ok; +loop(N, M, F, A) -> + _ = apply(M, F, A), + loop(N - 1, M, F, A). lspawn(Fun) -> {Pid, Ref} = spawn_monitor(fun() -> exit(Fun()) end), @@ -225,10 +252,10 @@ lspawn(Fun) -> {'DOWN', Ref, process, Pid, Rep} -> Rep end. -report_base64(Time) -> - Tps = round((?N*1000000)/Time), +report_mfa(Iter, Time, Mod) -> + Tps = round((Iter*1000000)/Time), ct_event:notify(#event{name = benchmark_data, - data = [{suite, "stdlib_base64"}, + data = [{suite, "stdlib_" ++ atom_to_list(Mod)}, {value, Tps}]}), Tps. @@ -267,12 +294,24 @@ simple_timer_mon(Config) when is_list(Config) -> generic(Config) when is_list(Config) -> comment(do_tests(generic, single_small, Config)). +generic_log(Config) when is_list(Config) -> + comment(do_tests(generic_log, single_small, Config)). + +generic_log100(Config) when is_list(Config) -> + comment(do_tests(generic_log100, single_small, Config)). + generic_timer(Config) when is_list(Config) -> comment(do_tests(generic_timer, single_small, Config)). generic_statem(Config) when is_list(Config) -> comment(do_tests(generic_statem, single_small, Config)). +generic_statem_log(Config) when is_list(Config) -> + comment(do_tests(generic_statem_log, single_small, Config)). + +generic_statem_log100(Config) when is_list(Config) -> + comment(do_tests(generic_statem_log100, single_small, Config)). + generic_statem_transit(Config) when is_list(Config) -> comment(do_tests(generic_statem_transit, single_small, Config)). @@ -344,9 +383,9 @@ norm(T, Ref) -> do_tests(Test, ParamSet, Config) -> BenchmarkSuite = ?config(benchmark_suite, Config), - {Client, ServerMod} = bench(Test), + {Client, ServerMod, ServerArg} = bench(Test), {Parallelism, Message} = bench_params(ParamSet), - Fun = create_clients(Message, ServerMod, Client, Parallelism), + Fun = create_clients(Message, ServerMod, ServerArg, Client, Parallelism), {TotalLoops, AllPidTime} = run_test(Fun), try ?CALLS_PER_LOOP * round((1000 * TotalLoops) / AllPidTime) of PerSecond -> @@ -461,27 +500,35 @@ generic_fsm_transit_client(N, M, P) -> generic_fsm_transit_client(N+1, M, P). bench(simple) -> - {fun simple_client/3, simple_server}; + {fun simple_client/3, simple_server, term}; bench(simple_timer) -> - {fun simple_client_timer/3, simple_server_timer}; + {fun simple_client_timer/3, simple_server_timer, term}; bench(simple_mon) -> - {fun simple_client_mon/3, simple_server_mon}; + {fun simple_client_mon/3, simple_server_mon, term}; bench(simple_timer_mon) -> - {fun simple_client_timer_mon/3, simple_server_timer_mon}; + {fun simple_client_timer_mon/3, simple_server_timer_mon, term}; bench(generic) -> - {fun generic_client/3, generic_server}; + {fun generic_client/3, generic_server, [term]}; +bench(generic_log) -> + {fun generic_client/3, generic_server, [term,{debug,[log]}]}; +bench(generic_log100) -> + {fun generic_client/3, generic_server, [term,{debug,[{log,100}]}]}; bench(generic_timer) -> - {fun generic_timer_client/3, generic_server_timer}; + {fun generic_timer_client/3, generic_server_timer, term}; bench(generic_statem) -> - {fun generic_statem_client/3, generic_statem}; + {fun generic_statem_client/3, generic_statem, [term]}; +bench(generic_statem_log) -> + {fun generic_statem_client/3, generic_statem, [term,{debug,[log]}]}; +bench(generic_statem_log100) -> + {fun generic_statem_client/3, generic_statem, [term,{debug,[{log,100}]}]}; bench(generic_statem_transit) -> - {fun generic_statem_transit_client/3, generic_statem}; + {fun generic_statem_transit_client/3, generic_statem, [term]}; bench(generic_statem_complex) -> - {fun generic_statem_complex_client/3, generic_statem_complex}; + {fun generic_statem_complex_client/3, generic_statem_complex, term}; bench(generic_fsm) -> - {fun generic_fsm_client/3, generic_fsm}; + {fun generic_fsm_client/3, generic_fsm, term}; bench(generic_fsm_transit) -> - {fun generic_fsm_transit_client/3, generic_fsm}. + {fun generic_fsm_transit_client/3, generic_fsm, term}. %% -> {Parallelism, MessageTerm} bench_params(single_small) -> {1, small()}; @@ -509,10 +556,9 @@ parallelism() -> _ -> 1 end. -create_clients(M, ServerMod, Client, Parallel) -> +create_clients(M, ServerMod, ServerArg, Client, Parallel) -> fun() -> - State = term, - ServerPid = ServerMod:start(State), + ServerPid = ServerMod:start(ServerArg), PidRefs = [spawn_monitor(fun() -> Client(0, M, ServerPid) end) || _ <- lists:seq(1, Parallel)], timer:sleep(?MAX_TIME), diff --git a/lib/stdlib/test/stdlib_bench_SUITE_data/generic_fsm.erl b/lib/stdlib/test/stdlib_bench_SUITE_data/generic_fsm.erl index 50f7df7a2a..1abd9b1f2f 100644 --- a/lib/stdlib/test/stdlib_bench_SUITE_data/generic_fsm.erl +++ b/lib/stdlib/test/stdlib_bench_SUITE_data/generic_fsm.erl @@ -24,7 +24,7 @@ -export([init/1, terminate/3]). -export([state1/3, state2/3]). --behaivour(gen_fsm). +-behaviour(gen_fsm). %% API diff --git a/lib/stdlib/test/stdlib_bench_SUITE_data/generic_server.erl b/lib/stdlib/test/stdlib_bench_SUITE_data/generic_server.erl index abd61dcdef..3707d00dee 100644 --- a/lib/stdlib/test/stdlib_bench_SUITE_data/generic_server.erl +++ b/lib/stdlib/test/stdlib_bench_SUITE_data/generic_server.erl @@ -8,8 +8,8 @@ -define(GEN_SERVER, gen_server). -start(State) -> - {ok, Pid} = ?GEN_SERVER:start(?MODULE, State, []), +start([State|Opts]) -> + {ok, Pid} = ?GEN_SERVER:start(?MODULE, State, Opts), Pid. init(State) -> diff --git a/lib/stdlib/test/stdlib_bench_SUITE_data/generic_statem.erl b/lib/stdlib/test/stdlib_bench_SUITE_data/generic_statem.erl index 2e0491f060..b106619568 100644 --- a/lib/stdlib/test/stdlib_bench_SUITE_data/generic_statem.erl +++ b/lib/stdlib/test/stdlib_bench_SUITE_data/generic_statem.erl @@ -28,8 +28,8 @@ %% API -start(Data) -> - {ok, Pid} = gen_statem:start(?MODULE, Data, []), +start([Data|Opts]) -> + {ok, Pid} = gen_statem:start(?MODULE, Data, Opts), Pid. stop(P) -> @@ -52,7 +52,9 @@ init(Data) -> state1({call, From}, {reply, M}, Data) -> {keep_state, Data, {reply, From, M}}; state1({call, From}, {transit, M}, Data) -> - {next_state, state2, Data, {reply, From, M}}. + {next_state, state2, Data, + [{reply, From, M},{state_timeout,5000,5000}]}. state2({call, From}, {transit, M}, Data) -> - {next_state, state1, Data, {reply, From, M}}. + {next_state, state1, Data, + [{reply, From, M},{state_timeout,5000,5000}]}. diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl index 251e09121c..c9aadd7f10 100644 --- a/lib/stdlib/test/string_SUITE.erl +++ b/lib/stdlib/test/string_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2018. All Rights Reserved. +%% Copyright Ericsson AB 2004-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -103,6 +103,15 @@ debug() -> test(?LINE,?FUNCTION_NAME,B,C,D, false), test(?LINE,?FUNCTION_NAME,hd(C),[B|tl(C)],D, false)). +-define(TRY(Exp), + fun() -> + try Exp + catch _E:Reason:_ST -> + %% io:format("~p:~w: ~p: ~.0p ~p~n", + %% [?FUNCTION_NAME, ?LINE,_E,Reason, hd(_ST)]), + {'EXIT', Reason} + end + end()). is_empty(_) -> ?TEST("", [], true), @@ -126,6 +135,10 @@ length(_) -> ?TEST(["abc"|<<"abc">>], [], 6), ?TEST(["abc",["def"]], [], 6), ?TEST([<<97/utf8, 778/utf8, 98/utf8>>, [776,111,776]], [], 3), %% åäö in nfd + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:length(InvalidUTF8)), + {'EXIT', {badarg, _}} = ?TRY(string:length(<<$a, InvalidUTF8/binary, $z>>)), ok. equal(_) -> @@ -226,6 +239,8 @@ to_graphemes(_) -> true = erlang:length(GCs) =:= erlang:length(string:to_graphemes(NFD)), true = erlang:length(GCs) =:= erlang:length(string:to_graphemes(unicode:characters_to_nfc_list(String))), + + {'EXIT', {badarg, _}} = ?TRY(string:to_graphemes(<<$a,192,192,$z>>)), ok. reverse(_) -> @@ -238,6 +253,11 @@ reverse(_) -> ?TEST(Str2, [], lists:reverse(Str2)), ?TEST(Str3, [], lists:reverse(Str3)), true = string:reverse(Str3) =:= lists:reverse(string:to_graphemes(Str3)), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:reverse(InvalidUTF8)), + {'EXIT', {badarg, _}} = ?TRY(string:reverse(<<$a, InvalidUTF8/binary, $z>>)), + ok. slice(_) -> @@ -258,6 +278,14 @@ slice(_) -> ?TEST([<<"aå"/utf8>>,"äöbcd"], [3,3], "öbc"), ?TEST([<<"aåä"/utf8>>,"öbcd"], [3,10], "öbcd"), + InvalidUTF8 = <<192,192>>, + [$b, $c|InvalidUTF8] = string:slice(["abc", InvalidUTF8], 1), + InvalidUTF8 = string:slice(["abc", InvalidUTF8], 3), + {'EXIT', {badarg, _}} = ?TRY(string:slice(["abc", InvalidUTF8], 1, 5)), + BadUtf8 = <<$a, InvalidUTF8/binary, "teststring">>, + {'EXIT', {badarg, _}} = ?TRY(string:slice(BadUtf8, 2)), + {'EXIT', {badarg, _}} = ?TRY(string:slice(BadUtf8, 1, 5)), + {'EXIT', {badarg, _}} = ?TRY(string:slice(BadUtf8, 0, 5)), ok. pad(_) -> @@ -270,6 +298,10 @@ pad(_) -> ?TEST(Str, [10, trailing, $.], "Hallå....."), ?TEST(Str++["f"], [10, trailing, $.], "Hallåf...."), ?TEST(Str++[" flåwer"], [10, trailing, $.], "Hallå flåwer"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:pad(InvalidUTF8, 10, both, $.)), + {'EXIT', {badarg, _}} = ?TRY(string:pad(<<$a, InvalidUTF8/binary, $z>>, 10, both, $.)), ok. trim(_) -> @@ -300,6 +332,11 @@ trim(_) -> ?TEST([[<<"!v">>|<<204,128,$v,204,129>>]],[trailing, [[$v,769]]], [$!,$v,768]), ?TEST([[[<<"v">>|<<204,129,118,204,128,118>>],769,118,769]], [trailing, [[118,769]]], [$v,769,$v,768]), ?TEST([<<"vv">>|<<204,128,118,204,128>>], [trailing, [[118,768]]], "v"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:trim(InvalidUTF8, both, "az")), + %% Not checked (using binary search) + %% {'EXIT', {badarg, _}} = ?TRY(string:trim(<<$a, $b, InvalidUTF8/binary, $z>>, both, "az")), ok. chomp(_) -> @@ -400,6 +437,13 @@ take(_) -> ?TEST([<<"e">>,778,"åäöe", <<778/utf8>>, $e, 779], [[[$e,778]], true, trailing], {[$e,778]++"åäöe"++[778], [$e,779]}), + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:take(InvalidUTF8, [$.], false, leading)), + %% Not checked (using binary search) + %% {'EXIT', {badarg, _}} = ?TRY(string:take(InvalidUTF8, [$.], true, leading)), + %% {'EXIT', {badarg, _}} = ?TRY(string:take(InvalidUTF8, [$.], false, trailing)), + {'EXIT', {badarg, _}} = ?TRY(string:take(InvalidUTF8, [$.], true, trailing)), + ok. @@ -416,6 +460,11 @@ uppercase(_) -> ?TEST("ljLJ", [], "LJLJ"), ?TEST("LJlj", [], "LJLJ"), ?TEST("ß sharp s", [], "SS SHARP S"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:uppercase(InvalidUTF8)), + {'EXIT', {badarg, _}} = ?TRY(string:uppercase(<<$a, InvalidUTF8/binary, $z>>)), + ok. lowercase(_) -> @@ -429,6 +478,10 @@ lowercase(_) -> ?TEST(["Mic",<<"HAŁ"/utf8>>], [], "michał"), ?TEST("ß SHARP S", [], "ß sharp s"), ?TEST("İ I WITH DOT ABOVE", [], "i̇ i with dot above"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:lowercase(InvalidUTF8)), + {'EXIT', {badarg, _}} = ?TRY(string:lowercase(<<$a, InvalidUTF8/binary, $z>>)), ok. titlecase(_) -> @@ -442,6 +495,10 @@ titlecase(_) -> ?TEST("ljLJ", [], "LjLJ"), ?TEST("LJlj", [], "Ljlj"), ?TEST("ß sharp s", [], "Ss sharp s"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:titlecase(InvalidUTF8)), + <<$A, _/binary>> = ?TRY(string:titlecase(<<$a, InvalidUTF8/binary, $z>>)), ok. casefold(_) -> @@ -456,6 +513,10 @@ casefold(_) -> ?TEST("ß SHARP S", [], "ss sharp s"), ?TEST("ẞ SHARP S", [], "ss sharp s"), ?TEST("İ I WITH DOT ABOVE", [], "i̇ i with dot above"), + + InvalidUTF8 = <<192,192>>, + {'EXIT', {badarg, _}} = ?TRY(string:casefold(InvalidUTF8)), + {'EXIT', {badarg, _}} = ?TRY(string:casefold(<<$a, InvalidUTF8/binary, $z>>)), ok. @@ -740,7 +801,7 @@ meas(Config) -> _ -> % No scaling, run at most 1.5 min Tester = spawn(Exec), receive {test_done, Tester} -> ok - after 90000 -> + after 118000 -> io:format("Timelimit reached stopping~n",[]), exit(Tester, die) end, @@ -754,19 +815,22 @@ do_measure(DataDir) -> io:format("~p~n",[byte_size(Bin)]), Do = fun(Name, Func, Mode) -> {N, Mean, Stddev, _} = time_func(Func, Mode, Bin, 20), - io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", + io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n", [Name, Mode, Mean/1000, Stddev/1000, N]) end, Do2 = fun(Name, Func, Mode) -> {N, Mean, Stddev, _} = time_func(Func, binary, <<>>, 20), - io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", + io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n", [Name, Mode, Mean/1000, Stddev/1000, N]) end, + %% lefty_list means a list balanced to the left, like + %% [[[30],31],32]. Only some functions check such lists. + Modes = [list, lefty_list, binary, {many_lists,1}, {many_lists, 4}], io:format("----------------------~n"), Do(old_tokens, fun(Str) -> string:tokens(Str, [$\n,$\r]) end, list), Tokens = {lexemes, fun(Str) -> string:lexemes(Str, [$\n,$\r]) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- Modes], S0 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....", S0B = <<"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....">>, @@ -824,17 +888,17 @@ do_measure(DataDir) -> io:format("--~n",[]), NthTokens = {nth_lexemes, fun(Str) -> string:nth_lexeme(Str, 18000, [$\n,$\r]) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- Modes], Do2(take_t, repeat(fun() -> string:take(S0, [$.,$y], false, trailing) end), list), Do2(take_t, repeat(fun() -> string:take(S0B, [$.,$y], false, trailing) end), binary), Do2(take_tc, repeat(fun() -> string:take(S0, [$x], true, trailing) end), list), Do2(take_tc, repeat(fun() -> string:take(S0B, [$x], true, trailing) end), binary), Length = {length, fun(Str) -> string:length(Str) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- Modes], Reverse = {reverse, fun(Str) -> string:reverse(Str) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- Modes], ok. @@ -1064,7 +1128,33 @@ time_func(N,Sum,SumSq, _, _, Res, _) -> {N, Mean, Stdev, Res}. mode(binary, Bin) -> Bin; -mode(list, Bin) -> unicode:characters_to_list(Bin). +mode(list, Bin) -> unicode:characters_to_list(Bin); +mode(lefty_list, Bin) -> + L = unicode:characters_to_list(Bin), + to_left(L); +mode({many_lists, N}, Bin) -> + group(unicode:characters_to_list(Bin), N). + +group([], _N) -> + []; +group(L, N) -> + try lists:split(N, L) of + {L1, L2} -> + [L1 | group(L2, N)] + catch + _:_ -> + [L] + end. + +to_left([]) -> + []; +to_left([H|L]) -> + to_left([H], L). + +to_left(V, []) -> + V; +to_left(V, [H|L]) -> + to_left([V,H], L). %% %% Old string lists Test cases starts here. diff --git a/lib/stdlib/test/unicode_util_SUITE.erl b/lib/stdlib/test/unicode_util_SUITE.erl index 044b4e5834..6f55f204f4 100644 --- a/lib/stdlib/test/unicode_util_SUITE.erl +++ b/lib/stdlib/test/unicode_util_SUITE.erl @@ -428,7 +428,15 @@ mode(deep_l, Bin) -> [unicode:characters_to_list(Bin)]. fetch(Str, F) -> case F(Str) of [] -> []; - [CP|R] -> [CP|fetch(R,F)] + [CP|R] -> + %% If input is a binary R should be binary + if is_binary(Str) == false -> ok; + is_binary(R); R =:= [] -> ok; + true -> + io:format("Char: ~tc Tail:~tP~n", [CP,R,10]), + exit({bug, F}) + end, + [CP|fetch(R,F)] end. %% *Test.txt file helpers diff --git a/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt index d7d8f90de0..6847953c23 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt @@ -1,6 +1,6 @@ -# GraphemeBreakTest-10.0.0.txt -# Date: 2017-04-14, 05:40:29 GMT -# © 2017 Unicode®, Inc. +# GraphemeBreakTest-11.0.0.txt +# Date: 2018-03-18, 13:30:33 GMT +# © 2018 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -23,828 +23,678 @@ # These samples may be extended or changed in the future. # ÷ 0020 ÷ 0020 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 0020 × 0308 ÷ 0020 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 0020 × 0308 ÷ 0020 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0020 ÷ 000D ÷ # ÷ [0.2] SPACE (Other) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 0020 × 0308 ÷ 000D ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 0020 × 0308 ÷ 000D ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ 0020 ÷ 000A ÷ # ÷ [0.2] SPACE (Other) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 0020 × 0308 ÷ 000A ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 0020 × 0308 ÷ 000A ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ 0020 ÷ 0001 ÷ # ÷ [0.2] SPACE (Other) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0020 × 0308 ÷ 0001 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0020 × 0300 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 0020 × 0308 × 0300 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 0020 × 0308 ÷ 0001 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 0020 × 034F ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0020 × 0308 × 034F ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0020 ÷ 1F1E6 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 0020 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ 0020 ÷ 0600 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 0020 × 0308 ÷ 0600 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 0020 × 0308 ÷ 0600 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ 0020 × 0903 ÷ # ÷ [0.2] SPACE (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 0020 × 0308 × 0903 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 0020 × 0308 × 0903 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ 0020 ÷ 1100 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 0020 × 0308 ÷ 1100 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 0020 × 0308 ÷ 1100 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ 0020 ÷ 1160 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 0020 × 0308 ÷ 1160 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 0020 × 0308 ÷ 1160 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ 0020 ÷ 11A8 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 0020 × 0308 ÷ 11A8 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 0020 × 0308 ÷ 11A8 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ 0020 ÷ AC00 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 0020 × 0308 ÷ AC00 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 0020 × 0308 ÷ AC00 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ 0020 ÷ AC01 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0020 × 0308 ÷ AC01 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0020 ÷ 1F1E6 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0020 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0020 ÷ 261D ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0020 × 0308 ÷ 261D ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0020 ÷ 1F3FB ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0020 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0020 × 200D ÷ # ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0020 × 0308 × 200D ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0020 ÷ 2640 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0020 × 0308 ÷ 2640 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0020 ÷ 1F466 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 0020 × 0308 ÷ 1F466 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 0020 × 0308 ÷ AC01 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 0020 ÷ 231A ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 0020 × 0308 ÷ 231A ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 0020 × 0300 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0020 × 0308 × 0300 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0020 × 200D ÷ # ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 0020 × 0308 × 200D ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 0020 ÷ 0378 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0020 × 0308 ÷ 0378 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 0020 × 0308 ÷ 0378 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 0020 ÷ D800 ÷ # ÷ [0.2] SPACE (Other) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0020 × 0308 ÷ D800 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 0020 × 0308 ÷ D800 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 000D ÷ 0020 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] SPACE (Other) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 000D ÷ 000D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 000D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 000D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ 000D × 000A ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) × [3.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 000A ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 000A ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ 000D ÷ 0001 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 0001 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 000D ÷ 0300 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 000D ÷ 0308 × 0300 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 0001 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 000D ÷ 034F ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 000D ÷ 0308 × 034F ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 000D ÷ 1F1E6 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 1F1E6 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ 000D ÷ 0600 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 0600 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 0600 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ 000D ÷ 0903 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 000D ÷ 0308 × 0903 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 000D ÷ 0308 × 0903 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ 000D ÷ 1100 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 1100 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 1100 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ 000D ÷ 1160 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 1160 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 1160 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ 000D ÷ 11A8 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 11A8 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 11A8 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ 000D ÷ AC00 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 000D ÷ 0308 ÷ AC00 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 000D ÷ 0308 ÷ AC00 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ 000D ÷ AC01 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 000D ÷ 0308 ÷ AC01 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 000D ÷ 1F1E6 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 1F1E6 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 000D ÷ 261D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 261D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 000D ÷ 1F3FB ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 000D ÷ 200D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 000D ÷ 0308 × 200D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 000D ÷ 2640 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 000D ÷ 1F466 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] BOY (EBG) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 000D ÷ 0308 ÷ AC01 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 000D ÷ 231A ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] WATCH (ExtPict) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 231A ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 000D ÷ 0300 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 000D ÷ 0308 × 0300 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 000D ÷ 200D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 000D ÷ 0308 × 200D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 000D ÷ 0378 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 000D ÷ D800 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 000D ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 000D ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 000A ÷ 0020 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] SPACE (Other) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 000A ÷ 000D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 000D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 000D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ 000A ÷ 000A ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 000A ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 000A ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ 000A ÷ 0001 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 0001 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 000A ÷ 0300 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 000A ÷ 0308 × 0300 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 0001 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 000A ÷ 034F ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 000A ÷ 0308 × 034F ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 000A ÷ 1F1E6 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 1F1E6 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ 000A ÷ 0600 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 0600 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 0600 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ 000A ÷ 0903 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 000A ÷ 0308 × 0903 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 000A ÷ 0308 × 0903 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ 000A ÷ 1100 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 1100 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 1100 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ 000A ÷ 1160 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 1160 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 1160 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ 000A ÷ 11A8 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 11A8 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 11A8 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ 000A ÷ AC00 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 000A ÷ 0308 ÷ AC00 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 000A ÷ 0308 ÷ AC00 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ 000A ÷ AC01 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 000A ÷ 0308 ÷ AC01 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 000A ÷ 1F1E6 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 1F1E6 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 000A ÷ 261D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 261D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 000A ÷ 1F3FB ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 000A ÷ 200D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 000A ÷ 0308 × 200D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 000A ÷ 2640 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 000A ÷ 1F466 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] BOY (EBG) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 000A ÷ 0308 ÷ AC01 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 000A ÷ 231A ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] WATCH (ExtPict) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 231A ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 000A ÷ 0300 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 000A ÷ 0308 × 0300 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 000A ÷ 200D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 000A ÷ 0308 × 200D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 000A ÷ 0378 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 000A ÷ D800 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 000A ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 000A ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 0001 ÷ 0020 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] SPACE (Other) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0001 ÷ 000D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 000D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 000D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ 0001 ÷ 000A ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 000A ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 000A ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ 0001 ÷ 0001 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 0001 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0001 ÷ 0300 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 0001 ÷ 0308 × 0300 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 0001 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 0001 ÷ 034F ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0001 ÷ 0308 × 034F ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0001 ÷ 1F1E6 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 1F1E6 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ 0001 ÷ 0600 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 0600 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 0600 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ 0001 ÷ 0903 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 0001 ÷ 0308 × 0903 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 0001 ÷ 0308 × 0903 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ 0001 ÷ 1100 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 1100 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 1100 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ 0001 ÷ 1160 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 1160 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 1160 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ 0001 ÷ 11A8 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 11A8 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 11A8 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ 0001 ÷ AC00 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ AC00 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ AC00 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ 0001 ÷ AC01 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ AC01 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0001 ÷ 1F1E6 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 1F1E6 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0001 ÷ 261D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 261D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0001 ÷ 1F3FB ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0001 ÷ 200D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0001 ÷ 0308 × 200D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0001 ÷ 2640 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0001 ÷ 1F466 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] BOY (EBG) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ AC01 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 0001 ÷ 231A ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] WATCH (ExtPict) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 231A ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 0001 ÷ 0300 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0001 ÷ 0308 × 0300 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0001 ÷ 200D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 0001 ÷ 0308 × 200D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 0001 ÷ 0378 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 0001 ÷ D800 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0300 ÷ 0020 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 0300 × 0308 ÷ 0020 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 0300 ÷ 000D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 0300 × 0308 ÷ 000D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 0300 ÷ 000A ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 0300 × 0308 ÷ 000A ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 0300 ÷ 0001 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0300 × 0308 ÷ 0001 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0300 × 0300 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 0300 × 0308 × 0300 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 0300 ÷ 0600 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 0300 × 0308 ÷ 0600 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 0300 × 0903 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 0300 × 0308 × 0903 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 0300 ÷ 1100 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 0300 × 0308 ÷ 1100 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 0300 ÷ 1160 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 0300 × 0308 ÷ 1160 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 0300 ÷ 11A8 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 0300 × 0308 ÷ 11A8 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 0300 ÷ AC00 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 0300 × 0308 ÷ AC00 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 0300 ÷ AC01 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0300 × 0308 ÷ AC01 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0300 ÷ 1F1E6 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0300 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0300 ÷ 261D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0300 × 0308 ÷ 261D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0300 ÷ 1F3FB ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0300 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0300 × 200D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0300 × 0308 × 200D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0300 ÷ 2640 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0300 × 0308 ÷ 2640 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0300 ÷ 1F466 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 0300 × 0308 ÷ 1F466 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 0300 ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0300 × 0308 ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0300 ÷ D800 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0300 × 0308 ÷ D800 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 034F ÷ 0020 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 034F × 0308 ÷ 0020 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 034F ÷ 000D ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 034F × 0308 ÷ 000D ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 034F ÷ 000A ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 034F × 0308 ÷ 000A ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 034F ÷ 0001 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 034F × 0308 ÷ 0001 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 034F × 034F ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 034F × 0308 × 034F ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 034F ÷ 1F1E6 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 034F × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 034F ÷ 0600 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 034F × 0308 ÷ 0600 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 034F × 0903 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 034F × 0308 × 0903 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 034F ÷ 1100 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 034F × 0308 ÷ 1100 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 034F ÷ 1160 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 034F × 0308 ÷ 1160 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 034F ÷ 11A8 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 034F × 0308 ÷ 11A8 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 034F ÷ AC00 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 034F × 0308 ÷ AC00 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 034F ÷ AC01 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 034F × 0308 ÷ AC01 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 034F ÷ 231A ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 034F × 0308 ÷ 231A ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 034F × 0300 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 034F × 0308 × 0300 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 034F × 200D ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 034F × 0308 × 200D ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 034F ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 034F × 0308 ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 034F ÷ D800 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 034F × 0308 ÷ D800 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 1F1E6 ÷ 0020 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 0020 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 1F1E6 ÷ 000D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 000D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 1F1E6 ÷ 000A ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 000A ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 1F1E6 ÷ 0001 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 0001 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 1F1E6 × 034F ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 1F1E6 × 0308 × 034F ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 1F1E6 × 1F1E6 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [12.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 1F1E6 ÷ 0600 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 0600 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 1F1E6 × 0903 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 1F1E6 × 0308 × 0903 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 1F1E6 ÷ 1100 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 1100 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 1F1E6 ÷ 1160 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 1160 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 1F1E6 ÷ 11A8 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 11A8 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 1F1E6 ÷ AC00 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ AC00 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 1F1E6 ÷ AC01 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ AC01 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 1F1E6 ÷ 231A ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 231A ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 1F1E6 × 0300 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 1F1E6 × 0308 × 0300 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 1F1E6 × 200D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 1F1E6 × 0308 × 200D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 1F1E6 ÷ 0378 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 0378 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 1F1E6 ÷ D800 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ D800 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 0600 × 0020 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] SPACE (Other) ÷ [0.3] -÷ 0600 × 0308 ÷ 0020 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 0600 × 0308 ÷ 0020 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0600 ÷ 000D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 0600 × 0308 ÷ 000D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 0600 × 0308 ÷ 000D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ 0600 ÷ 000A ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 0600 × 0308 ÷ 000A ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 0600 × 0308 ÷ 000A ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ 0600 ÷ 0001 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0600 × 0308 ÷ 0001 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0600 × 0300 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 0600 × 0308 × 0300 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 0600 × 0308 ÷ 0001 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 0600 × 034F ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0600 × 0308 × 034F ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0600 × 1F1E6 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 0600 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ 0600 × 0600 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 0600 × 0308 ÷ 0600 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 0600 × 0308 ÷ 0600 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ 0600 × 0903 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 0600 × 0308 × 0903 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 0600 × 0308 × 0903 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ 0600 × 1100 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 0600 × 0308 ÷ 1100 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 0600 × 0308 ÷ 1100 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ 0600 × 1160 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 0600 × 0308 ÷ 1160 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 0600 × 0308 ÷ 1160 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ 0600 × 11A8 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 0600 × 0308 ÷ 11A8 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 0600 × 0308 ÷ 11A8 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ 0600 × AC00 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 0600 × 0308 ÷ AC00 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 0600 × 0308 ÷ AC00 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ 0600 × AC01 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0600 × 0308 ÷ AC01 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0600 × 1F1E6 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0600 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0600 × 261D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0600 × 0308 ÷ 261D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0600 × 1F3FB ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0600 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0600 × 200D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0600 × 0308 × 200D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0600 × 2640 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0600 × 0308 ÷ 2640 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0600 × 1F466 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] BOY (EBG) ÷ [0.3] -÷ 0600 × 0308 ÷ 1F466 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 0600 × 0308 ÷ AC01 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 0600 × 231A ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] WATCH (ExtPict) ÷ [0.3] +÷ 0600 × 0308 ÷ 231A ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 0600 × 0300 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0600 × 0308 × 0300 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0600 × 200D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 0600 × 0308 × 200D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 0600 × 0378 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] <reserved-0378> (Other) ÷ [0.3] -÷ 0600 × 0308 ÷ 0378 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 0600 × 0308 ÷ 0378 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 0600 ÷ D800 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0600 × 0308 ÷ D800 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 0600 × 0308 ÷ D800 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 0903 ÷ 0020 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 0903 × 0308 ÷ 0020 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 0903 × 0308 ÷ 0020 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0903 ÷ 000D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 0903 × 0308 ÷ 000D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 0903 × 0308 ÷ 000D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ 0903 ÷ 000A ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 0903 × 0308 ÷ 000A ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 0903 × 0308 ÷ 000A ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ 0903 ÷ 0001 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0903 × 0308 ÷ 0001 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0903 × 0300 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 0903 × 0308 × 0300 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 0903 × 0308 ÷ 0001 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 0903 × 034F ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0903 × 0308 × 034F ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0903 ÷ 1F1E6 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 0903 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ 0903 ÷ 0600 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 0903 × 0308 ÷ 0600 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 0903 × 0308 ÷ 0600 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ 0903 × 0903 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 0903 × 0308 × 0903 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 0903 × 0308 × 0903 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ 0903 ÷ 1100 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 0903 × 0308 ÷ 1100 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 0903 × 0308 ÷ 1100 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ 0903 ÷ 1160 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 0903 × 0308 ÷ 1160 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 0903 × 0308 ÷ 1160 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ 0903 ÷ 11A8 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 0903 × 0308 ÷ 11A8 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 0903 × 0308 ÷ 11A8 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ 0903 ÷ AC00 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 0903 × 0308 ÷ AC00 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 0903 × 0308 ÷ AC00 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ 0903 ÷ AC01 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0903 × 0308 ÷ AC01 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0903 ÷ 1F1E6 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0903 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0903 ÷ 261D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0903 × 0308 ÷ 261D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0903 ÷ 1F3FB ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0903 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0903 × 200D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0903 × 0308 × 200D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0903 ÷ 2640 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0903 × 0308 ÷ 2640 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0903 ÷ 1F466 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 0903 × 0308 ÷ 1F466 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 0903 × 0308 ÷ AC01 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 0903 ÷ 231A ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 0903 × 0308 ÷ 231A ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 0903 × 0300 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0903 × 0308 × 0300 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0903 × 200D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 0903 × 0308 × 200D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 0903 ÷ 0378 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0903 × 0308 ÷ 0378 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 0903 × 0308 ÷ 0378 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 0903 ÷ D800 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0903 × 0308 ÷ D800 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 0903 × 0308 ÷ D800 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 1100 ÷ 0020 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 1100 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 1100 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 1100 ÷ 000D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 1100 × 0308 ÷ 000D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 1100 × 0308 ÷ 000D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ 1100 ÷ 000A ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 1100 × 0308 ÷ 000A ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 1100 × 0308 ÷ 000A ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ 1100 ÷ 0001 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 1100 × 0308 ÷ 0001 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 1100 × 0300 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 1100 × 0308 × 0300 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 1100 × 0308 ÷ 0001 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 1100 × 034F ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 1100 × 0308 × 034F ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 1100 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 1100 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ 1100 ÷ 0600 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 1100 × 0308 ÷ 0600 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 1100 × 0308 ÷ 0600 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ 1100 × 0903 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 1100 × 0308 × 0903 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 1100 × 0308 × 0903 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ 1100 × 1100 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 1100 × 0308 ÷ 1100 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 1100 × 0308 ÷ 1100 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ 1100 × 1160 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 1100 × 0308 ÷ 1160 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 1100 × 0308 ÷ 1160 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ 1100 ÷ 11A8 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 1100 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 1100 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ 1100 × AC00 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 1100 × 0308 ÷ AC00 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 1100 × 0308 ÷ AC00 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ 1100 × AC01 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 1100 × 0308 ÷ AC01 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 1100 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 1100 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 1100 ÷ 261D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 1100 × 0308 ÷ 261D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 1100 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 1100 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 1100 × 200D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1100 × 0308 × 200D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1100 ÷ 2640 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 1100 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 1100 ÷ 1F466 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 1100 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 1100 × 0308 ÷ AC01 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 1100 ÷ 231A ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 1100 × 0308 ÷ 231A ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 1100 × 0300 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 1100 × 0308 × 0300 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 1100 × 200D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 1100 × 0308 × 200D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 1100 ÷ 0378 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 1100 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 1100 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 1100 ÷ D800 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 1100 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 1100 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 1160 ÷ 0020 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 1160 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 1160 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 1160 ÷ 000D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 1160 × 0308 ÷ 000D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 1160 × 0308 ÷ 000D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ 1160 ÷ 000A ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 1160 × 0308 ÷ 000A ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 1160 × 0308 ÷ 000A ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ 1160 ÷ 0001 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 1160 × 0308 ÷ 0001 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 1160 × 0300 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 1160 × 0308 × 0300 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 1160 × 0308 ÷ 0001 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 1160 × 034F ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 1160 × 0308 × 034F ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 1160 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 1160 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ 1160 ÷ 0600 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 1160 × 0308 ÷ 0600 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 1160 × 0308 ÷ 0600 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ 1160 × 0903 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 1160 × 0308 × 0903 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 1160 × 0308 × 0903 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ 1160 ÷ 1100 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 1160 × 0308 ÷ 1100 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 1160 × 0308 ÷ 1100 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ 1160 × 1160 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [7.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 1160 × 0308 ÷ 1160 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 1160 × 0308 ÷ 1160 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ 1160 × 11A8 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [7.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 1160 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 1160 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ 1160 ÷ AC00 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 1160 × 0308 ÷ AC00 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 1160 × 0308 ÷ AC00 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ 1160 ÷ AC01 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 1160 × 0308 ÷ AC01 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 1160 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 1160 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 1160 ÷ 261D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 1160 × 0308 ÷ 261D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 1160 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 1160 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 1160 × 200D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1160 × 0308 × 200D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1160 ÷ 2640 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 1160 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 1160 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 1160 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 1160 × 0308 ÷ AC01 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 1160 ÷ 231A ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 1160 × 0308 ÷ 231A ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 1160 × 0300 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 1160 × 0308 × 0300 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 1160 × 200D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 1160 × 0308 × 200D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 1160 ÷ 0378 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 1160 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 1160 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 1160 ÷ D800 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 1160 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 1160 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 11A8 ÷ 0020 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 11A8 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 11A8 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 11A8 ÷ 000D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 11A8 × 0308 ÷ 000D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 11A8 × 0308 ÷ 000D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ 11A8 ÷ 000A ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 11A8 × 0308 ÷ 000A ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 11A8 × 0308 ÷ 000A ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ 11A8 ÷ 0001 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 11A8 × 0308 ÷ 0001 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 11A8 × 0300 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 11A8 × 0308 × 0300 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 11A8 × 0308 ÷ 0001 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 11A8 × 034F ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 11A8 × 0308 × 034F ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 11A8 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 11A8 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ 11A8 ÷ 0600 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 11A8 × 0308 ÷ 0600 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 11A8 × 0308 ÷ 0600 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ 11A8 × 0903 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 11A8 × 0308 × 0903 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 11A8 × 0308 × 0903 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ 11A8 ÷ 1100 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 11A8 × 0308 ÷ 1100 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 11A8 × 0308 ÷ 1100 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ 11A8 ÷ 1160 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 11A8 × 0308 ÷ 1160 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 11A8 × 0308 ÷ 1160 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ 11A8 × 11A8 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [8.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 11A8 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 11A8 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ 11A8 ÷ AC00 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 11A8 × 0308 ÷ AC00 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 11A8 × 0308 ÷ AC00 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ 11A8 ÷ AC01 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 11A8 × 0308 ÷ AC01 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 11A8 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 11A8 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 11A8 ÷ 261D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 11A8 × 0308 ÷ 261D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 11A8 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 11A8 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 11A8 × 200D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 11A8 × 0308 × 200D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 11A8 ÷ 2640 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 11A8 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 11A8 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 11A8 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 11A8 × 0308 ÷ AC01 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 11A8 ÷ 231A ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 11A8 × 0308 ÷ 231A ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 11A8 × 0300 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 11A8 × 0308 × 0300 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 11A8 × 200D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 11A8 × 0308 × 200D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 11A8 ÷ 0378 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 11A8 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 11A8 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 11A8 ÷ D800 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 11A8 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 11A8 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ AC00 ÷ 0020 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ AC00 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ AC00 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ AC00 ÷ 000D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ AC00 × 0308 ÷ 000D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ AC00 × 0308 ÷ 000D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ AC00 ÷ 000A ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ AC00 × 0308 ÷ 000A ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ AC00 × 0308 ÷ 000A ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ AC00 ÷ 0001 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ AC00 × 0308 ÷ 0001 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ AC00 × 0300 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ AC00 × 0308 × 0300 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ AC00 × 0308 ÷ 0001 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ AC00 × 034F ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ AC00 × 0308 × 034F ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ AC00 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ AC00 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ AC00 ÷ 0600 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ AC00 × 0308 ÷ 0600 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ AC00 × 0308 ÷ 0600 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ AC00 × 0903 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ AC00 × 0308 × 0903 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ AC00 × 0308 × 0903 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ AC00 ÷ 1100 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ AC00 × 0308 ÷ 1100 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ AC00 × 0308 ÷ 1100 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ AC00 × 1160 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [7.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ AC00 × 0308 ÷ 1160 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ AC00 × 0308 ÷ 1160 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ AC00 × 11A8 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [7.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ AC00 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ AC00 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ AC00 ÷ AC00 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ AC00 × 0308 ÷ AC00 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ AC00 × 0308 ÷ AC00 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ AC00 ÷ AC01 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ AC00 × 0308 ÷ AC01 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ AC00 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ AC00 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ AC00 ÷ 261D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ AC00 × 0308 ÷ 261D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ AC00 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ AC00 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ AC00 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ AC00 × 0308 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ AC00 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ AC00 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ AC00 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ AC00 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ AC00 × 0308 ÷ AC01 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ AC00 ÷ 231A ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ AC00 × 0308 ÷ 231A ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ AC00 × 0300 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ AC00 × 0308 × 0300 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ AC00 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ AC00 × 0308 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ AC00 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ AC00 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ AC00 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ AC00 ÷ D800 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ AC00 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ AC00 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ AC01 ÷ 0020 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ AC01 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ AC01 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ AC01 ÷ 000D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ AC01 × 0308 ÷ 000D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ AC01 × 0308 ÷ 000D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ AC01 ÷ 000A ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ AC01 × 0308 ÷ 000A ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ AC01 × 0308 ÷ 000A ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ AC01 ÷ 0001 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ AC01 × 0308 ÷ 0001 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ AC01 × 0300 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ AC01 × 0308 × 0300 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ AC01 × 0308 ÷ 0001 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ AC01 × 034F ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ AC01 × 0308 × 034F ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ AC01 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ AC01 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ AC01 ÷ 0600 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ AC01 × 0308 ÷ 0600 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ AC01 × 0308 ÷ 0600 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ AC01 × 0903 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ AC01 × 0308 × 0903 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ AC01 × 0308 × 0903 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ AC01 ÷ 1100 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ AC01 × 0308 ÷ 1100 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ AC01 × 0308 ÷ 1100 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ AC01 ÷ 1160 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ AC01 × 0308 ÷ 1160 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ AC01 × 0308 ÷ 1160 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ AC01 × 11A8 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [8.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ AC01 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ AC01 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ AC01 ÷ AC00 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ AC01 × 0308 ÷ AC00 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ AC01 × 0308 ÷ AC00 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ AC01 ÷ AC01 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ AC01 × 0308 ÷ AC01 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ AC01 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ AC01 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ AC01 ÷ 261D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ AC01 × 0308 ÷ 261D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ AC01 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ AC01 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ AC01 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ AC01 × 0308 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ AC01 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ AC01 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ AC01 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ AC01 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ AC01 × 0308 ÷ AC01 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ AC01 ÷ 231A ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ AC01 × 0308 ÷ 231A ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ AC01 × 0300 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ AC01 × 0308 × 0300 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ AC01 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ AC01 × 0308 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ AC01 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ AC01 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ AC01 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ AC01 ÷ D800 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ AC01 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 1F1E6 ÷ 0020 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 0020 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 1F1E6 ÷ 000D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 000D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 1F1E6 ÷ 000A ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 000A ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 1F1E6 ÷ 0001 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 0001 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 1F1E6 × 0300 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 1F1E6 × 0308 × 0300 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 1F1E6 ÷ 0600 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 0600 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 1F1E6 × 0903 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 1F1E6 × 0308 × 0903 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 1F1E6 ÷ 1100 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 1100 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 1F1E6 ÷ 1160 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 1160 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 1F1E6 ÷ 11A8 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 11A8 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 1F1E6 ÷ AC00 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ AC00 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 1F1E6 ÷ AC01 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ AC01 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 1F1E6 × 1F1E6 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [12.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 1F1E6 ÷ 261D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 261D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 1F1E6 ÷ 1F3FB ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 1F1E6 × 200D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1F1E6 × 0308 × 200D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1F1E6 ÷ 2640 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 2640 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 1F1E6 ÷ 1F466 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 1F466 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 1F1E6 ÷ 0378 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 0378 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 1F1E6 ÷ D800 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ D800 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 261D ÷ 0020 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 261D × 0308 ÷ 0020 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 261D ÷ 000D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 261D × 0308 ÷ 000D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 261D ÷ 000A ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 261D × 0308 ÷ 000A ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 261D ÷ 0001 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 261D × 0308 ÷ 0001 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 261D × 0300 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 261D × 0308 × 0300 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 261D ÷ 0600 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 261D × 0308 ÷ 0600 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 261D × 0903 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 261D × 0308 × 0903 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 261D ÷ 1100 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 261D × 0308 ÷ 1100 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 261D ÷ 1160 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 261D × 0308 ÷ 1160 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 261D ÷ 11A8 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 261D × 0308 ÷ 11A8 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 261D ÷ AC00 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 261D × 0308 ÷ AC00 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 261D ÷ AC01 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 261D × 0308 ÷ AC01 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 261D ÷ 1F1E6 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 261D × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 261D ÷ 261D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 261D × 0308 ÷ 261D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 261D × 1F3FB ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 261D × 0308 × 1F3FB ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 261D × 200D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 261D × 0308 × 200D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 261D ÷ 2640 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 261D × 0308 ÷ 2640 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 261D ÷ 1F466 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 261D × 0308 ÷ 1F466 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 261D ÷ 0378 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 261D × 0308 ÷ 0378 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 261D ÷ D800 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 261D × 0308 ÷ D800 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 1F3FB ÷ 0020 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 0020 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 1F3FB ÷ 000D ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 000D ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 1F3FB ÷ 000A ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 000A ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 1F3FB ÷ 0001 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 0001 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 1F3FB × 0300 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 1F3FB × 0308 × 0300 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 1F3FB ÷ 0600 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 0600 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 1F3FB × 0903 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 1F3FB × 0308 × 0903 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 1F3FB ÷ 1100 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 1100 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 1F3FB ÷ 1160 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 1160 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 1F3FB ÷ 11A8 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 11A8 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 1F3FB ÷ AC00 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 1F3FB × 0308 ÷ AC00 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 1F3FB ÷ AC01 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 1F3FB × 0308 ÷ AC01 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 1F3FB ÷ 1F1E6 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 1F3FB ÷ 261D ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 261D ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 1F3FB ÷ 1F3FB ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 1F3FB ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 1F3FB × 200D ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1F3FB × 0308 × 200D ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1F3FB ÷ 2640 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 2640 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 1F3FB ÷ 1F466 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 1F466 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 1F3FB ÷ 0378 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 0378 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 1F3FB ÷ D800 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 1F3FB × 0308 ÷ D800 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 200D ÷ 0020 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 200D × 0308 ÷ 0020 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 200D ÷ 000D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 200D × 0308 ÷ 000D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 200D ÷ 000A ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 200D × 0308 ÷ 000A ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 200D ÷ 0001 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 200D × 0308 ÷ 0001 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 200D × 0300 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 200D × 0308 × 0300 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 200D ÷ 0600 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 200D × 0308 ÷ 0600 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 200D × 0903 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 200D × 0308 × 0903 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 200D ÷ 1100 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 200D × 0308 ÷ 1100 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 200D ÷ 1160 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 200D × 0308 ÷ 1160 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 200D ÷ 11A8 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 200D × 0308 ÷ 11A8 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 200D ÷ AC00 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 200D × 0308 ÷ AC00 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 200D ÷ AC01 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 200D × 0308 ÷ AC01 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 200D ÷ 1F1E6 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 200D × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 200D ÷ 261D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 200D × 0308 ÷ 261D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 200D ÷ 1F3FB ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 200D × 0308 ÷ 1F3FB ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 200D × 200D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 200D × 0308 × 200D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 200D × 2640 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 200D × 0308 ÷ 2640 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 200D × 1F466 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] BOY (EBG) ÷ [0.3] -÷ 200D × 0308 ÷ 1F466 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 200D ÷ 0378 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 200D × 0308 ÷ 0378 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 200D ÷ D800 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 200D × 0308 ÷ D800 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 2640 ÷ 0020 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 2640 × 0308 ÷ 0020 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 2640 ÷ 000D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 2640 × 0308 ÷ 000D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 2640 ÷ 000A ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 2640 × 0308 ÷ 000A ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 2640 ÷ 0001 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 2640 × 0308 ÷ 0001 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 2640 × 0300 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 2640 × 0308 × 0300 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 2640 ÷ 0600 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 2640 × 0308 ÷ 0600 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 2640 × 0903 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 2640 × 0308 × 0903 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 2640 ÷ 1100 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 2640 × 0308 ÷ 1100 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 2640 ÷ 1160 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 2640 × 0308 ÷ 1160 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 2640 ÷ 11A8 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 2640 × 0308 ÷ 11A8 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 2640 ÷ AC00 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 2640 × 0308 ÷ AC00 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 2640 ÷ AC01 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 2640 × 0308 ÷ AC01 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 2640 ÷ 1F1E6 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 2640 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 2640 ÷ 261D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 2640 × 0308 ÷ 261D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 2640 ÷ 1F3FB ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 2640 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 2640 × 200D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 2640 × 0308 × 200D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 2640 ÷ 2640 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 2640 × 0308 ÷ 2640 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 2640 ÷ 1F466 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 2640 × 0308 ÷ 1F466 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 2640 ÷ 0378 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 2640 × 0308 ÷ 0378 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 2640 ÷ D800 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 2640 × 0308 ÷ D800 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 1F466 ÷ 0020 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 1F466 × 0308 ÷ 0020 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 1F466 ÷ 000D ÷ # ÷ [0.2] BOY (EBG) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 1F466 × 0308 ÷ 000D ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 1F466 ÷ 000A ÷ # ÷ [0.2] BOY (EBG) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 1F466 × 0308 ÷ 000A ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 1F466 ÷ 0001 ÷ # ÷ [0.2] BOY (EBG) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 1F466 × 0308 ÷ 0001 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 1F466 × 0300 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 1F466 × 0308 × 0300 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 1F466 ÷ 0600 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 1F466 × 0308 ÷ 0600 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 1F466 × 0903 ÷ # ÷ [0.2] BOY (EBG) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 1F466 × 0308 × 0903 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 1F466 ÷ 1100 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 1F466 × 0308 ÷ 1100 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 1F466 ÷ 1160 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 1F466 × 0308 ÷ 1160 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 1F466 ÷ 11A8 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 1F466 × 0308 ÷ 11A8 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 1F466 ÷ AC00 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 1F466 × 0308 ÷ AC00 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 1F466 ÷ AC01 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 1F466 × 0308 ÷ AC01 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 1F466 ÷ 1F1E6 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 1F466 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 1F466 ÷ 261D ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 1F466 × 0308 ÷ 261D ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 1F466 × 1F3FB ÷ # ÷ [0.2] BOY (EBG) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 1F466 × 0308 × 1F3FB ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 1F466 × 200D ÷ # ÷ [0.2] BOY (EBG) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1F466 × 0308 × 200D ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1F466 ÷ 2640 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 1F466 × 0308 ÷ 2640 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 1F466 ÷ 1F466 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 1F466 × 0308 ÷ 1F466 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 1F466 ÷ 0378 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 1F466 × 0308 ÷ 0378 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 1F466 ÷ D800 ÷ # ÷ [0.2] BOY (EBG) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 1F466 × 0308 ÷ D800 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ AC01 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 231A ÷ 0020 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 231A × 0308 ÷ 0020 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 231A ÷ 000D ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 231A × 0308 ÷ 000D ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 231A ÷ 000A ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 231A × 0308 ÷ 000A ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 231A ÷ 0001 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 231A × 0308 ÷ 0001 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 231A × 034F ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 231A × 0308 × 034F ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 231A ÷ 1F1E6 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 231A × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 231A ÷ 0600 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 231A × 0308 ÷ 0600 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 231A × 0903 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 231A × 0308 × 0903 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 231A ÷ 1100 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 231A × 0308 ÷ 1100 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 231A ÷ 1160 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 231A × 0308 ÷ 1160 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 231A ÷ 11A8 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 231A × 0308 ÷ 11A8 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 231A ÷ AC00 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 231A × 0308 ÷ AC00 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 231A ÷ AC01 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 231A × 0308 ÷ AC01 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 231A ÷ 231A ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 231A × 0308 ÷ 231A ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 231A × 0300 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 231A × 0308 × 0300 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 231A × 200D ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 231A × 0308 × 200D ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 231A ÷ 0378 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 231A × 0308 ÷ 0378 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 231A ÷ D800 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 231A × 0308 ÷ D800 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 0300 ÷ 0020 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 0300 × 0308 ÷ 0020 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 0300 ÷ 000D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 0300 × 0308 ÷ 000D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 0300 ÷ 000A ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 0300 × 0308 ÷ 000A ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 0300 ÷ 0001 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 0300 × 0308 ÷ 0001 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 0300 × 034F ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0300 × 0308 × 034F ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0300 ÷ 1F1E6 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 0300 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 0300 ÷ 0600 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 0300 × 0308 ÷ 0600 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 0300 × 0903 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 0300 × 0308 × 0903 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 0300 ÷ 1100 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 0300 × 0308 ÷ 1100 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 0300 ÷ 1160 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 0300 × 0308 ÷ 1160 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 0300 ÷ 11A8 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 0300 × 0308 ÷ 11A8 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 0300 ÷ AC00 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 0300 × 0308 ÷ AC00 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 0300 ÷ AC01 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 0300 × 0308 ÷ AC01 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 0300 ÷ 231A ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 0300 × 0308 ÷ 231A ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 0300 × 0300 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0300 × 0308 × 0300 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0300 × 200D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 0300 × 0308 × 200D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 0300 ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 0300 × 0308 ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 0300 ÷ D800 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 0300 × 0308 ÷ D800 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 200D ÷ 0020 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 200D × 0308 ÷ 0020 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 200D ÷ 000D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 200D × 0308 ÷ 000D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 200D ÷ 000A ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 200D × 0308 ÷ 000A ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 200D ÷ 0001 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 200D × 0308 ÷ 0001 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 200D × 034F ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 200D × 0308 × 034F ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 200D ÷ 1F1E6 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 200D × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 200D ÷ 0600 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 200D × 0308 ÷ 0600 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 200D × 0903 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 200D × 0308 × 0903 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 200D ÷ 1100 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 200D × 0308 ÷ 1100 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 200D ÷ 1160 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 200D × 0308 ÷ 1160 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 200D ÷ 11A8 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 200D × 0308 ÷ 11A8 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 200D ÷ AC00 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 200D × 0308 ÷ AC00 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 200D ÷ AC01 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 200D × 0308 ÷ AC01 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 200D ÷ 231A ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 200D × 0308 ÷ 231A ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 200D × 0300 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 200D × 0308 × 0300 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 200D × 200D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 200D × 0308 × 200D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 200D ÷ 0378 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 200D × 0308 ÷ 0378 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 200D ÷ D800 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 200D × 0308 ÷ D800 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 0378 ÷ 0020 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 0378 × 0308 ÷ 0020 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 0378 × 0308 ÷ 0020 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0378 ÷ 000D ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 0378 × 0308 ÷ 000D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 0378 × 0308 ÷ 000D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ 0378 ÷ 000A ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 0378 × 0308 ÷ 000A ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 0378 × 0308 ÷ 000A ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ 0378 ÷ 0001 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0378 × 0308 ÷ 0001 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 0378 × 0300 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 0378 × 0308 × 0300 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 0378 × 0308 ÷ 0001 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 0378 × 034F ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0378 × 0308 × 034F ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ 0378 ÷ 1F1E6 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 0378 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ 0378 ÷ 0600 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 0378 × 0308 ÷ 0600 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 0378 × 0308 ÷ 0600 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ 0378 × 0903 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 0378 × 0308 × 0903 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 0378 × 0308 × 0903 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ 0378 ÷ 1100 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 0378 × 0308 ÷ 1100 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 0378 × 0308 ÷ 1100 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ 0378 ÷ 1160 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 0378 × 0308 ÷ 1160 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 0378 × 0308 ÷ 1160 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ 0378 ÷ 11A8 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 0378 × 0308 ÷ 11A8 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 0378 × 0308 ÷ 11A8 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ 0378 ÷ AC00 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 0378 × 0308 ÷ AC00 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 0378 × 0308 ÷ AC00 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ 0378 ÷ AC01 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0378 × 0308 ÷ AC01 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 0378 ÷ 1F1E6 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0378 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 0378 ÷ 261D ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0378 × 0308 ÷ 261D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 0378 ÷ 1F3FB ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0378 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 0378 × 200D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0378 × 0308 × 200D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0378 ÷ 2640 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0378 × 0308 ÷ 2640 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 0378 ÷ 1F466 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 0378 × 0308 ÷ 1F466 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 0378 × 0308 ÷ AC01 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 0378 ÷ 231A ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 0378 × 0308 ÷ 231A ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ 0378 × 0300 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0378 × 0308 × 0300 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ 0378 × 200D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 0378 × 0308 × 200D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 0378 ÷ 0378 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0378 × 0308 ÷ 0378 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 0378 × 0308 ÷ 0378 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 0378 ÷ D800 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0378 × 0308 ÷ D800 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 0378 × 0308 ÷ D800 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ D800 ÷ 0020 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] SPACE (Other) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ D800 ÷ 000D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 000D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 000D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] ÷ D800 ÷ 000A ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 000A ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 000A ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] ÷ D800 ÷ 0001 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <START OF HEADING> (Control) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 0001 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ D800 ÷ 0300 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ D800 ÷ 0308 × 0300 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 0001 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ D800 ÷ 034F ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ D800 ÷ 0308 × 034F ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] +÷ D800 ÷ 1F1E6 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 1F1E6 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] ÷ D800 ÷ 0600 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 0600 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 0600 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] ÷ D800 ÷ 0903 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ D800 ÷ 0308 × 0903 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ D800 ÷ 0308 × 0903 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] ÷ D800 ÷ 1100 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 1100 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 1100 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ D800 ÷ 1160 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 1160 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 1160 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] ÷ D800 ÷ 11A8 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 11A8 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 11A8 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] ÷ D800 ÷ AC00 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ D800 ÷ 0308 ÷ AC00 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ D800 ÷ 0308 ÷ AC00 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] ÷ D800 ÷ AC01 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ D800 ÷ 0308 ÷ AC01 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ D800 ÷ 1F1E6 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 1F1E6 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ D800 ÷ 261D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 261D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ D800 ÷ 1F3FB ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ D800 ÷ 200D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ D800 ÷ 0308 × 200D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ D800 ÷ 2640 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ D800 ÷ 1F466 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] BOY (EBG) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ D800 ÷ 0308 ÷ AC01 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ D800 ÷ 231A ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] WATCH (ExtPict) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 231A ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] +÷ D800 ÷ 0300 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ D800 ÷ 0308 × 0300 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] +÷ D800 ÷ 200D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ D800 ÷ 0308 × 200D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ D800 ÷ 0378 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ D800 ÷ D800 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <surrogate-D800> (Control) ÷ [0.3] -÷ D800 ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 000D × 000A ÷ 0061 ÷ 000A ÷ 0308 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) × [3.0] <LINE FEED (LF)> (LF) ÷ [4.0] LATIN SMALL LETTER A (Other) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [0.3] -÷ 0061 × 0308 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [0.3] -÷ 0020 × 200D ÷ 0646 ÷ # ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] ARABIC LETTER NOON (Other) ÷ [0.3] -÷ 0646 × 200D ÷ 0020 ÷ # ÷ [0.2] ARABIC LETTER NOON (Other) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ D800 ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 000D × 000A ÷ 0061 ÷ 000A ÷ 0308 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) × [3.0] <LINE FEED (LF)> (LF) ÷ [4.0] LATIN SMALL LETTER A (Other) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [0.3] +÷ 0061 × 0308 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [0.3] +÷ 0020 × 200D ÷ 0646 ÷ # ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] ARABIC LETTER NOON (Other) ÷ [0.3] +÷ 0646 × 200D ÷ 0020 ÷ # ÷ [0.2] ARABIC LETTER NOON (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 1100 × 1100 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ AC00 × 11A8 ÷ 1100 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [7.0] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ AC01 × 11A8 ÷ 1100 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [8.0] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] ÷ 1F1E6 × 1F1E7 ÷ 1F1E8 ÷ 0062 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [12.0] REGIONAL INDICATOR SYMBOL LETTER B (RI) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER C (RI) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] ÷ 0061 ÷ 1F1E6 × 1F1E7 ÷ 1F1E8 ÷ 0062 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [13.0] REGIONAL INDICATOR SYMBOL LETTER B (RI) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER C (RI) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] -÷ 0061 ÷ 1F1E6 × 1F1E7 × 200D ÷ 1F1E8 ÷ 0062 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [13.0] REGIONAL INDICATOR SYMBOL LETTER B (RI) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER C (RI) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] -÷ 0061 ÷ 1F1E6 × 200D ÷ 1F1E7 × 1F1E8 ÷ 0062 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER B (RI) × [13.0] REGIONAL INDICATOR SYMBOL LETTER C (RI) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] +÷ 0061 ÷ 1F1E6 × 1F1E7 × 200D ÷ 1F1E8 ÷ 0062 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [13.0] REGIONAL INDICATOR SYMBOL LETTER B (RI) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER C (RI) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] +÷ 0061 ÷ 1F1E6 × 200D ÷ 1F1E7 × 1F1E8 ÷ 0062 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER B (RI) × [13.0] REGIONAL INDICATOR SYMBOL LETTER C (RI) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] ÷ 0061 ÷ 1F1E6 × 1F1E7 ÷ 1F1E8 × 1F1E9 ÷ 0062 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [13.0] REGIONAL INDICATOR SYMBOL LETTER B (RI) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER C (RI) × [13.0] REGIONAL INDICATOR SYMBOL LETTER D (RI) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] -÷ 0061 × 200D ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0061 × 0308 ÷ 0062 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] +÷ 0061 × 200D ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] +÷ 0061 × 0308 ÷ 0062 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] ÷ 0061 × 0903 ÷ 0062 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3] ÷ 0061 ÷ 0600 × 0062 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) × [9.2] LATIN SMALL LETTER B (Other) ÷ [0.3] -÷ 261D × 1F3FB ÷ 261D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 1F466 × 1F3FB ÷ # ÷ [0.2] BOY (EBG) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 200D × 1F466 × 1F3FB ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] BOY (EBG) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 200D × 2640 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] -÷ 200D × 1F466 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] BOY (EBG) ÷ [0.3] -÷ 1F466 ÷ 1F466 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 1F476 × 1F3FF ÷ 1F476 ÷ # ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) ÷ [0.3] +÷ 0061 × 1F3FF ÷ 1F476 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) ÷ [0.3] +÷ 0061 × 1F3FF ÷ 1F476 × 200D × 1F6D1 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [999.0] BABY (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3] +÷ 1F476 × 1F3FF × 0308 × 200D × 1F476 × 1F3FF ÷ # ÷ [0.2] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] BABY (ExtPict) × [9.0] EMOJI MODIFIER FITZPATRICK TYPE-6 (Extend) ÷ [0.3] +÷ 1F6D1 × 200D × 1F6D1 ÷ # ÷ [0.2] OCTAGONAL SIGN (ExtPict) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3] +÷ 0061 × 200D ÷ 1F6D1 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] OCTAGONAL SIGN (ExtPict) ÷ [0.3] +÷ 2701 × 200D × 2701 ÷ # ÷ [0.2] UPPER BLADE SCISSORS (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] UPPER BLADE SCISSORS (Other) ÷ [0.3] +÷ 0061 × 200D ÷ 2701 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] UPPER BLADE SCISSORS (Other) ÷ [0.3] # -# Lines: 822 +# Lines: 672 # # EOF diff --git a/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt index 6715446aba..0e9e678a85 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt @@ -1,6 +1,6 @@ -# LineBreakTest-10.0.0.txt -# Date: 2017-04-14, 05:40:30 GMT -# © 2017 Unicode®, Inc. +# LineBreakTest-11.0.0.txt +# Date: 2018-05-20, 09:03:09 GMT +# © 2018 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -6242,174 +6242,174 @@ × 0001 × 0020 ÷ 3041 ÷ # × [0.3] <START OF HEADING> (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] × 0001 × 0308 × 3041 ÷ # × [0.3] <START OF HEADING> (CM1_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [21.03] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] × 0001 × 0308 × 0020 ÷ 3041 ÷ # × [0.3] <START OF HEADING> (CM1_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] -× 200D × 0023 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [28.0] NUMBER SIGN (AL) ÷ [0.3] +× 200D × 0023 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] NUMBER SIGN (AL) ÷ [0.3] × 200D × 0020 ÷ 0023 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] NUMBER SIGN (AL) ÷ [0.3] -× 200D × 0308 × 0023 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [28.0] NUMBER SIGN (AL) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 0023 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] NUMBER SIGN (AL) ÷ [0.3] -× 200D ÷ 2014 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [999.0] EM DASH (B2) ÷ [0.3] +× 200D × 0308 × 0023 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [28.0] NUMBER SIGN (AL) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 0023 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] NUMBER SIGN (AL) ÷ [0.3] +× 200D × 2014 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] EM DASH (B2) ÷ [0.3] × 200D × 0020 ÷ 2014 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] EM DASH (B2) ÷ [0.3] -× 200D × 0308 ÷ 2014 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] EM DASH (B2) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 2014 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] EM DASH (B2) ÷ [0.3] -× 200D × 0009 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [21.01] <CHARACTER TABULATION> (BA) ÷ [0.3] +× 200D × 0308 ÷ 2014 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] EM DASH (B2) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 2014 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] EM DASH (B2) ÷ [0.3] +× 200D × 0009 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] <CHARACTER TABULATION> (BA) ÷ [0.3] × 200D × 0020 ÷ 0009 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] <CHARACTER TABULATION> (BA) ÷ [0.3] -× 200D × 0308 × 0009 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [21.01] <CHARACTER TABULATION> (BA) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 0009 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] <CHARACTER TABULATION> (BA) ÷ [0.3] -× 200D ÷ 00B4 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [999.0] ACUTE ACCENT (BB) ÷ [0.3] +× 200D × 0308 × 0009 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [21.01] <CHARACTER TABULATION> (BA) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 0009 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] <CHARACTER TABULATION> (BA) ÷ [0.3] +× 200D × 00B4 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] ACUTE ACCENT (BB) ÷ [0.3] × 200D × 0020 ÷ 00B4 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] ACUTE ACCENT (BB) ÷ [0.3] -× 200D × 0308 ÷ 00B4 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] ACUTE ACCENT (BB) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 00B4 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] ACUTE ACCENT (BB) ÷ [0.3] +× 200D × 0308 ÷ 00B4 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] ACUTE ACCENT (BB) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 00B4 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] ACUTE ACCENT (BB) ÷ [0.3] × 200D × 000B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [6.0] <LINE TABULATION> (BK) ÷ [0.3] × 200D × 0020 × 000B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [6.0] <LINE TABULATION> (BK) ÷ [0.3] -× 200D × 0308 × 000B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [6.0] <LINE TABULATION> (BK) ÷ [0.3] -× 200D × 0308 × 0020 × 000B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [6.0] <LINE TABULATION> (BK) ÷ [0.3] -× 200D ÷ FFFC ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [20.01] OBJECT REPLACEMENT CHARACTER (CB) ÷ [0.3] +× 200D × 0308 × 000B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [6.0] <LINE TABULATION> (BK) ÷ [0.3] +× 200D × 0308 × 0020 × 000B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [6.0] <LINE TABULATION> (BK) ÷ [0.3] +× 200D × FFFC ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] OBJECT REPLACEMENT CHARACTER (CB) ÷ [0.3] × 200D × 0020 ÷ FFFC ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] OBJECT REPLACEMENT CHARACTER (CB) ÷ [0.3] -× 200D × 0308 ÷ FFFC ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [20.01] OBJECT REPLACEMENT CHARACTER (CB) ÷ [0.3] -× 200D × 0308 × 0020 ÷ FFFC ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] OBJECT REPLACEMENT CHARACTER (CB) ÷ [0.3] -× 200D × 007D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [13.04] RIGHT CURLY BRACKET (CL) ÷ [0.3] +× 200D × 0308 ÷ FFFC ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [20.01] OBJECT REPLACEMENT CHARACTER (CB) ÷ [0.3] +× 200D × 0308 × 0020 ÷ FFFC ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] OBJECT REPLACEMENT CHARACTER (CB) ÷ [0.3] +× 200D × 007D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] RIGHT CURLY BRACKET (CL) ÷ [0.3] × 200D × 0020 × 007D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [13.02] RIGHT CURLY BRACKET (CL) ÷ [0.3] -× 200D × 0308 × 007D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [13.04] RIGHT CURLY BRACKET (CL) ÷ [0.3] -× 200D × 0308 × 0020 × 007D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [13.02] RIGHT CURLY BRACKET (CL) ÷ [0.3] -× 200D × 0029 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [13.04] RIGHT PARENTHESIS (CP) ÷ [0.3] +× 200D × 0308 × 007D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [13.04] RIGHT CURLY BRACKET (CL) ÷ [0.3] +× 200D × 0308 × 0020 × 007D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [13.02] RIGHT CURLY BRACKET (CL) ÷ [0.3] +× 200D × 0029 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] RIGHT PARENTHESIS (CP) ÷ [0.3] × 200D × 0020 × 0029 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [13.02] RIGHT PARENTHESIS (CP) ÷ [0.3] -× 200D × 0308 × 0029 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [13.04] RIGHT PARENTHESIS (CP) ÷ [0.3] -× 200D × 0308 × 0020 × 0029 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [13.02] RIGHT PARENTHESIS (CP) ÷ [0.3] +× 200D × 0308 × 0029 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [13.04] RIGHT PARENTHESIS (CP) ÷ [0.3] +× 200D × 0308 × 0020 × 0029 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [13.02] RIGHT PARENTHESIS (CP) ÷ [0.3] × 200D × 000D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [6.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] × 200D × 0020 × 000D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [6.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -× 200D × 0308 × 000D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [6.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -× 200D × 0308 × 0020 × 000D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [6.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -× 200D × 0021 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [13.01] EXCLAMATION MARK (EX) ÷ [0.3] +× 200D × 0308 × 000D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [6.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +× 200D × 0308 × 0020 × 000D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [6.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +× 200D × 0021 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] EXCLAMATION MARK (EX) ÷ [0.3] × 200D × 0020 × 0021 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [13.01] EXCLAMATION MARK (EX) ÷ [0.3] -× 200D × 0308 × 0021 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [13.01] EXCLAMATION MARK (EX) ÷ [0.3] -× 200D × 0308 × 0020 × 0021 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [13.01] EXCLAMATION MARK (EX) ÷ [0.3] -× 200D × 00A0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [12.3] NO-BREAK SPACE (GL) ÷ [0.3] +× 200D × 0308 × 0021 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [13.01] EXCLAMATION MARK (EX) ÷ [0.3] +× 200D × 0308 × 0020 × 0021 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [13.01] EXCLAMATION MARK (EX) ÷ [0.3] +× 200D × 00A0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] NO-BREAK SPACE (GL) ÷ [0.3] × 200D × 0020 ÷ 00A0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] NO-BREAK SPACE (GL) ÷ [0.3] -× 200D × 0308 × 00A0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [12.3] NO-BREAK SPACE (GL) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 00A0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] NO-BREAK SPACE (GL) ÷ [0.3] -× 200D ÷ AC00 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [999.0] HANGUL SYLLABLE GA (H2) ÷ [0.3] +× 200D × 0308 × 00A0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [12.3] NO-BREAK SPACE (GL) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 00A0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] NO-BREAK SPACE (GL) ÷ [0.3] +× 200D × AC00 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] HANGUL SYLLABLE GA (H2) ÷ [0.3] × 200D × 0020 ÷ AC00 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL SYLLABLE GA (H2) ÷ [0.3] -× 200D × 0308 ÷ AC00 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] HANGUL SYLLABLE GA (H2) ÷ [0.3] -× 200D × 0308 × 0020 ÷ AC00 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL SYLLABLE GA (H2) ÷ [0.3] -× 200D ÷ AC01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [999.0] HANGUL SYLLABLE GAG (H3) ÷ [0.3] +× 200D × 0308 ÷ AC00 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] HANGUL SYLLABLE GA (H2) ÷ [0.3] +× 200D × 0308 × 0020 ÷ AC00 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL SYLLABLE GA (H2) ÷ [0.3] +× 200D × AC01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] HANGUL SYLLABLE GAG (H3) ÷ [0.3] × 200D × 0020 ÷ AC01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL SYLLABLE GAG (H3) ÷ [0.3] -× 200D × 0308 ÷ AC01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] HANGUL SYLLABLE GAG (H3) ÷ [0.3] -× 200D × 0308 × 0020 ÷ AC01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL SYLLABLE GAG (H3) ÷ [0.3] -× 200D × 05D0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [28.0] HEBREW LETTER ALEF (HL) ÷ [0.3] +× 200D × 0308 ÷ AC01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] HANGUL SYLLABLE GAG (H3) ÷ [0.3] +× 200D × 0308 × 0020 ÷ AC01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL SYLLABLE GAG (H3) ÷ [0.3] +× 200D × 05D0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] HEBREW LETTER ALEF (HL) ÷ [0.3] × 200D × 0020 ÷ 05D0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] HEBREW LETTER ALEF (HL) ÷ [0.3] -× 200D × 0308 × 05D0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [28.0] HEBREW LETTER ALEF (HL) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 05D0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HEBREW LETTER ALEF (HL) ÷ [0.3] -× 200D × 002D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [21.02] HYPHEN-MINUS (HY) ÷ [0.3] +× 200D × 0308 × 05D0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [28.0] HEBREW LETTER ALEF (HL) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 05D0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HEBREW LETTER ALEF (HL) ÷ [0.3] +× 200D × 002D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] HYPHEN-MINUS (HY) ÷ [0.3] × 200D × 0020 ÷ 002D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] HYPHEN-MINUS (HY) ÷ [0.3] -× 200D × 0308 × 002D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [21.02] HYPHEN-MINUS (HY) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 002D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HYPHEN-MINUS (HY) ÷ [0.3] +× 200D × 0308 × 002D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [21.02] HYPHEN-MINUS (HY) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 002D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HYPHEN-MINUS (HY) ÷ [0.3] × 200D × 231A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] WATCH (ID) ÷ [0.3] × 200D × 0020 ÷ 231A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] WATCH (ID) ÷ [0.3] -× 200D × 0308 ÷ 231A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] WATCH (ID) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 231A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] WATCH (ID) ÷ [0.3] -× 200D × 2024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [22.01] ONE DOT LEADER (IN) ÷ [0.3] +× 200D × 0308 ÷ 231A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] WATCH (ID) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 231A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] WATCH (ID) ÷ [0.3] +× 200D × 2024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] ONE DOT LEADER (IN) ÷ [0.3] × 200D × 0020 ÷ 2024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] ONE DOT LEADER (IN) ÷ [0.3] -× 200D × 0308 × 2024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [22.01] ONE DOT LEADER (IN) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 2024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] ONE DOT LEADER (IN) ÷ [0.3] -× 200D × 002C ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [13.04] COMMA (IS) ÷ [0.3] +× 200D × 0308 × 2024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [22.01] ONE DOT LEADER (IN) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 2024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] ONE DOT LEADER (IN) ÷ [0.3] +× 200D × 002C ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMMA (IS) ÷ [0.3] × 200D × 0020 × 002C ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [13.02] COMMA (IS) ÷ [0.3] -× 200D × 0308 × 002C ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [13.04] COMMA (IS) ÷ [0.3] -× 200D × 0308 × 0020 × 002C ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [13.02] COMMA (IS) ÷ [0.3] -× 200D ÷ 1100 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [999.0] HANGUL CHOSEONG KIYEOK (JL) ÷ [0.3] +× 200D × 0308 × 002C ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [13.04] COMMA (IS) ÷ [0.3] +× 200D × 0308 × 0020 × 002C ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [13.02] COMMA (IS) ÷ [0.3] +× 200D × 1100 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] HANGUL CHOSEONG KIYEOK (JL) ÷ [0.3] × 200D × 0020 ÷ 1100 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL CHOSEONG KIYEOK (JL) ÷ [0.3] -× 200D × 0308 ÷ 1100 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] HANGUL CHOSEONG KIYEOK (JL) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 1100 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL CHOSEONG KIYEOK (JL) ÷ [0.3] -× 200D ÷ 11A8 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [999.0] HANGUL JONGSEONG KIYEOK (JT) ÷ [0.3] +× 200D × 0308 ÷ 1100 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] HANGUL CHOSEONG KIYEOK (JL) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 1100 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL CHOSEONG KIYEOK (JL) ÷ [0.3] +× 200D × 11A8 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] HANGUL JONGSEONG KIYEOK (JT) ÷ [0.3] × 200D × 0020 ÷ 11A8 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL JONGSEONG KIYEOK (JT) ÷ [0.3] -× 200D × 0308 ÷ 11A8 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] HANGUL JONGSEONG KIYEOK (JT) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 11A8 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL JONGSEONG KIYEOK (JT) ÷ [0.3] -× 200D ÷ 1160 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [999.0] HANGUL JUNGSEONG FILLER (JV) ÷ [0.3] +× 200D × 0308 ÷ 11A8 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] HANGUL JONGSEONG KIYEOK (JT) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 11A8 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL JONGSEONG KIYEOK (JT) ÷ [0.3] +× 200D × 1160 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] HANGUL JUNGSEONG FILLER (JV) ÷ [0.3] × 200D × 0020 ÷ 1160 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL JUNGSEONG FILLER (JV) ÷ [0.3] -× 200D × 0308 ÷ 1160 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] HANGUL JUNGSEONG FILLER (JV) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 1160 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL JUNGSEONG FILLER (JV) ÷ [0.3] +× 200D × 0308 ÷ 1160 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] HANGUL JUNGSEONG FILLER (JV) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 1160 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HANGUL JUNGSEONG FILLER (JV) ÷ [0.3] × 200D × 000A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [6.0] <LINE FEED (LF)> (LF) ÷ [0.3] × 200D × 0020 × 000A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [6.0] <LINE FEED (LF)> (LF) ÷ [0.3] -× 200D × 0308 × 000A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [6.0] <LINE FEED (LF)> (LF) ÷ [0.3] -× 200D × 0308 × 0020 × 000A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [6.0] <LINE FEED (LF)> (LF) ÷ [0.3] +× 200D × 0308 × 000A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [6.0] <LINE FEED (LF)> (LF) ÷ [0.3] +× 200D × 0308 × 0020 × 000A ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [6.0] <LINE FEED (LF)> (LF) ÷ [0.3] × 200D × 0085 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [6.0] <NEXT LINE (NEL)> (NL) ÷ [0.3] × 200D × 0020 × 0085 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [6.0] <NEXT LINE (NEL)> (NL) ÷ [0.3] -× 200D × 0308 × 0085 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [6.0] <NEXT LINE (NEL)> (NL) ÷ [0.3] -× 200D × 0308 × 0020 × 0085 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [6.0] <NEXT LINE (NEL)> (NL) ÷ [0.3] -× 200D × 17D6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [21.03] KHMER SIGN CAMNUC PII KUUH (NS) ÷ [0.3] +× 200D × 0308 × 0085 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [6.0] <NEXT LINE (NEL)> (NL) ÷ [0.3] +× 200D × 0308 × 0020 × 0085 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [6.0] <NEXT LINE (NEL)> (NL) ÷ [0.3] +× 200D × 17D6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] KHMER SIGN CAMNUC PII KUUH (NS) ÷ [0.3] × 200D × 0020 ÷ 17D6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] KHMER SIGN CAMNUC PII KUUH (NS) ÷ [0.3] -× 200D × 0308 × 17D6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [21.03] KHMER SIGN CAMNUC PII KUUH (NS) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 17D6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] KHMER SIGN CAMNUC PII KUUH (NS) ÷ [0.3] -× 200D × 0030 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [23.02] DIGIT ZERO (NU) ÷ [0.3] +× 200D × 0308 × 17D6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [21.03] KHMER SIGN CAMNUC PII KUUH (NS) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 17D6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] KHMER SIGN CAMNUC PII KUUH (NS) ÷ [0.3] +× 200D × 0030 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] DIGIT ZERO (NU) ÷ [0.3] × 200D × 0020 ÷ 0030 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] DIGIT ZERO (NU) ÷ [0.3] -× 200D × 0308 × 0030 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [23.02] DIGIT ZERO (NU) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 0030 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] DIGIT ZERO (NU) ÷ [0.3] -× 200D × 0028 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [30.01] LEFT PARENTHESIS (OP) ÷ [0.3] +× 200D × 0308 × 0030 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [23.02] DIGIT ZERO (NU) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 0030 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] DIGIT ZERO (NU) ÷ [0.3] +× 200D × 0028 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] LEFT PARENTHESIS (OP) ÷ [0.3] × 200D × 0020 ÷ 0028 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] LEFT PARENTHESIS (OP) ÷ [0.3] -× 200D × 0308 × 0028 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [30.01] LEFT PARENTHESIS (OP) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 0028 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] LEFT PARENTHESIS (OP) ÷ [0.3] -× 200D × 0025 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [24.03] PERCENT SIGN (PO) ÷ [0.3] +× 200D × 0308 × 0028 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [30.01] LEFT PARENTHESIS (OP) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 0028 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] LEFT PARENTHESIS (OP) ÷ [0.3] +× 200D × 0025 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] PERCENT SIGN (PO) ÷ [0.3] × 200D × 0020 ÷ 0025 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] PERCENT SIGN (PO) ÷ [0.3] -× 200D × 0308 × 0025 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [24.03] PERCENT SIGN (PO) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 0025 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] PERCENT SIGN (PO) ÷ [0.3] -× 200D × 0024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [24.03] DOLLAR SIGN (PR) ÷ [0.3] +× 200D × 0308 × 0025 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [24.03] PERCENT SIGN (PO) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 0025 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] PERCENT SIGN (PO) ÷ [0.3] +× 200D × 0024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] DOLLAR SIGN (PR) ÷ [0.3] × 200D × 0020 ÷ 0024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] DOLLAR SIGN (PR) ÷ [0.3] -× 200D × 0308 × 0024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [24.03] DOLLAR SIGN (PR) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 0024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] DOLLAR SIGN (PR) ÷ [0.3] -× 200D × 0022 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [19.01] QUOTATION MARK (QU) ÷ [0.3] +× 200D × 0308 × 0024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [24.03] DOLLAR SIGN (PR) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 0024 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] DOLLAR SIGN (PR) ÷ [0.3] +× 200D × 0022 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] QUOTATION MARK (QU) ÷ [0.3] × 200D × 0020 ÷ 0022 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] QUOTATION MARK (QU) ÷ [0.3] -× 200D × 0308 × 0022 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [19.01] QUOTATION MARK (QU) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 0022 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] QUOTATION MARK (QU) ÷ [0.3] +× 200D × 0308 × 0022 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [19.01] QUOTATION MARK (QU) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 0022 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] QUOTATION MARK (QU) ÷ [0.3] × 200D × 0020 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [0.3] × 200D × 0020 × 0020 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [7.01] SPACE (SP) ÷ [0.3] -× 200D × 0308 × 0020 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [0.3] -× 200D × 0308 × 0020 × 0020 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [7.01] SPACE (SP) ÷ [0.3] -× 200D × 002F ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [13.04] SOLIDUS (SY) ÷ [0.3] +× 200D × 0308 × 0020 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [0.3] +× 200D × 0308 × 0020 × 0020 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [7.01] SPACE (SP) ÷ [0.3] +× 200D × 002F ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] SOLIDUS (SY) ÷ [0.3] × 200D × 0020 × 002F ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [13.02] SOLIDUS (SY) ÷ [0.3] -× 200D × 0308 × 002F ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [13.04] SOLIDUS (SY) ÷ [0.3] -× 200D × 0308 × 0020 × 002F ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [13.02] SOLIDUS (SY) ÷ [0.3] -× 200D × 2060 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [11.01] WORD JOINER (WJ) ÷ [0.3] +× 200D × 0308 × 002F ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [13.04] SOLIDUS (SY) ÷ [0.3] +× 200D × 0308 × 0020 × 002F ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [13.02] SOLIDUS (SY) ÷ [0.3] +× 200D × 2060 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] WORD JOINER (WJ) ÷ [0.3] × 200D × 0020 × 2060 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [11.01] WORD JOINER (WJ) ÷ [0.3] -× 200D × 0308 × 2060 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [11.01] WORD JOINER (WJ) ÷ [0.3] -× 200D × 0308 × 0020 × 2060 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [11.01] WORD JOINER (WJ) ÷ [0.3] +× 200D × 0308 × 2060 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [11.01] WORD JOINER (WJ) ÷ [0.3] +× 200D × 0308 × 0020 × 2060 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [11.01] WORD JOINER (WJ) ÷ [0.3] × 200D × 200B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.02] ZERO WIDTH SPACE (ZW) ÷ [0.3] × 200D × 0020 × 200B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) × [7.02] ZERO WIDTH SPACE (ZW) ÷ [0.3] -× 200D × 0308 × 200B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.02] ZERO WIDTH SPACE (ZW) ÷ [0.3] -× 200D × 0308 × 0020 × 200B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [7.02] ZERO WIDTH SPACE (ZW) ÷ [0.3] -× 200D ÷ 1F1E6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +× 200D × 0308 × 200B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.02] ZERO WIDTH SPACE (ZW) ÷ [0.3] +× 200D × 0308 × 0020 × 200B ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) × [7.02] ZERO WIDTH SPACE (ZW) ÷ [0.3] +× 200D × 1F1E6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] × 200D × 0020 ÷ 1F1E6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -× 200D × 0308 ÷ 1F1E6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 1F1E6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +× 200D × 0308 ÷ 1F1E6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 1F1E6 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] × 200D × 261D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] WHITE UP POINTING INDEX (EB) ÷ [0.3] × 200D × 0020 ÷ 261D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] WHITE UP POINTING INDEX (EB) ÷ [0.3] -× 200D × 0308 ÷ 261D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] WHITE UP POINTING INDEX (EB) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 261D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] WHITE UP POINTING INDEX (EB) ÷ [0.3] +× 200D × 0308 ÷ 261D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] WHITE UP POINTING INDEX (EB) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 261D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] WHITE UP POINTING INDEX (EB) ÷ [0.3] × 200D × 1F3FB ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (EM) ÷ [0.3] × 200D × 0020 ÷ 1F3FB ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (EM) ÷ [0.3] -× 200D × 0308 ÷ 1F3FB ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (EM) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 1F3FB ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (EM) ÷ [0.3] -× 200D × 0001 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] <START OF HEADING> (CM1_CM) ÷ [0.3] +× 200D × 0308 ÷ 1F3FB ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (EM) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 1F3FB ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (EM) ÷ [0.3] +× 200D × 0001 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] <START OF HEADING> (CM1_CM) ÷ [0.3] × 200D × 0020 ÷ 0001 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] <START OF HEADING> (CM1_CM) ÷ [0.3] -× 200D × 0308 × 0001 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [9.0] <START OF HEADING> (CM1_CM) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 0001 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] <START OF HEADING> (CM1_CM) ÷ [0.3] -× 200D × 200D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [0.3] +× 200D × 0308 × 0001 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [9.0] <START OF HEADING> (CM1_CM) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 0001 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] <START OF HEADING> (CM1_CM) ÷ [0.3] +× 200D × 200D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [0.3] × 200D × 0020 ÷ 200D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [0.3] -× 200D × 0308 × 200D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [9.0] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 200D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [0.3] -× 200D × 00A7 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [28.0] SECTION SIGN (AI_AL) ÷ [0.3] +× 200D × 0308 × 200D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [9.0] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 200D ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) ÷ [0.3] +× 200D × 00A7 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] SECTION SIGN (AI_AL) ÷ [0.3] × 200D × 0020 ÷ 00A7 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] SECTION SIGN (AI_AL) ÷ [0.3] -× 200D × 0308 × 00A7 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [28.0] SECTION SIGN (AI_AL) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 00A7 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] SECTION SIGN (AI_AL) ÷ [0.3] -× 200D × 50005 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [28.0] <reserved-50005> (XX_AL) ÷ [0.3] +× 200D × 0308 × 00A7 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [28.0] SECTION SIGN (AI_AL) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 00A7 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] SECTION SIGN (AI_AL) ÷ [0.3] +× 200D × 50005 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] <reserved-50005> (XX_AL) ÷ [0.3] × 200D × 0020 ÷ 50005 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] <reserved-50005> (XX_AL) ÷ [0.3] -× 200D × 0308 × 50005 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [28.0] <reserved-50005> (XX_AL) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 50005 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] <reserved-50005> (XX_AL) ÷ [0.3] -× 200D × 0E01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [28.0] THAI CHARACTER KO KAI (SA_AL) ÷ [0.3] +× 200D × 0308 × 50005 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [28.0] <reserved-50005> (XX_AL) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 50005 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] <reserved-50005> (XX_AL) ÷ [0.3] +× 200D × 0E01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] THAI CHARACTER KO KAI (SA_AL) ÷ [0.3] × 200D × 0020 ÷ 0E01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] THAI CHARACTER KO KAI (SA_AL) ÷ [0.3] -× 200D × 0308 × 0E01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [28.0] THAI CHARACTER KO KAI (SA_AL) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 0E01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] THAI CHARACTER KO KAI (SA_AL) ÷ [0.3] -× 200D × 3041 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [21.03] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] +× 200D × 0308 × 0E01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [28.0] THAI CHARACTER KO KAI (SA_AL) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 0E01 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] THAI CHARACTER KO KAI (SA_AL) ÷ [0.3] +× 200D × 3041 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] × 200D × 0020 ÷ 3041 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [18.0] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] -× 200D × 0308 × 3041 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [21.03] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] -× 200D × 0308 × 0020 ÷ 3041 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] +× 200D × 0308 × 3041 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [21.03] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] +× 200D × 0308 × 0020 ÷ 3041 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] × 00A7 × 0023 ÷ # × [0.3] SECTION SIGN (AI_AL) × [28.0] NUMBER SIGN (AL) ÷ [0.3] × 00A7 × 0020 ÷ 0023 ÷ # × [0.3] SECTION SIGN (AI_AL) × [7.01] SPACE (SP) ÷ [18.0] NUMBER SIGN (AL) ÷ [0.3] × 00A7 × 0308 × 0023 ÷ # × [0.3] SECTION SIGN (AI_AL) × [9.0] COMBINING DIAERESIS (CM1_CM) × [28.0] NUMBER SIGN (AL) ÷ [0.3] @@ -7084,7 +7084,7 @@ × 3041 × 0308 × 0020 ÷ 3041 ÷ # × [0.3] HIRAGANA LETTER SMALL A (CJ_NS) × [9.0] COMBINING DIAERESIS (CM1_CM) × [7.01] SPACE (SP) ÷ [18.0] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] × 000D × 000A ÷ 0061 × 000A ÷ 0308 ÷ # × [0.3] <CARRIAGE RETURN (CR)> (CR) × [5.01] <LINE FEED (LF)> (LF) ÷ [5.03] LATIN SMALL LETTER A (AL) × [6.0] <LINE FEED (LF)> (LF) ÷ [5.03] COMBINING DIAERESIS (CM1_CM) ÷ [0.3] × 0061 × 0308 ÷ # × [0.3] LATIN SMALL LETTER A (AL) × [9.0] COMBINING DIAERESIS (CM1_CM) ÷ [0.3] -× 0020 ÷ 200D × 0646 ÷ # × [0.3] SPACE (SP) ÷ [18.0] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [28.0] ARABIC LETTER NOON (AL) ÷ [0.3] +× 0020 ÷ 200D × 0646 ÷ # × [0.3] SPACE (SP) ÷ [18.0] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] ARABIC LETTER NOON (AL) ÷ [0.3] × 0646 × 200D × 0020 ÷ # × [0.3] ARABIC LETTER NOON (AL) × [9.0] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [7.01] SPACE (SP) ÷ [0.3] × 000B ÷ 3041 ÷ # × [0.3] <LINE TABULATION> (BK) ÷ [4.0] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] × 000D ÷ 3041 ÷ # × [0.3] <CARRIAGE RETURN (CR)> (CR) ÷ [5.02] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] @@ -7093,8 +7093,8 @@ × 3041 × 2060 ÷ # × [0.3] HIRAGANA LETTER SMALL A (CJ_NS) × [11.01] WORD JOINER (WJ) ÷ [0.3] × 2060 × 3041 ÷ # × [0.3] WORD JOINER (WJ) × [11.02] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] × 3041 × 0308 × 00A0 ÷ # × [0.3] HIRAGANA LETTER SMALL A (CJ_NS) × [9.0] COMBINING DIAERESIS (CM1_CM) × [12.2] NO-BREAK SPACE (GL) ÷ [0.3] -× 200D × 00A0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [12.3] NO-BREAK SPACE (GL) ÷ [0.3] -× 200D × 002F ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [13.04] SOLIDUS (SY) ÷ [0.3] +× 200D × 00A0 ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] NO-BREAK SPACE (GL) ÷ [0.3] +× 200D × 002F ÷ # × [0.3] ZERO WIDTH JOINER (ZWJ_O_ZWJ_CM) × [8.1] SOLIDUS (SY) ÷ [0.3] × 2014 × 2014 ÷ # × [0.3] EM DASH (B2) × [17.0] EM DASH (B2) ÷ [0.3] × 3041 ÷ FFFC ÷ # × [0.3] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [20.01] OBJECT REPLACEMENT CHARACTER (CB) ÷ [0.3] × FFFC ÷ 3041 ÷ # × [0.3] OBJECT REPLACEMENT CHARACTER (CB) ÷ [20.02] HIRAGANA LETTER SMALL A (CJ_NS) ÷ [0.3] diff --git a/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt index 71f2371c5e..72a31bcdf1 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt @@ -1,6 +1,6 @@ -# NormalizationTest-10.0.0.txt -# Date: 2017-03-08, 08:41:55 GMT -# © 2017 Unicode®, Inc. +# NormalizationTest-11.0.0.txt +# Date: 2018-02-19, 18:33:08 GMT +# © 2018 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -17479,6 +17479,8 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 07F2 059A 0316 302A 0062;0061 302A 07F2 0316 059A 0062;0061 302A 07F2 0316 059A 0062;0061 302A 07F2 0316 059A 0062;0061 302A 07F2 0316 059A 0062; # (a◌߲◌֚◌̖◌〪b; a◌〪◌߲◌̖◌֚b; a◌〪◌߲◌̖◌֚b; a◌〪◌߲◌̖◌֚b; a◌〪◌߲◌̖◌֚b; ) LATIN SMALL LETTER A, NKO COMBINING NASALIZATION MARK, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B 0061 0315 0300 05AE 07F3 0062;00E0 05AE 07F3 0315 0062;0061 05AE 0300 07F3 0315 0062;00E0 05AE 07F3 0315 0062;0061 05AE 0300 07F3 0315 0062; # (a◌̕◌̀◌֮◌߳b; à◌֮◌߳◌̕b; a◌֮◌̀◌߳◌̕b; à◌֮◌߳◌̕b; a◌֮◌̀◌߳◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NKO COMBINING DOUBLE DOT ABOVE, LATIN SMALL LETTER B 0061 07F3 0315 0300 05AE 0062;0061 05AE 07F3 0300 0315 0062;0061 05AE 07F3 0300 0315 0062;0061 05AE 07F3 0300 0315 0062;0061 05AE 07F3 0300 0315 0062; # (a◌߳◌̕◌̀◌֮b; a◌֮◌߳◌̀◌̕b; a◌֮◌߳◌̀◌̕b; a◌֮◌߳◌̀◌̕b; a◌֮◌߳◌̀◌̕b; ) LATIN SMALL LETTER A, NKO COMBINING DOUBLE DOT ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 07FD 0062;0061 302A 0316 07FD 059A 0062;0061 302A 0316 07FD 059A 0062;0061 302A 0316 07FD 059A 0062;0061 302A 0316 07FD 059A 0062; # (a◌֚◌̖◌〪◌߽b; a◌〪◌̖◌߽◌֚b; a◌〪◌̖◌߽◌֚b; a◌〪◌̖◌߽◌֚b; a◌〪◌̖◌߽◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, NKO DANTAYALAN, LATIN SMALL LETTER B +0061 07FD 059A 0316 302A 0062;0061 302A 07FD 0316 059A 0062;0061 302A 07FD 0316 059A 0062;0061 302A 07FD 0316 059A 0062;0061 302A 07FD 0316 059A 0062; # (a◌߽◌֚◌̖◌〪b; a◌〪◌߽◌̖◌֚b; a◌〪◌߽◌̖◌֚b; a◌〪◌߽◌̖◌֚b; a◌〪◌߽◌̖◌֚b; ) LATIN SMALL LETTER A, NKO DANTAYALAN, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B 0061 0315 0300 05AE 0816 0062;00E0 05AE 0816 0315 0062;0061 05AE 0300 0816 0315 0062;00E0 05AE 0816 0315 0062;0061 05AE 0300 0816 0315 0062; # (a◌̕◌̀◌֮◌ࠖb; à◌֮◌ࠖ◌̕b; a◌֮◌̀◌ࠖ◌̕b; à◌֮◌ࠖ◌̕b; a◌֮◌̀◌ࠖ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SAMARITAN MARK IN, LATIN SMALL LETTER B 0061 0816 0315 0300 05AE 0062;0061 05AE 0816 0300 0315 0062;0061 05AE 0816 0300 0315 0062;0061 05AE 0816 0300 0315 0062;0061 05AE 0816 0300 0315 0062; # (a◌ࠖ◌̕◌̀◌֮b; a◌֮◌ࠖ◌̀◌̕b; a◌֮◌ࠖ◌̀◌̕b; a◌֮◌ࠖ◌̀◌̕b; a◌֮◌ࠖ◌̀◌̕b; ) LATIN SMALL LETTER A, SAMARITAN MARK IN, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 0315 0300 05AE 0817 0062;00E0 05AE 0817 0315 0062;0061 05AE 0300 0817 0315 0062;00E0 05AE 0817 0315 0062;0061 05AE 0300 0817 0315 0062; # (a◌̕◌̀◌֮◌ࠗb; à◌֮◌ࠗ◌̕b; a◌֮◌̀◌ࠗ◌̕b; à◌֮◌ࠗ◌̕b; a◌֮◌̀◌ࠗ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SAMARITAN MARK IN-ALAF, LATIN SMALL LETTER B @@ -17527,6 +17529,8 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 085A 059A 0316 302A 0062;0061 302A 085A 0316 059A 0062;0061 302A 085A 0316 059A 0062;0061 302A 085A 0316 059A 0062;0061 302A 085A 0316 059A 0062; # (a◌࡚◌֚◌̖◌〪b; a◌〪◌࡚◌̖◌֚b; a◌〪◌࡚◌̖◌֚b; a◌〪◌࡚◌̖◌֚b; a◌〪◌࡚◌̖◌֚b; ) LATIN SMALL LETTER A, MANDAIC VOCALIZATION MARK, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B 0061 059A 0316 302A 085B 0062;0061 302A 0316 085B 059A 0062;0061 302A 0316 085B 059A 0062;0061 302A 0316 085B 059A 0062;0061 302A 0316 085B 059A 0062; # (a◌֚◌̖◌〪◌࡛b; a◌〪◌̖◌࡛◌֚b; a◌〪◌̖◌࡛◌֚b; a◌〪◌̖◌࡛◌֚b; a◌〪◌̖◌࡛◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MANDAIC GEMINATION MARK, LATIN SMALL LETTER B 0061 085B 059A 0316 302A 0062;0061 302A 085B 0316 059A 0062;0061 302A 085B 0316 059A 0062;0061 302A 085B 0316 059A 0062;0061 302A 085B 0316 059A 0062; # (a◌࡛◌֚◌̖◌〪b; a◌〪◌࡛◌̖◌֚b; a◌〪◌࡛◌̖◌֚b; a◌〪◌࡛◌̖◌֚b; a◌〪◌࡛◌̖◌֚b; ) LATIN SMALL LETTER A, MANDAIC GEMINATION MARK, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 08D3 0062;0061 302A 0316 08D3 059A 0062;0061 302A 0316 08D3 059A 0062;0061 302A 0316 08D3 059A 0062;0061 302A 0316 08D3 059A 0062; # (a◌֚◌̖◌〪◌࣓b; a◌〪◌̖◌࣓◌֚b; a◌〪◌̖◌࣓◌֚b; a◌〪◌̖◌࣓◌֚b; a◌〪◌̖◌࣓◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, ARABIC SMALL LOW WAW, LATIN SMALL LETTER B +0061 08D3 059A 0316 302A 0062;0061 302A 08D3 0316 059A 0062;0061 302A 08D3 0316 059A 0062;0061 302A 08D3 0316 059A 0062;0061 302A 08D3 0316 059A 0062; # (a◌࣓◌֚◌̖◌〪b; a◌〪◌࣓◌̖◌֚b; a◌〪◌࣓◌̖◌֚b; a◌〪◌࣓◌̖◌֚b; a◌〪◌࣓◌̖◌֚b; ) LATIN SMALL LETTER A, ARABIC SMALL LOW WAW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B 0061 0315 0300 05AE 08D4 0062;00E0 05AE 08D4 0315 0062;0061 05AE 0300 08D4 0315 0062;00E0 05AE 08D4 0315 0062;0061 05AE 0300 08D4 0315 0062; # (a◌̕◌̀◌֮◌ࣔb; à◌֮◌ࣔ◌̕b; a◌֮◌̀◌ࣔ◌̕b; à◌֮◌ࣔ◌̕b; a◌֮◌̀◌ࣔ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH WORD AR-RUB, LATIN SMALL LETTER B 0061 08D4 0315 0300 05AE 0062;0061 05AE 08D4 0300 0315 0062;0061 05AE 08D4 0300 0315 0062;0061 05AE 08D4 0300 0315 0062;0061 05AE 08D4 0300 0315 0062; # (a◌ࣔ◌̕◌̀◌֮b; a◌֮◌ࣔ◌̀◌̕b; a◌֮◌ࣔ◌̀◌̕b; a◌֮◌ࣔ◌̀◌̕b; a◌֮◌ࣔ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH WORD AR-RUB, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 0315 0300 05AE 08D5 0062;00E0 05AE 08D5 0315 0062;0061 05AE 0300 08D5 0315 0062;00E0 05AE 08D5 0315 0062;0061 05AE 0300 08D5 0315 0062; # (a◌̕◌̀◌֮◌ࣕb; à◌֮◌ࣕ◌̕b; a◌֮◌̀◌ࣕ◌̕b; à◌֮◌ࣕ◌̕b; a◌֮◌̀◌ࣕ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH SAD, LATIN SMALL LETTER B @@ -17629,6 +17633,8 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 09BC 3099 093C 0334 0062;0061 0334 09BC 093C 3099 0062;0061 0334 09BC 093C 3099 0062;0061 0334 09BC 093C 3099 0062;0061 0334 09BC 093C 3099 0062; # (a◌়◌゙◌़◌̴b; a◌̴◌়◌़◌゙b; a◌̴◌়◌़◌゙b; a◌̴◌়◌़◌゙b; a◌̴◌়◌़◌゙b; ) LATIN SMALL LETTER A, BENGALI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 05B0 094D 3099 09CD 0062;0061 3099 094D 09CD 05B0 0062;0061 3099 094D 09CD 05B0 0062;0061 3099 094D 09CD 05B0 0062;0061 3099 094D 09CD 05B0 0062; # (a◌ְ◌्◌゙◌্b; a◌゙◌्◌্◌ְb; a◌゙◌्◌্◌ְb; a◌゙◌्◌্◌ְb; a◌゙◌्◌্◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, BENGALI SIGN VIRAMA, LATIN SMALL LETTER B 0061 09CD 05B0 094D 3099 0062;0061 3099 09CD 094D 05B0 0062;0061 3099 09CD 094D 05B0 0062;0061 3099 09CD 094D 05B0 0062;0061 3099 09CD 094D 05B0 0062; # (a◌্◌ְ◌्◌゙b; a◌゙◌্◌्◌ְb; a◌゙◌্◌्◌ְb; a◌゙◌্◌्◌ְb; a◌゙◌্◌्◌ְb; ) LATIN SMALL LETTER A, BENGALI SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 09FE 0062;00E0 05AE 09FE 0315 0062;0061 05AE 0300 09FE 0315 0062;00E0 05AE 09FE 0315 0062;0061 05AE 0300 09FE 0315 0062; # (a◌̕◌̀◌֮◌৾b; à◌֮◌৾◌̕b; a◌֮◌̀◌৾◌̕b; à◌֮◌৾◌̕b; a◌֮◌̀◌৾◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, BENGALI SANDHI MARK, LATIN SMALL LETTER B +0061 09FE 0315 0300 05AE 0062;0061 05AE 09FE 0300 0315 0062;0061 05AE 09FE 0300 0315 0062;0061 05AE 09FE 0300 0315 0062;0061 05AE 09FE 0300 0315 0062; # (a◌৾◌̕◌̀◌֮b; a◌֮◌৾◌̀◌̕b; a◌֮◌৾◌̀◌̕b; a◌֮◌৾◌̀◌̕b; a◌֮◌৾◌̀◌̕b; ) LATIN SMALL LETTER A, BENGALI SANDHI MARK, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 3099 093C 0334 0A3C 0062;0061 0334 093C 0A3C 3099 0062;0061 0334 093C 0A3C 3099 0062;0061 0334 093C 0A3C 3099 0062;0061 0334 093C 0A3C 3099 0062; # (a◌゙◌़◌̴◌਼b; a◌̴◌़◌਼◌゙b; a◌̴◌़◌਼◌゙b; a◌̴◌़◌਼◌゙b; a◌̴◌़◌਼◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, GURMUKHI SIGN NUKTA, LATIN SMALL LETTER B 0061 0A3C 3099 093C 0334 0062;0061 0334 0A3C 093C 3099 0062;0061 0334 0A3C 093C 3099 0062;0061 0334 0A3C 093C 3099 0062;0061 0334 0A3C 093C 3099 0062; # (a◌਼◌゙◌़◌̴b; a◌̴◌਼◌़◌゙b; a◌̴◌਼◌़◌゙b; a◌̴◌਼◌़◌゙b; a◌̴◌਼◌़◌゙b; ) LATIN SMALL LETTER A, GURMUKHI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 05B0 094D 3099 0A4D 0062;0061 3099 094D 0A4D 05B0 0062;0061 3099 094D 0A4D 05B0 0062;0061 3099 094D 0A4D 05B0 0062;0061 3099 094D 0A4D 05B0 0062; # (a◌ְ◌्◌゙◌੍b; a◌゙◌्◌੍◌ְb; a◌゙◌्◌੍◌ְb; a◌゙◌्◌੍◌ְb; a◌゙◌्◌੍◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, GURMUKHI SIGN VIRAMA, LATIN SMALL LETTER B @@ -18329,6 +18335,36 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 10AE5 0315 0300 05AE 0062;0061 05AE 10AE5 0300 0315 0062;0061 05AE 10AE5 0300 0315 0062;0061 05AE 10AE5 0300 0315 0062;0061 05AE 10AE5 0300 0315 0062; # (a◌𐫥◌̕◌̀◌֮b; a◌֮◌𐫥◌̀◌̕b; a◌֮◌𐫥◌̀◌̕b; a◌֮◌𐫥◌̀◌̕b; a◌֮◌𐫥◌̀◌̕b; ) LATIN SMALL LETTER A, MANICHAEAN ABBREVIATION MARK ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 059A 0316 302A 10AE6 0062;0061 302A 0316 10AE6 059A 0062;0061 302A 0316 10AE6 059A 0062;0061 302A 0316 10AE6 059A 0062;0061 302A 0316 10AE6 059A 0062; # (a◌֚◌̖◌〪◌𐫦b; a◌〪◌̖◌𐫦◌֚b; a◌〪◌̖◌𐫦◌֚b; a◌〪◌̖◌𐫦◌֚b; a◌〪◌̖◌𐫦◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MANICHAEAN ABBREVIATION MARK BELOW, LATIN SMALL LETTER B 0061 10AE6 059A 0316 302A 0062;0061 302A 10AE6 0316 059A 0062;0061 302A 10AE6 0316 059A 0062;0061 302A 10AE6 0316 059A 0062;0061 302A 10AE6 0316 059A 0062; # (a◌𐫦◌֚◌̖◌〪b; a◌〪◌𐫦◌̖◌֚b; a◌〪◌𐫦◌̖◌֚b; a◌〪◌𐫦◌̖◌֚b; a◌〪◌𐫦◌̖◌֚b; ) LATIN SMALL LETTER A, MANICHAEAN ABBREVIATION MARK BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 10D24 0062;00E0 05AE 10D24 0315 0062;0061 05AE 0300 10D24 0315 0062;00E0 05AE 10D24 0315 0062;0061 05AE 0300 10D24 0315 0062; # (a◌̕◌̀◌֮◌𐴤b; à◌֮◌𐴤◌̕b; a◌֮◌̀◌𐴤◌̕b; à◌֮◌𐴤◌̕b; a◌֮◌̀◌𐴤◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HANIFI ROHINGYA SIGN HARBAHAY, LATIN SMALL LETTER B +0061 10D24 0315 0300 05AE 0062;0061 05AE 10D24 0300 0315 0062;0061 05AE 10D24 0300 0315 0062;0061 05AE 10D24 0300 0315 0062;0061 05AE 10D24 0300 0315 0062; # (a◌𐴤◌̕◌̀◌֮b; a◌֮◌𐴤◌̀◌̕b; a◌֮◌𐴤◌̀◌̕b; a◌֮◌𐴤◌̀◌̕b; a◌֮◌𐴤◌̀◌̕b; ) LATIN SMALL LETTER A, HANIFI ROHINGYA SIGN HARBAHAY, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 10D25 0062;00E0 05AE 10D25 0315 0062;0061 05AE 0300 10D25 0315 0062;00E0 05AE 10D25 0315 0062;0061 05AE 0300 10D25 0315 0062; # (a◌̕◌̀◌֮◌𐴥b; à◌֮◌𐴥◌̕b; a◌֮◌̀◌𐴥◌̕b; à◌֮◌𐴥◌̕b; a◌֮◌̀◌𐴥◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HANIFI ROHINGYA SIGN TAHALA, LATIN SMALL LETTER B +0061 10D25 0315 0300 05AE 0062;0061 05AE 10D25 0300 0315 0062;0061 05AE 10D25 0300 0315 0062;0061 05AE 10D25 0300 0315 0062;0061 05AE 10D25 0300 0315 0062; # (a◌𐴥◌̕◌̀◌֮b; a◌֮◌𐴥◌̀◌̕b; a◌֮◌𐴥◌̀◌̕b; a◌֮◌𐴥◌̀◌̕b; a◌֮◌𐴥◌̀◌̕b; ) LATIN SMALL LETTER A, HANIFI ROHINGYA SIGN TAHALA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 10D26 0062;00E0 05AE 10D26 0315 0062;0061 05AE 0300 10D26 0315 0062;00E0 05AE 10D26 0315 0062;0061 05AE 0300 10D26 0315 0062; # (a◌̕◌̀◌֮◌𐴦b; à◌֮◌𐴦◌̕b; a◌֮◌̀◌𐴦◌̕b; à◌֮◌𐴦◌̕b; a◌֮◌̀◌𐴦◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HANIFI ROHINGYA SIGN TANA, LATIN SMALL LETTER B +0061 10D26 0315 0300 05AE 0062;0061 05AE 10D26 0300 0315 0062;0061 05AE 10D26 0300 0315 0062;0061 05AE 10D26 0300 0315 0062;0061 05AE 10D26 0300 0315 0062; # (a◌𐴦◌̕◌̀◌֮b; a◌֮◌𐴦◌̀◌̕b; a◌֮◌𐴦◌̀◌̕b; a◌֮◌𐴦◌̀◌̕b; a◌֮◌𐴦◌̀◌̕b; ) LATIN SMALL LETTER A, HANIFI ROHINGYA SIGN TANA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 10D27 0062;00E0 05AE 10D27 0315 0062;0061 05AE 0300 10D27 0315 0062;00E0 05AE 10D27 0315 0062;0061 05AE 0300 10D27 0315 0062; # (a◌̕◌̀◌֮◌𐴧b; à◌֮◌𐴧◌̕b; a◌֮◌̀◌𐴧◌̕b; à◌֮◌𐴧◌̕b; a◌֮◌̀◌𐴧◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HANIFI ROHINGYA SIGN TASSI, LATIN SMALL LETTER B +0061 10D27 0315 0300 05AE 0062;0061 05AE 10D27 0300 0315 0062;0061 05AE 10D27 0300 0315 0062;0061 05AE 10D27 0300 0315 0062;0061 05AE 10D27 0300 0315 0062; # (a◌𐴧◌̕◌̀◌֮b; a◌֮◌𐴧◌̀◌̕b; a◌֮◌𐴧◌̀◌̕b; a◌֮◌𐴧◌̀◌̕b; a◌֮◌𐴧◌̀◌̕b; ) LATIN SMALL LETTER A, HANIFI ROHINGYA SIGN TASSI, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 10F46 0062;0061 302A 0316 10F46 059A 0062;0061 302A 0316 10F46 059A 0062;0061 302A 0316 10F46 059A 0062;0061 302A 0316 10F46 059A 0062; # (a◌֚◌̖◌〪◌𐽆b; a◌〪◌̖◌𐽆◌֚b; a◌〪◌̖◌𐽆◌֚b; a◌〪◌̖◌𐽆◌֚b; a◌〪◌̖◌𐽆◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SOGDIAN COMBINING DOT BELOW, LATIN SMALL LETTER B +0061 10F46 059A 0316 302A 0062;0061 302A 10F46 0316 059A 0062;0061 302A 10F46 0316 059A 0062;0061 302A 10F46 0316 059A 0062;0061 302A 10F46 0316 059A 0062; # (a◌𐽆◌֚◌̖◌〪b; a◌〪◌𐽆◌̖◌֚b; a◌〪◌𐽆◌̖◌֚b; a◌〪◌𐽆◌̖◌֚b; a◌〪◌𐽆◌̖◌֚b; ) LATIN SMALL LETTER A, SOGDIAN COMBINING DOT BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 10F47 0062;0061 302A 0316 10F47 059A 0062;0061 302A 0316 10F47 059A 0062;0061 302A 0316 10F47 059A 0062;0061 302A 0316 10F47 059A 0062; # (a◌֚◌̖◌〪◌𐽇b; a◌〪◌̖◌𐽇◌֚b; a◌〪◌̖◌𐽇◌֚b; a◌〪◌̖◌𐽇◌֚b; a◌〪◌̖◌𐽇◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SOGDIAN COMBINING TWO DOTS BELOW, LATIN SMALL LETTER B +0061 10F47 059A 0316 302A 0062;0061 302A 10F47 0316 059A 0062;0061 302A 10F47 0316 059A 0062;0061 302A 10F47 0316 059A 0062;0061 302A 10F47 0316 059A 0062; # (a◌𐽇◌֚◌̖◌〪b; a◌〪◌𐽇◌̖◌֚b; a◌〪◌𐽇◌̖◌֚b; a◌〪◌𐽇◌̖◌֚b; a◌〪◌𐽇◌̖◌֚b; ) LATIN SMALL LETTER A, SOGDIAN COMBINING TWO DOTS BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 10F48 0062;00E0 05AE 10F48 0315 0062;0061 05AE 0300 10F48 0315 0062;00E0 05AE 10F48 0315 0062;0061 05AE 0300 10F48 0315 0062; # (a◌̕◌̀◌֮◌𐽈b; à◌֮◌𐽈◌̕b; a◌֮◌̀◌𐽈◌̕b; à◌֮◌𐽈◌̕b; a◌֮◌̀◌𐽈◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SOGDIAN COMBINING DOT ABOVE, LATIN SMALL LETTER B +0061 10F48 0315 0300 05AE 0062;0061 05AE 10F48 0300 0315 0062;0061 05AE 10F48 0300 0315 0062;0061 05AE 10F48 0300 0315 0062;0061 05AE 10F48 0300 0315 0062; # (a◌𐽈◌̕◌̀◌֮b; a◌֮◌𐽈◌̀◌̕b; a◌֮◌𐽈◌̀◌̕b; a◌֮◌𐽈◌̀◌̕b; a◌֮◌𐽈◌̀◌̕b; ) LATIN SMALL LETTER A, SOGDIAN COMBINING DOT ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 10F49 0062;00E0 05AE 10F49 0315 0062;0061 05AE 0300 10F49 0315 0062;00E0 05AE 10F49 0315 0062;0061 05AE 0300 10F49 0315 0062; # (a◌̕◌̀◌֮◌𐽉b; à◌֮◌𐽉◌̕b; a◌֮◌̀◌𐽉◌̕b; à◌֮◌𐽉◌̕b; a◌֮◌̀◌𐽉◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SOGDIAN COMBINING TWO DOTS ABOVE, LATIN SMALL LETTER B +0061 10F49 0315 0300 05AE 0062;0061 05AE 10F49 0300 0315 0062;0061 05AE 10F49 0300 0315 0062;0061 05AE 10F49 0300 0315 0062;0061 05AE 10F49 0300 0315 0062; # (a◌𐽉◌̕◌̀◌֮b; a◌֮◌𐽉◌̀◌̕b; a◌֮◌𐽉◌̀◌̕b; a◌֮◌𐽉◌̀◌̕b; a◌֮◌𐽉◌̀◌̕b; ) LATIN SMALL LETTER A, SOGDIAN COMBINING TWO DOTS ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 10F4A 0062;00E0 05AE 10F4A 0315 0062;0061 05AE 0300 10F4A 0315 0062;00E0 05AE 10F4A 0315 0062;0061 05AE 0300 10F4A 0315 0062; # (a◌̕◌̀◌֮◌𐽊b; à◌֮◌𐽊◌̕b; a◌֮◌̀◌𐽊◌̕b; à◌֮◌𐽊◌̕b; a◌֮◌̀◌𐽊◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SOGDIAN COMBINING CURVE ABOVE, LATIN SMALL LETTER B +0061 10F4A 0315 0300 05AE 0062;0061 05AE 10F4A 0300 0315 0062;0061 05AE 10F4A 0300 0315 0062;0061 05AE 10F4A 0300 0315 0062;0061 05AE 10F4A 0300 0315 0062; # (a◌𐽊◌̕◌̀◌֮b; a◌֮◌𐽊◌̀◌̕b; a◌֮◌𐽊◌̀◌̕b; a◌֮◌𐽊◌̀◌̕b; a◌֮◌𐽊◌̀◌̕b; ) LATIN SMALL LETTER A, SOGDIAN COMBINING CURVE ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 10F4B 0062;0061 302A 0316 10F4B 059A 0062;0061 302A 0316 10F4B 059A 0062;0061 302A 0316 10F4B 059A 0062;0061 302A 0316 10F4B 059A 0062; # (a◌֚◌̖◌〪◌𐽋b; a◌〪◌̖◌𐽋◌֚b; a◌〪◌̖◌𐽋◌֚b; a◌〪◌̖◌𐽋◌֚b; a◌〪◌̖◌𐽋◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SOGDIAN COMBINING CURVE BELOW, LATIN SMALL LETTER B +0061 10F4B 059A 0316 302A 0062;0061 302A 10F4B 0316 059A 0062;0061 302A 10F4B 0316 059A 0062;0061 302A 10F4B 0316 059A 0062;0061 302A 10F4B 0316 059A 0062; # (a◌𐽋◌֚◌̖◌〪b; a◌〪◌𐽋◌̖◌֚b; a◌〪◌𐽋◌̖◌֚b; a◌〪◌𐽋◌̖◌֚b; a◌〪◌𐽋◌̖◌֚b; ) LATIN SMALL LETTER A, SOGDIAN COMBINING CURVE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 10F4C 0062;00E0 05AE 10F4C 0315 0062;0061 05AE 0300 10F4C 0315 0062;00E0 05AE 10F4C 0315 0062;0061 05AE 0300 10F4C 0315 0062; # (a◌̕◌̀◌֮◌𐽌b; à◌֮◌𐽌◌̕b; a◌֮◌̀◌𐽌◌̕b; à◌֮◌𐽌◌̕b; a◌֮◌̀◌𐽌◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SOGDIAN COMBINING HOOK ABOVE, LATIN SMALL LETTER B +0061 10F4C 0315 0300 05AE 0062;0061 05AE 10F4C 0300 0315 0062;0061 05AE 10F4C 0300 0315 0062;0061 05AE 10F4C 0300 0315 0062;0061 05AE 10F4C 0300 0315 0062; # (a◌𐽌◌̕◌̀◌֮b; a◌֮◌𐽌◌̀◌̕b; a◌֮◌𐽌◌̀◌̕b; a◌֮◌𐽌◌̀◌̕b; a◌֮◌𐽌◌̀◌̕b; ) LATIN SMALL LETTER A, SOGDIAN COMBINING HOOK ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 10F4D 0062;0061 302A 0316 10F4D 059A 0062;0061 302A 0316 10F4D 059A 0062;0061 302A 0316 10F4D 059A 0062;0061 302A 0316 10F4D 059A 0062; # (a◌֚◌̖◌〪◌𐽍b; a◌〪◌̖◌𐽍◌֚b; a◌〪◌̖◌𐽍◌֚b; a◌〪◌̖◌𐽍◌֚b; a◌〪◌̖◌𐽍◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SOGDIAN COMBINING HOOK BELOW, LATIN SMALL LETTER B +0061 10F4D 059A 0316 302A 0062;0061 302A 10F4D 0316 059A 0062;0061 302A 10F4D 0316 059A 0062;0061 302A 10F4D 0316 059A 0062;0061 302A 10F4D 0316 059A 0062; # (a◌𐽍◌֚◌̖◌〪b; a◌〪◌𐽍◌̖◌֚b; a◌〪◌𐽍◌̖◌֚b; a◌〪◌𐽍◌̖◌֚b; a◌〪◌𐽍◌̖◌֚b; ) LATIN SMALL LETTER A, SOGDIAN COMBINING HOOK BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 10F4E 0062;0061 302A 0316 10F4E 059A 0062;0061 302A 0316 10F4E 059A 0062;0061 302A 0316 10F4E 059A 0062;0061 302A 0316 10F4E 059A 0062; # (a◌֚◌̖◌〪◌𐽎b; a◌〪◌̖◌𐽎◌֚b; a◌〪◌̖◌𐽎◌֚b; a◌〪◌̖◌𐽎◌֚b; a◌〪◌̖◌𐽎◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SOGDIAN COMBINING LONG HOOK BELOW, LATIN SMALL LETTER B +0061 10F4E 059A 0316 302A 0062;0061 302A 10F4E 0316 059A 0062;0061 302A 10F4E 0316 059A 0062;0061 302A 10F4E 0316 059A 0062;0061 302A 10F4E 0316 059A 0062; # (a◌𐽎◌֚◌̖◌〪b; a◌〪◌𐽎◌̖◌֚b; a◌〪◌𐽎◌̖◌֚b; a◌〪◌𐽎◌̖◌֚b; a◌〪◌𐽎◌̖◌֚b; ) LATIN SMALL LETTER A, SOGDIAN COMBINING LONG HOOK BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 10F4F 0062;0061 302A 0316 10F4F 059A 0062;0061 302A 0316 10F4F 059A 0062;0061 302A 0316 10F4F 059A 0062;0061 302A 0316 10F4F 059A 0062; # (a◌֚◌̖◌〪◌𐽏b; a◌〪◌̖◌𐽏◌֚b; a◌〪◌̖◌𐽏◌֚b; a◌〪◌̖◌𐽏◌֚b; a◌〪◌̖◌𐽏◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SOGDIAN COMBINING RESH BELOW, LATIN SMALL LETTER B +0061 10F4F 059A 0316 302A 0062;0061 302A 10F4F 0316 059A 0062;0061 302A 10F4F 0316 059A 0062;0061 302A 10F4F 0316 059A 0062;0061 302A 10F4F 0316 059A 0062; # (a◌𐽏◌֚◌̖◌〪b; a◌〪◌𐽏◌̖◌֚b; a◌〪◌𐽏◌̖◌֚b; a◌〪◌𐽏◌̖◌֚b; a◌〪◌𐽏◌̖◌֚b; ) LATIN SMALL LETTER A, SOGDIAN COMBINING RESH BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 10F50 0062;0061 302A 0316 10F50 059A 0062;0061 302A 0316 10F50 059A 0062;0061 302A 0316 10F50 059A 0062;0061 302A 0316 10F50 059A 0062; # (a◌֚◌̖◌〪◌𐽐b; a◌〪◌̖◌𐽐◌֚b; a◌〪◌̖◌𐽐◌֚b; a◌〪◌̖◌𐽐◌֚b; a◌〪◌̖◌𐽐◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SOGDIAN COMBINING STROKE BELOW, LATIN SMALL LETTER B +0061 10F50 059A 0316 302A 0062;0061 302A 10F50 0316 059A 0062;0061 302A 10F50 0316 059A 0062;0061 302A 10F50 0316 059A 0062;0061 302A 10F50 0316 059A 0062; # (a◌𐽐◌֚◌̖◌〪b; a◌〪◌𐽐◌̖◌֚b; a◌〪◌𐽐◌̖◌֚b; a◌〪◌𐽐◌̖◌֚b; a◌〪◌𐽐◌̖◌֚b; ) LATIN SMALL LETTER A, SOGDIAN COMBINING STROKE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B 0061 05B0 094D 3099 11046 0062;0061 3099 094D 11046 05B0 0062;0061 3099 094D 11046 05B0 0062;0061 3099 094D 11046 05B0 0062;0061 3099 094D 11046 05B0 0062; # (a◌ְ◌्◌゙◌𑁆b; a◌゙◌्◌𑁆◌ְb; a◌゙◌्◌𑁆◌ְb; a◌゙◌्◌𑁆◌ְb; a◌゙◌्◌𑁆◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, BRAHMI VIRAMA, LATIN SMALL LETTER B 0061 11046 05B0 094D 3099 0062;0061 3099 11046 094D 05B0 0062;0061 3099 11046 094D 05B0 0062;0061 3099 11046 094D 05B0 0062;0061 3099 11046 094D 05B0 0062; # (a◌𑁆◌ְ◌्◌゙b; a◌゙◌𑁆◌्◌ְb; a◌゙◌𑁆◌्◌ְb; a◌゙◌𑁆◌्◌ְb; a◌゙◌𑁆◌्◌ְb; ) LATIN SMALL LETTER A, BRAHMI VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 05B0 094D 3099 1107F 0062;0061 3099 094D 1107F 05B0 0062;0061 3099 094D 1107F 05B0 0062;0061 3099 094D 1107F 05B0 0062;0061 3099 094D 1107F 05B0 0062; # (a◌ְ◌्◌゙◌𑁿b; a◌゙◌्◌𑁿◌ְb; a◌゙◌्◌𑁿◌ְb; a◌゙◌्◌𑁿◌ְb; a◌゙◌्◌𑁿◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, BRAHMI NUMBER JOINER, LATIN SMALL LETTER B @@ -18361,6 +18397,8 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 112E9 3099 093C 0334 0062;0061 0334 112E9 093C 3099 0062;0061 0334 112E9 093C 3099 0062;0061 0334 112E9 093C 3099 0062;0061 0334 112E9 093C 3099 0062; # (a◌𑋩◌゙◌़◌̴b; a◌̴◌𑋩◌़◌゙b; a◌̴◌𑋩◌़◌゙b; a◌̴◌𑋩◌़◌゙b; a◌̴◌𑋩◌़◌゙b; ) LATIN SMALL LETTER A, KHUDAWADI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 05B0 094D 3099 112EA 0062;0061 3099 094D 112EA 05B0 0062;0061 3099 094D 112EA 05B0 0062;0061 3099 094D 112EA 05B0 0062;0061 3099 094D 112EA 05B0 0062; # (a◌ְ◌्◌゙◌𑋪b; a◌゙◌्◌𑋪◌ְb; a◌゙◌्◌𑋪◌ְb; a◌゙◌्◌𑋪◌ְb; a◌゙◌्◌𑋪◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, KHUDAWADI SIGN VIRAMA, LATIN SMALL LETTER B 0061 112EA 05B0 094D 3099 0062;0061 3099 112EA 094D 05B0 0062;0061 3099 112EA 094D 05B0 0062;0061 3099 112EA 094D 05B0 0062;0061 3099 112EA 094D 05B0 0062; # (a◌𑋪◌ְ◌्◌゙b; a◌゙◌𑋪◌्◌ְb; a◌゙◌𑋪◌्◌ְb; a◌゙◌𑋪◌्◌ְb; a◌゙◌𑋪◌्◌ְb; ) LATIN SMALL LETTER A, KHUDAWADI SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 3099 093C 0334 1133B 0062;0061 0334 093C 1133B 3099 0062;0061 0334 093C 1133B 3099 0062;0061 0334 093C 1133B 3099 0062;0061 0334 093C 1133B 3099 0062; # (a◌゙◌़◌̴◌𑌻b; a◌̴◌़◌𑌻◌゙b; a◌̴◌़◌𑌻◌゙b; a◌̴◌़◌𑌻◌゙b; a◌̴◌़◌𑌻◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING BINDU BELOW, LATIN SMALL LETTER B +0061 1133B 3099 093C 0334 0062;0061 0334 1133B 093C 3099 0062;0061 0334 1133B 093C 3099 0062;0061 0334 1133B 093C 3099 0062;0061 0334 1133B 093C 3099 0062; # (a◌𑌻◌゙◌़◌̴b; a◌̴◌𑌻◌़◌゙b; a◌̴◌𑌻◌़◌゙b; a◌̴◌𑌻◌़◌゙b; a◌̴◌𑌻◌़◌゙b; ) LATIN SMALL LETTER A, COMBINING BINDU BELOW, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 3099 093C 0334 1133C 0062;0061 0334 093C 1133C 3099 0062;0061 0334 093C 1133C 3099 0062;0061 0334 093C 1133C 3099 0062;0061 0334 093C 1133C 3099 0062; # (a◌゙◌़◌̴◌𑌼b; a◌̴◌़◌𑌼◌゙b; a◌̴◌़◌𑌼◌゙b; a◌̴◌़◌𑌼◌゙b; a◌̴◌़◌𑌼◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, GRANTHA SIGN NUKTA, LATIN SMALL LETTER B 0061 1133C 3099 093C 0334 0062;0061 0334 1133C 093C 3099 0062;0061 0334 1133C 093C 3099 0062;0061 0334 1133C 093C 3099 0062;0061 0334 1133C 093C 3099 0062; # (a◌𑌼◌゙◌़◌̴b; a◌̴◌𑌼◌़◌゙b; a◌̴◌𑌼◌़◌゙b; a◌̴◌𑌼◌़◌゙b; a◌̴◌𑌼◌़◌゙b; ) LATIN SMALL LETTER A, GRANTHA SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 05B0 094D 3099 1134D 0062;0061 3099 094D 1134D 05B0 0062;0061 3099 094D 1134D 05B0 0062;0061 3099 094D 1134D 05B0 0062;0061 3099 094D 1134D 05B0 0062; # (a◌ְ◌्◌゙𑍍b; a◌゙◌्𑍍◌ְb; a◌゙◌्𑍍◌ְb; a◌゙◌्𑍍◌ְb; a◌゙◌्𑍍◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, GRANTHA SIGN VIRAMA, LATIN SMALL LETTER B @@ -18393,6 +18431,8 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 11442 05B0 094D 3099 0062;0061 3099 11442 094D 05B0 0062;0061 3099 11442 094D 05B0 0062;0061 3099 11442 094D 05B0 0062;0061 3099 11442 094D 05B0 0062; # (a◌𑑂◌ְ◌्◌゙b; a◌゙◌𑑂◌्◌ְb; a◌゙◌𑑂◌्◌ְb; a◌゙◌𑑂◌्◌ְb; a◌゙◌𑑂◌्◌ְb; ) LATIN SMALL LETTER A, NEWA SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 3099 093C 0334 11446 0062;0061 0334 093C 11446 3099 0062;0061 0334 093C 11446 3099 0062;0061 0334 093C 11446 3099 0062;0061 0334 093C 11446 3099 0062; # (a◌゙◌़◌̴◌𑑆b; a◌̴◌़◌𑑆◌゙b; a◌̴◌़◌𑑆◌゙b; a◌̴◌़◌𑑆◌゙b; a◌̴◌़◌𑑆◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, NEWA SIGN NUKTA, LATIN SMALL LETTER B 0061 11446 3099 093C 0334 0062;0061 0334 11446 093C 3099 0062;0061 0334 11446 093C 3099 0062;0061 0334 11446 093C 3099 0062;0061 0334 11446 093C 3099 0062; # (a◌𑑆◌゙◌़◌̴b; a◌̴◌𑑆◌़◌゙b; a◌̴◌𑑆◌़◌゙b; a◌̴◌𑑆◌़◌゙b; a◌̴◌𑑆◌़◌゙b; ) LATIN SMALL LETTER A, NEWA SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 0315 0300 05AE 1145E 0062;00E0 05AE 1145E 0315 0062;0061 05AE 0300 1145E 0315 0062;00E0 05AE 1145E 0315 0062;0061 05AE 0300 1145E 0315 0062; # (a◌̕◌̀◌֮◌𑑞b; à◌֮◌𑑞◌̕b; a◌֮◌̀◌𑑞◌̕b; à◌֮◌𑑞◌̕b; a◌֮◌̀◌𑑞◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NEWA SANDHI MARK, LATIN SMALL LETTER B +0061 1145E 0315 0300 05AE 0062;0061 05AE 1145E 0300 0315 0062;0061 05AE 1145E 0300 0315 0062;0061 05AE 1145E 0300 0315 0062;0061 05AE 1145E 0300 0315 0062; # (a◌𑑞◌̕◌̀◌֮b; a◌֮◌𑑞◌̀◌̕b; a◌֮◌𑑞◌̀◌̕b; a◌֮◌𑑞◌̀◌̕b; a◌֮◌𑑞◌̀◌̕b; ) LATIN SMALL LETTER A, NEWA SANDHI MARK, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 05B0 094D 3099 114C2 0062;0061 3099 094D 114C2 05B0 0062;0061 3099 094D 114C2 05B0 0062;0061 3099 094D 114C2 05B0 0062;0061 3099 094D 114C2 05B0 0062; # (a◌ְ◌्◌゙◌𑓂b; a◌゙◌्◌𑓂◌ְb; a◌゙◌्◌𑓂◌ְb; a◌゙◌्◌𑓂◌ְb; a◌゙◌्◌𑓂◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, TIRHUTA SIGN VIRAMA, LATIN SMALL LETTER B 0061 114C2 05B0 094D 3099 0062;0061 3099 114C2 094D 05B0 0062;0061 3099 114C2 094D 05B0 0062;0061 3099 114C2 094D 05B0 0062;0061 3099 114C2 094D 05B0 0062; # (a◌𑓂◌ְ◌्◌゙b; a◌゙◌𑓂◌्◌ְb; a◌゙◌𑓂◌्◌ְb; a◌゙◌𑓂◌्◌ְb; a◌゙◌𑓂◌्◌ְb; ) LATIN SMALL LETTER A, TIRHUTA SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 3099 093C 0334 114C3 0062;0061 0334 093C 114C3 3099 0062;0061 0334 093C 114C3 3099 0062;0061 0334 093C 114C3 3099 0062;0061 0334 093C 114C3 3099 0062; # (a◌゙◌़◌̴◌𑓃b; a◌̴◌़◌𑓃◌゙b; a◌̴◌़◌𑓃◌゙b; a◌̴◌़◌𑓃◌゙b; a◌̴◌़◌𑓃◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, TIRHUTA SIGN NUKTA, LATIN SMALL LETTER B @@ -18409,6 +18449,10 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 116B7 3099 093C 0334 0062;0061 0334 116B7 093C 3099 0062;0061 0334 116B7 093C 3099 0062;0061 0334 116B7 093C 3099 0062;0061 0334 116B7 093C 3099 0062; # (a◌𑚷◌゙◌़◌̴b; a◌̴◌𑚷◌़◌゙b; a◌̴◌𑚷◌़◌゙b; a◌̴◌𑚷◌़◌゙b; a◌̴◌𑚷◌़◌゙b; ) LATIN SMALL LETTER A, TAKRI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 05B0 094D 3099 1172B 0062;0061 3099 094D 1172B 05B0 0062;0061 3099 094D 1172B 05B0 0062;0061 3099 094D 1172B 05B0 0062;0061 3099 094D 1172B 05B0 0062; # (a◌ְ◌्◌゙◌𑜫b; a◌゙◌्◌𑜫◌ְb; a◌゙◌्◌𑜫◌ְb; a◌゙◌्◌𑜫◌ְb; a◌゙◌्◌𑜫◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, AHOM SIGN KILLER, LATIN SMALL LETTER B 0061 1172B 05B0 094D 3099 0062;0061 3099 1172B 094D 05B0 0062;0061 3099 1172B 094D 05B0 0062;0061 3099 1172B 094D 05B0 0062;0061 3099 1172B 094D 05B0 0062; # (a◌𑜫◌ְ◌्◌゙b; a◌゙◌𑜫◌्◌ְb; a◌゙◌𑜫◌्◌ְb; a◌゙◌𑜫◌्◌ְb; a◌゙◌𑜫◌्◌ְb; ) LATIN SMALL LETTER A, AHOM SIGN KILLER, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 11839 0062;0061 3099 094D 11839 05B0 0062;0061 3099 094D 11839 05B0 0062;0061 3099 094D 11839 05B0 0062;0061 3099 094D 11839 05B0 0062; # (a◌ְ◌्◌゙◌𑠹b; a◌゙◌्◌𑠹◌ְb; a◌゙◌्◌𑠹◌ְb; a◌゙◌्◌𑠹◌ְb; a◌゙◌्◌𑠹◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DOGRA SIGN VIRAMA, LATIN SMALL LETTER B +0061 11839 05B0 094D 3099 0062;0061 3099 11839 094D 05B0 0062;0061 3099 11839 094D 05B0 0062;0061 3099 11839 094D 05B0 0062;0061 3099 11839 094D 05B0 0062; # (a◌𑠹◌ְ◌्◌゙b; a◌゙◌𑠹◌्◌ְb; a◌゙◌𑠹◌्◌ְb; a◌゙◌𑠹◌्◌ְb; a◌゙◌𑠹◌्◌ְb; ) LATIN SMALL LETTER A, DOGRA SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 3099 093C 0334 1183A 0062;0061 0334 093C 1183A 3099 0062;0061 0334 093C 1183A 3099 0062;0061 0334 093C 1183A 3099 0062;0061 0334 093C 1183A 3099 0062; # (a◌゙◌़◌̴◌𑠺b; a◌̴◌़◌𑠺◌゙b; a◌̴◌़◌𑠺◌゙b; a◌̴◌़◌𑠺◌゙b; a◌̴◌़◌𑠺◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, DOGRA SIGN NUKTA, LATIN SMALL LETTER B +0061 1183A 3099 093C 0334 0062;0061 0334 1183A 093C 3099 0062;0061 0334 1183A 093C 3099 0062;0061 0334 1183A 093C 3099 0062;0061 0334 1183A 093C 3099 0062; # (a◌𑠺◌゙◌़◌̴b; a◌̴◌𑠺◌़◌゙b; a◌̴◌𑠺◌़◌゙b; a◌̴◌𑠺◌़◌゙b; a◌̴◌𑠺◌़◌゙b; ) LATIN SMALL LETTER A, DOGRA SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 05B0 094D 3099 11A34 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062; # (a◌ְ◌्◌゙◌𑨴b; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, ZANABAZAR SQUARE SIGN VIRAMA, LATIN SMALL LETTER B 0061 11A34 05B0 094D 3099 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062; # (a◌𑨴◌ְ◌्◌゙b; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; ) LATIN SMALL LETTER A, ZANABAZAR SQUARE SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 05B0 094D 3099 11A47 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062; # (a◌ְ◌्◌゙◌𑩇b; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, ZANABAZAR SQUARE SUBJOINER, LATIN SMALL LETTER B @@ -18423,6 +18467,8 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 11D44 05B0 094D 3099 0062;0061 3099 11D44 094D 05B0 0062;0061 3099 11D44 094D 05B0 0062;0061 3099 11D44 094D 05B0 0062;0061 3099 11D44 094D 05B0 0062; # (a◌𑵄◌ְ◌्◌゙b; a◌゙◌𑵄◌्◌ְb; a◌゙◌𑵄◌्◌ְb; a◌゙◌𑵄◌्◌ְb; a◌゙◌𑵄◌्◌ְb; ) LATIN SMALL LETTER A, MASARAM GONDI SIGN HALANTA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 05B0 094D 3099 11D45 0062;0061 3099 094D 11D45 05B0 0062;0061 3099 094D 11D45 05B0 0062;0061 3099 094D 11D45 05B0 0062;0061 3099 094D 11D45 05B0 0062; # (a◌ְ◌्◌゙◌𑵅b; a◌゙◌्◌𑵅◌ְb; a◌゙◌्◌𑵅◌ְb; a◌゙◌्◌𑵅◌ְb; a◌゙◌्◌𑵅◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MASARAM GONDI VIRAMA, LATIN SMALL LETTER B 0061 11D45 05B0 094D 3099 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062; # (a◌𑵅◌ְ◌्◌゙b; a◌゙◌𑵅◌्◌ְb; a◌゙◌𑵅◌्◌ְb; a◌゙◌𑵅◌्◌ְb; a◌゙◌𑵅◌्◌ְb; ) LATIN SMALL LETTER A, MASARAM GONDI VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 11D97 0062;0061 3099 094D 11D97 05B0 0062;0061 3099 094D 11D97 05B0 0062;0061 3099 094D 11D97 05B0 0062;0061 3099 094D 11D97 05B0 0062; # (a◌ְ◌्◌゙◌𑶗b; a◌゙◌्◌𑶗◌ְb; a◌゙◌्◌𑶗◌ְb; a◌゙◌्◌𑶗◌ְb; a◌゙◌्◌𑶗◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, GUNJALA GONDI VIRAMA, LATIN SMALL LETTER B +0061 11D97 05B0 094D 3099 0062;0061 3099 11D97 094D 05B0 0062;0061 3099 11D97 094D 05B0 0062;0061 3099 11D97 094D 05B0 0062;0061 3099 11D97 094D 05B0 0062; # (a◌𑶗◌ְ◌्◌゙b; a◌゙◌𑶗◌्◌ְb; a◌゙◌𑶗◌्◌ְb; a◌゙◌𑶗◌्◌ְb; a◌゙◌𑶗◌्◌ְb; ) LATIN SMALL LETTER A, GUNJALA GONDI VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 093C 0334 16AF0 0062;0061 0334 16AF0 093C 0062;0061 0334 16AF0 093C 0062;0061 0334 16AF0 093C 0062;0061 0334 16AF0 093C 0062; # (a◌़◌̴◌𖫰b; a◌̴◌𖫰◌़b; a◌̴◌𖫰◌़b; a◌̴◌𖫰◌़b; a◌̴◌𖫰◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, BASSA VAH COMBINING HIGH TONE, LATIN SMALL LETTER B 0061 16AF0 093C 0334 0062;0061 16AF0 0334 093C 0062;0061 16AF0 0334 093C 0062;0061 16AF0 0334 093C 0062;0061 16AF0 0334 093C 0062; # (a◌𖫰◌़◌̴b; a◌𖫰◌̴◌़b; a◌𖫰◌̴◌़b; a◌𖫰◌̴◌़b; a◌𖫰◌̴◌़b; ) LATIN SMALL LETTER A, BASSA VAH COMBINING HIGH TONE, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 093C 0334 16AF1 0062;0061 0334 16AF1 093C 0062;0061 0334 16AF1 093C 0062;0061 0334 16AF1 093C 0062;0061 0334 16AF1 093C 0062; # (a◌़◌̴◌𖫱b; a◌̴◌𖫱◌़b; a◌̴◌𖫱◌़b; a◌̴◌𖫱◌़b; a◌̴◌𖫱◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, BASSA VAH COMBINING LOW TONE, LATIN SMALL LETTER B diff --git a/lib/stdlib/test/uri_string_SUITE.erl b/lib/stdlib/test/uri_string_SUITE.erl index 4fc0d76be8..ddaead9c7c 100644 --- a/lib/stdlib/test/uri_string_SUITE.erl +++ b/lib/stdlib/test/uri_string_SUITE.erl @@ -862,9 +862,11 @@ transcode_negative(_Config) -> compose_query(_Config) -> [] = uri_string:compose_query([]), "foo=1&bar=2" = uri_string:compose_query([{<<"foo">>,"1"}, {"bar", "2"}]), + "foo=1&bar" = uri_string:compose_query([{<<"foo">>,"1"}, {"bar", true}]), "foo=1&b%C3%A4r=2" = uri_string:compose_query([{"foo","1"}, {"bär", "2"}],[{encoding,utf8}]), "foo=1&b%C3%A4r=2" = uri_string:compose_query([{"foo","1"}, {"bär", "2"}],[{encoding,unicode}]), "foo=1&b%E4r=2" = uri_string:compose_query([{"foo","1"}, {"bär", "2"}],[{encoding,latin1}]), + "foo&b%E4r=2" = uri_string:compose_query([{"foo",true}, {"bär", "2"}],[{encoding,latin1}]), "foo+bar=1&%E5%90%88=2" = uri_string:compose_query([{"foo bar","1"}, {"合", "2"}]), "foo+bar=1&%26%2321512%3B=2" = uri_string:compose_query([{"foo bar","1"}, {"合", "2"}],[{encoding,latin1}]), @@ -906,11 +908,13 @@ dissect_query(_Config) -> [{"föo bar","1"},{"ö","2"}] = uri_string:dissect_query("föo+bar=1&%C3%B6=2"), [{<<"föo bar"/utf8>>,<<"1">>},{<<"ö"/utf8>>,<<"2">>}] = - uri_string:dissect_query(<<"föo+bar=1&%C3%B6=2"/utf8>>). + uri_string:dissect_query(<<"föo+bar=1&%C3%B6=2"/utf8>>), + [{"foo1",true},{"bar","2"}] = + uri_string:dissect_query("foo1&bar=2"), + [{<<"foo1">>,<<"1">>},{<<"bar">>,true}] = + uri_string:dissect_query(<<"foo1=1&bar">>). dissect_query_negative(_Config) -> - {error,missing_value,"&"} = - uri_string:dissect_query("foo1&bar=2"), {error,invalid_percent_encoding,"%XX%B6"} = uri_string:dissect_query("foo=%XX%B6&bar=2"), {error,invalid_input,[153]} = uri_string:dissect_query("foo=%99%B6&bar=2"), diff --git a/lib/stdlib/uc_spec/CaseFolding.txt b/lib/stdlib/uc_spec/CaseFolding.txt index efdf18e441..cce350f49c 100644 --- a/lib/stdlib/uc_spec/CaseFolding.txt +++ b/lib/stdlib/uc_spec/CaseFolding.txt @@ -1,6 +1,6 @@ -# CaseFolding-10.0.0.txt -# Date: 2017-04-14, 05:40:18 GMT -# © 2017 Unicode®, Inc. +# CaseFolding-11.0.0.txt +# Date: 2018-01-31, 08:20:09 GMT +# © 2018 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -603,6 +603,52 @@ 1C86; C; 044A; # CYRILLIC SMALL LETTER TALL HARD SIGN 1C87; C; 0463; # CYRILLIC SMALL LETTER TALL YAT 1C88; C; A64B; # CYRILLIC SMALL LETTER UNBLENDED UK +1C90; C; 10D0; # GEORGIAN MTAVRULI CAPITAL LETTER AN +1C91; C; 10D1; # GEORGIAN MTAVRULI CAPITAL LETTER BAN +1C92; C; 10D2; # GEORGIAN MTAVRULI CAPITAL LETTER GAN +1C93; C; 10D3; # GEORGIAN MTAVRULI CAPITAL LETTER DON +1C94; C; 10D4; # GEORGIAN MTAVRULI CAPITAL LETTER EN +1C95; C; 10D5; # GEORGIAN MTAVRULI CAPITAL LETTER VIN +1C96; C; 10D6; # GEORGIAN MTAVRULI CAPITAL LETTER ZEN +1C97; C; 10D7; # GEORGIAN MTAVRULI CAPITAL LETTER TAN +1C98; C; 10D8; # GEORGIAN MTAVRULI CAPITAL LETTER IN +1C99; C; 10D9; # GEORGIAN MTAVRULI CAPITAL LETTER KAN +1C9A; C; 10DA; # GEORGIAN MTAVRULI CAPITAL LETTER LAS +1C9B; C; 10DB; # GEORGIAN MTAVRULI CAPITAL LETTER MAN +1C9C; C; 10DC; # GEORGIAN MTAVRULI CAPITAL LETTER NAR +1C9D; C; 10DD; # GEORGIAN MTAVRULI CAPITAL LETTER ON +1C9E; C; 10DE; # GEORGIAN MTAVRULI CAPITAL LETTER PAR +1C9F; C; 10DF; # GEORGIAN MTAVRULI CAPITAL LETTER ZHAR +1CA0; C; 10E0; # GEORGIAN MTAVRULI CAPITAL LETTER RAE +1CA1; C; 10E1; # GEORGIAN MTAVRULI CAPITAL LETTER SAN +1CA2; C; 10E2; # GEORGIAN MTAVRULI CAPITAL LETTER TAR +1CA3; C; 10E3; # GEORGIAN MTAVRULI CAPITAL LETTER UN +1CA4; C; 10E4; # GEORGIAN MTAVRULI CAPITAL LETTER PHAR +1CA5; C; 10E5; # GEORGIAN MTAVRULI CAPITAL LETTER KHAR +1CA6; C; 10E6; # GEORGIAN MTAVRULI CAPITAL LETTER GHAN +1CA7; C; 10E7; # GEORGIAN MTAVRULI CAPITAL LETTER QAR +1CA8; C; 10E8; # GEORGIAN MTAVRULI CAPITAL LETTER SHIN +1CA9; C; 10E9; # GEORGIAN MTAVRULI CAPITAL LETTER CHIN +1CAA; C; 10EA; # GEORGIAN MTAVRULI CAPITAL LETTER CAN +1CAB; C; 10EB; # GEORGIAN MTAVRULI CAPITAL LETTER JIL +1CAC; C; 10EC; # GEORGIAN MTAVRULI CAPITAL LETTER CIL +1CAD; C; 10ED; # GEORGIAN MTAVRULI CAPITAL LETTER CHAR +1CAE; C; 10EE; # GEORGIAN MTAVRULI CAPITAL LETTER XAN +1CAF; C; 10EF; # GEORGIAN MTAVRULI CAPITAL LETTER JHAN +1CB0; C; 10F0; # GEORGIAN MTAVRULI CAPITAL LETTER HAE +1CB1; C; 10F1; # GEORGIAN MTAVRULI CAPITAL LETTER HE +1CB2; C; 10F2; # GEORGIAN MTAVRULI CAPITAL LETTER HIE +1CB3; C; 10F3; # GEORGIAN MTAVRULI CAPITAL LETTER WE +1CB4; C; 10F4; # GEORGIAN MTAVRULI CAPITAL LETTER HAR +1CB5; C; 10F5; # GEORGIAN MTAVRULI CAPITAL LETTER HOE +1CB6; C; 10F6; # GEORGIAN MTAVRULI CAPITAL LETTER FI +1CB7; C; 10F7; # GEORGIAN MTAVRULI CAPITAL LETTER YN +1CB8; C; 10F8; # GEORGIAN MTAVRULI CAPITAL LETTER ELIFI +1CB9; C; 10F9; # GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN +1CBA; C; 10FA; # GEORGIAN MTAVRULI CAPITAL LETTER AIN +1CBD; C; 10FD; # GEORGIAN MTAVRULI CAPITAL LETTER AEN +1CBE; C; 10FE; # GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN +1CBF; C; 10FF; # GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN 1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW 1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE 1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW @@ -1180,6 +1226,7 @@ A7B2; C; 029D; # LATIN CAPITAL LETTER J WITH CROSSED-TAIL A7B3; C; AB53; # LATIN CAPITAL LETTER CHI A7B4; C; A7B5; # LATIN CAPITAL LETTER BETA A7B6; C; A7B7; # LATIN CAPITAL LETTER OMEGA +A7B8; C; A7B9; # LATIN CAPITAL LETTER U WITH STROKE AB70; C; 13A0; # CHEROKEE SMALL LETTER A AB71; C; 13A1; # CHEROKEE SMALL LETTER E AB72; C; 13A2; # CHEROKEE SMALL LETTER I @@ -1457,6 +1504,38 @@ FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z 118BD; C; 118DD; # WARANG CITI CAPITAL LETTER SSUU 118BE; C; 118DE; # WARANG CITI CAPITAL LETTER SII 118BF; C; 118DF; # WARANG CITI CAPITAL LETTER VIYO +16E40; C; 16E60; # MEDEFAIDRIN CAPITAL LETTER M +16E41; C; 16E61; # MEDEFAIDRIN CAPITAL LETTER S +16E42; C; 16E62; # MEDEFAIDRIN CAPITAL LETTER V +16E43; C; 16E63; # MEDEFAIDRIN CAPITAL LETTER W +16E44; C; 16E64; # MEDEFAIDRIN CAPITAL LETTER ATIU +16E45; C; 16E65; # MEDEFAIDRIN CAPITAL LETTER Z +16E46; C; 16E66; # MEDEFAIDRIN CAPITAL LETTER KP +16E47; C; 16E67; # MEDEFAIDRIN CAPITAL LETTER P +16E48; C; 16E68; # MEDEFAIDRIN CAPITAL LETTER T +16E49; C; 16E69; # MEDEFAIDRIN CAPITAL LETTER G +16E4A; C; 16E6A; # MEDEFAIDRIN CAPITAL LETTER F +16E4B; C; 16E6B; # MEDEFAIDRIN CAPITAL LETTER I +16E4C; C; 16E6C; # MEDEFAIDRIN CAPITAL LETTER K +16E4D; C; 16E6D; # MEDEFAIDRIN CAPITAL LETTER A +16E4E; C; 16E6E; # MEDEFAIDRIN CAPITAL LETTER J +16E4F; C; 16E6F; # MEDEFAIDRIN CAPITAL LETTER E +16E50; C; 16E70; # MEDEFAIDRIN CAPITAL LETTER B +16E51; C; 16E71; # MEDEFAIDRIN CAPITAL LETTER C +16E52; C; 16E72; # MEDEFAIDRIN CAPITAL LETTER U +16E53; C; 16E73; # MEDEFAIDRIN CAPITAL LETTER YU +16E54; C; 16E74; # MEDEFAIDRIN CAPITAL LETTER L +16E55; C; 16E75; # MEDEFAIDRIN CAPITAL LETTER Q +16E56; C; 16E76; # MEDEFAIDRIN CAPITAL LETTER HP +16E57; C; 16E77; # MEDEFAIDRIN CAPITAL LETTER NY +16E58; C; 16E78; # MEDEFAIDRIN CAPITAL LETTER X +16E59; C; 16E79; # MEDEFAIDRIN CAPITAL LETTER D +16E5A; C; 16E7A; # MEDEFAIDRIN CAPITAL LETTER OE +16E5B; C; 16E7B; # MEDEFAIDRIN CAPITAL LETTER N +16E5C; C; 16E7C; # MEDEFAIDRIN CAPITAL LETTER R +16E5D; C; 16E7D; # MEDEFAIDRIN CAPITAL LETTER O +16E5E; C; 16E7E; # MEDEFAIDRIN CAPITAL LETTER AI +16E5F; C; 16E7F; # MEDEFAIDRIN CAPITAL LETTER Y 1E900; C; 1E922; # ADLAM CAPITAL LETTER ALIF 1E901; C; 1E923; # ADLAM CAPITAL LETTER DAALI 1E902; C; 1E924; # ADLAM CAPITAL LETTER LAAM diff --git a/lib/stdlib/uc_spec/CompositionExclusions.txt b/lib/stdlib/uc_spec/CompositionExclusions.txt index ff42508686..ea63595bd3 100644 --- a/lib/stdlib/uc_spec/CompositionExclusions.txt +++ b/lib/stdlib/uc_spec/CompositionExclusions.txt @@ -1,5 +1,5 @@ -# CompositionExclusions-10.0.0.txt -# Date: 2017-02-15, 00:00:00 GMT [KW, LI] +# CompositionExclusions-11.0.0.txt +# Date: 2017-12-06, 00:00:00 GMT [KW, LI] # © 2017 Unicode®, Inc. # For terms of use, see http://www.unicode.org/terms_of_use.html # diff --git a/lib/stdlib/uc_spec/GraphemeBreakProperty.txt b/lib/stdlib/uc_spec/GraphemeBreakProperty.txt index 32bb12e47e..52052e6e33 100644 --- a/lib/stdlib/uc_spec/GraphemeBreakProperty.txt +++ b/lib/stdlib/uc_spec/GraphemeBreakProperty.txt @@ -1,6 +1,6 @@ -# GraphemeBreakProperty-10.0.0.txt -# Date: 2017-03-12, 07:03:41 GMT -# © 2017 Unicode®, Inc. +# GraphemeBreakProperty-11.0.0.txt +# Date: 2018-03-16, 20:34:02 GMT +# © 2018 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -24,12 +24,13 @@ 08E2 ; Prepend # Cf ARABIC DISPUTED END OF AYAH 0D4E ; Prepend # Lo MALAYALAM LETTER DOT REPH 110BD ; Prepend # Cf KAITHI NUMBER SIGN +110CD ; Prepend # Cf KAITHI NUMBER SIGN ABOVE 111C2..111C3 ; Prepend # Lo [2] SHARADA SIGN JIHVAMULIYA..SHARADA SIGN UPADHMANIYA 11A3A ; Prepend # Lo ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA 11A86..11A89 ; Prepend # Lo [4] SOYOMBO CLUSTER-INITIAL LETTER RA..SOYOMBO CLUSTER-INITIAL LETTER SA 11D46 ; Prepend # Lo MASARAM GONDI REPHA -# Total code points: 19 +# Total code points: 20 # ================================================ @@ -95,12 +96,13 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF> 0730..074A ; Extend # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH 07A6..07B0 ; Extend # Mn [11] THAANA ABAFILI..THAANA SUKUN 07EB..07F3 ; Extend # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE +07FD ; Extend # Mn NKO DANTAYALAN 0816..0819 ; Extend # Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH 081B..0823 ; Extend # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A 0825..0827 ; Extend # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U 0829..082D ; Extend # Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA 0859..085B ; Extend # Mn [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK -08D4..08E1 ; Extend # Mn [14] ARABIC SMALL HIGH WORD AR-RUB..ARABIC SMALL HIGH SIGN SAFHA +08D3..08E1 ; Extend # Mn [15] ARABIC SMALL LOW WAW..ARABIC SMALL HIGH SIGN SAFHA 08E3..0902 ; Extend # Mn [32] ARABIC TURNED DAMMA BELOW..DEVANAGARI SIGN ANUSVARA 093A ; Extend # Mn DEVANAGARI VOWEL SIGN OE 093C ; Extend # Mn DEVANAGARI SIGN NUKTA @@ -115,6 +117,7 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF> 09CD ; Extend # Mn BENGALI SIGN VIRAMA 09D7 ; Extend # Mc BENGALI AU LENGTH MARK 09E2..09E3 ; Extend # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +09FE ; Extend # Mn BENGALI SANDHI MARK 0A01..0A02 ; Extend # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI 0A3C ; Extend # Mn GURMUKHI SIGN NUKTA 0A41..0A42 ; Extend # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU @@ -145,6 +148,7 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF> 0BCD ; Extend # Mn TAMIL SIGN VIRAMA 0BD7 ; Extend # Mc TAMIL AU LENGTH MARK 0C00 ; Extend # Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE +0C04 ; Extend # Mn TELUGU SIGN COMBINING ANUSVARA ABOVE 0C3E..0C40 ; Extend # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II 0C46..0C48 ; Extend # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI 0C4A..0C4D ; Extend # Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA @@ -273,6 +277,7 @@ A80B ; Extend # Mn SYLOTI NAGRI SIGN ANUSVARA A825..A826 ; Extend # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E A8C4..A8C5 ; Extend # Mn [2] SAURASHTRA SIGN VIRAMA..SAURASHTRA SIGN CANDRABINDU A8E0..A8F1 ; Extend # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA +A8FF ; Extend # Mn DEVANAGARI VOWEL SIGN AY A926..A92D ; Extend # Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU A947..A951 ; Extend # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R A980..A982 ; Extend # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR @@ -309,6 +314,8 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 10A38..10A3A ; Extend # Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW 10A3F ; Extend # Mn KHAROSHTHI VIRAMA 10AE5..10AE6 ; Extend # Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW +10D24..10D27 ; Extend # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI +10F46..10F50 ; Extend # Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW 11001 ; Extend # Mn BRAHMI SIGN ANUSVARA 11038..11046 ; Extend # Mn [15] BRAHMI VOWEL SIGN AA..BRAHMI VIRAMA 1107F..11081 ; Extend # Mn [3] BRAHMI NUMBER JOINER..KAITHI SIGN ANUSVARA @@ -320,7 +327,7 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 11173 ; Extend # Mn MAHAJANI SIGN NUKTA 11180..11181 ; Extend # Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA 111B6..111BE ; Extend # Mn [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O -111CA..111CC ; Extend # Mn [3] SHARADA SIGN NUKTA..SHARADA EXTRA SHORT VOWEL MARK +111C9..111CC ; Extend # Mn [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK 1122F..11231 ; Extend # Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI 11234 ; Extend # Mn KHOJKI SIGN ANUSVARA 11236..11237 ; Extend # Mn [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA @@ -328,7 +335,7 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 112DF ; Extend # Mn KHUDAWADI SIGN ANUSVARA 112E3..112EA ; Extend # Mn [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA 11300..11301 ; Extend # Mn [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU -1133C ; Extend # Mn GRANTHA SIGN NUKTA +1133B..1133C ; Extend # Mn [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA 1133E ; Extend # Mc GRANTHA VOWEL SIGN AA 11340 ; Extend # Mn GRANTHA VOWEL SIGN II 11357 ; Extend # Mc GRANTHA AU LENGTH MARK @@ -337,6 +344,7 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 11438..1143F ; Extend # Mn [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI 11442..11444 ; Extend # Mn [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA 11446 ; Extend # Mn NEWA SIGN NUKTA +1145E ; Extend # Mn NEWA SANDHI MARK 114B0 ; Extend # Mc TIRHUTA VOWEL SIGN AA 114B3..114B8 ; Extend # Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL 114BA ; Extend # Mn TIRHUTA VOWEL SIGN SHORT E @@ -358,8 +366,9 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 1171D..1171F ; Extend # Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA 11722..11725 ; Extend # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU 11727..1172B ; Extend # Mn [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER -11A01..11A06 ; Extend # Mn [6] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL SIGN O -11A09..11A0A ; Extend # Mn [2] ZANABAZAR SQUARE VOWEL SIGN REVERSED I..ZANABAZAR SQUARE VOWEL LENGTH MARK +1182F..11837 ; Extend # Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA +11839..1183A ; Extend # Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA +11A01..11A0A ; Extend # Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK 11A33..11A38 ; Extend # Mn [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA 11A3B..11A3E ; Extend # Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA 11A47 ; Extend # Mn ZANABAZAR SQUARE SUBJOINER @@ -379,6 +388,10 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 11D3C..11D3D ; Extend # Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O 11D3F..11D45 ; Extend # Mn [7] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI VIRAMA 11D47 ; Extend # Mn MASARAM GONDI RA-KARA +11D90..11D91 ; Extend # Mn [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI +11D95 ; Extend # Mn GUNJALA GONDI SIGN ANUSVARA +11D97 ; Extend # Mn GUNJALA GONDI VIRAMA +11EF3..11EF4 ; Extend # Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U 16AF0..16AF4 ; Extend # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE 16B30..16B36 ; Extend # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM 16F8F..16F92 ; Extend # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW @@ -403,10 +416,11 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 1E026..1E02A ; Extend # Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA 1E8D0..1E8D6 ; Extend # Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS 1E944..1E94A ; Extend # Mn [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA +1F3FB..1F3FF ; Extend # Sk [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 E0020..E007F ; Extend # Cf [96] TAG SPACE..CANCEL TAG E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 -# Total code points: 1901 +# Total code points: 1948 # ================================================ @@ -517,6 +531,7 @@ ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK 110B0..110B2 ; SpacingMark # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II 110B7..110B8 ; SpacingMark # Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU 1112C ; SpacingMark # Mc CHAKMA VOWEL SIGN E +11145..11146 ; SpacingMark # Mc [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI 11182 ; SpacingMark # Mc SHARADA SIGN VISARGA 111B3..111B5 ; SpacingMark # Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II 111BF..111C0 ; SpacingMark # Mc [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA @@ -549,7 +564,8 @@ ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK 116B6 ; SpacingMark # Mc TAKRI SIGN VIRAMA 11720..11721 ; SpacingMark # Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA 11726 ; SpacingMark # Mc AHOM VOWEL SIGN E -11A07..11A08 ; SpacingMark # Mc [2] ZANABAZAR SQUARE VOWEL SIGN AI..ZANABAZAR SQUARE VOWEL SIGN AU +1182C..1182E ; SpacingMark # Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II +11838 ; SpacingMark # Mc DOGRA SIGN VISARGA 11A39 ; SpacingMark # Mc ZANABAZAR SQUARE SIGN VISARGA 11A57..11A58 ; SpacingMark # Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU 11A97 ; SpacingMark # Mc SOYOMBO SIGN VISARGA @@ -558,11 +574,15 @@ ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK 11CA9 ; SpacingMark # Mc MARCHEN SUBJOINED LETTER YA 11CB1 ; SpacingMark # Mc MARCHEN VOWEL SIGN I 11CB4 ; SpacingMark # Mc MARCHEN VOWEL SIGN O +11D8A..11D8E ; SpacingMark # Mc [5] GUNJALA GONDI VOWEL SIGN AA..GUNJALA GONDI VOWEL SIGN UU +11D93..11D94 ; SpacingMark # Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU +11D96 ; SpacingMark # Mc GUNJALA GONDI SIGN VISARGA +11EF5..11EF6 ; SpacingMark # Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O 16F51..16F7E ; SpacingMark # Mc [46] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN NG 1D166 ; SpacingMark # Mc MUSICAL SYMBOL COMBINING SPRECHGESANG STEM 1D16D ; SpacingMark # Mc MUSICAL SYMBOL COMBINING AUGMENTATION DOT -# Total code points: 348 +# Total code points: 362 # ================================================ @@ -1395,81 +1415,8 @@ D789..D7A3 ; LVT # Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH # ================================================ -261D ; E_Base # So WHITE UP POINTING INDEX -26F9 ; E_Base # So PERSON WITH BALL -270A..270D ; E_Base # So [4] RAISED FIST..WRITING HAND -1F385 ; E_Base # So FATHER CHRISTMAS -1F3C2..1F3C4 ; E_Base # So [3] SNOWBOARDER..SURFER -1F3C7 ; E_Base # So HORSE RACING -1F3CA..1F3CC ; E_Base # So [3] SWIMMER..GOLFER -1F442..1F443 ; E_Base # So [2] EAR..NOSE -1F446..1F450 ; E_Base # So [11] WHITE UP POINTING BACKHAND INDEX..OPEN HANDS SIGN -1F46E ; E_Base # So POLICE OFFICER -1F470..1F478 ; E_Base # So [9] BRIDE WITH VEIL..PRINCESS -1F47C ; E_Base # So BABY ANGEL -1F481..1F483 ; E_Base # So [3] INFORMATION DESK PERSON..DANCER -1F485..1F487 ; E_Base # So [3] NAIL POLISH..HAIRCUT -1F4AA ; E_Base # So FLEXED BICEPS -1F574..1F575 ; E_Base # So [2] MAN IN BUSINESS SUIT LEVITATING..SLEUTH OR SPY -1F57A ; E_Base # So MAN DANCING -1F590 ; E_Base # So RAISED HAND WITH FINGERS SPLAYED -1F595..1F596 ; E_Base # So [2] REVERSED HAND WITH MIDDLE FINGER EXTENDED..RAISED HAND WITH PART BETWEEN MIDDLE AND RING FINGERS -1F645..1F647 ; E_Base # So [3] FACE WITH NO GOOD GESTURE..PERSON BOWING DEEPLY -1F64B..1F64F ; E_Base # So [5] HAPPY PERSON RAISING ONE HAND..PERSON WITH FOLDED HANDS -1F6A3 ; E_Base # So ROWBOAT -1F6B4..1F6B6 ; E_Base # So [3] BICYCLIST..PEDESTRIAN -1F6C0 ; E_Base # So BATH -1F6CC ; E_Base # So SLEEPING ACCOMMODATION -1F918..1F91C ; E_Base # So [5] SIGN OF THE HORNS..RIGHT-FACING FIST -1F91E..1F91F ; E_Base # So [2] HAND WITH INDEX AND MIDDLE FINGERS CROSSED..I LOVE YOU HAND SIGN -1F926 ; E_Base # So FACE PALM -1F930..1F939 ; E_Base # So [10] PREGNANT WOMAN..JUGGLING -1F93D..1F93E ; E_Base # So [2] WATER POLO..HANDBALL -1F9D1..1F9DD ; E_Base # So [13] ADULT..ELF - -# Total code points: 98 - -# ================================================ - -1F3FB..1F3FF ; E_Modifier # Sk [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 - -# Total code points: 5 - -# ================================================ - 200D ; ZWJ # Cf ZERO WIDTH JOINER # Total code points: 1 -# ================================================ - -2640 ; Glue_After_Zwj # So FEMALE SIGN -2642 ; Glue_After_Zwj # So MALE SIGN -2695..2696 ; Glue_After_Zwj # So [2] STAFF OF AESCULAPIUS..SCALES -2708 ; Glue_After_Zwj # So AIRPLANE -2764 ; Glue_After_Zwj # So HEAVY BLACK HEART -1F308 ; Glue_After_Zwj # So RAINBOW -1F33E ; Glue_After_Zwj # So EAR OF RICE -1F373 ; Glue_After_Zwj # So COOKING -1F393 ; Glue_After_Zwj # So GRADUATION CAP -1F3A4 ; Glue_After_Zwj # So MICROPHONE -1F3A8 ; Glue_After_Zwj # So ARTIST PALETTE -1F3EB ; Glue_After_Zwj # So SCHOOL -1F3ED ; Glue_After_Zwj # So FACTORY -1F48B ; Glue_After_Zwj # So KISS MARK -1F4BB..1F4BC ; Glue_After_Zwj # So [2] PERSONAL COMPUTER..BRIEFCASE -1F527 ; Glue_After_Zwj # So WRENCH -1F52C ; Glue_After_Zwj # So MICROSCOPE -1F5E8 ; Glue_After_Zwj # So LEFT SPEECH BUBBLE -1F680 ; Glue_After_Zwj # So ROCKET -1F692 ; Glue_After_Zwj # So FIRE ENGINE - -# Total code points: 22 - -# ================================================ - -1F466..1F469 ; E_Base_GAZ # So [4] BOY..WOMAN - -# Total code points: 4 - # EOF diff --git a/lib/stdlib/uc_spec/PropList.txt b/lib/stdlib/uc_spec/PropList.txt index 9a2d0e4b1c..ef86795abe 100644 --- a/lib/stdlib/uc_spec/PropList.txt +++ b/lib/stdlib/uc_spec/PropList.txt @@ -1,6 +1,6 @@ -# PropList-10.0.0.txt -# Date: 2017-03-10, 08:25:30 GMT -# © 2017 Unicode®, Inc. +# PropList-11.0.0.txt +# Date: 2018-03-15, 04:28:35 GMT +# © 2018 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -125,7 +125,7 @@ FF63 ; Quotation_Mark # Pe HALFWIDTH RIGHT CORNER BRACKET 05C3 ; Terminal_Punctuation # Po HEBREW PUNCTUATION SOF PASUQ 060C ; Terminal_Punctuation # Po ARABIC COMMA 061B ; Terminal_Punctuation # Po ARABIC SEMICOLON -061F ; Terminal_Punctuation # Po ARABIC QUESTION MARK +061E..061F ; Terminal_Punctuation # Po [2] ARABIC TRIPLE DOT PUNCTUATION MARK..ARABIC QUESTION MARK 06D4 ; Terminal_Punctuation # Po ARABIC FULL STOP 0700..070A ; Terminal_Punctuation # Po [11] SYRIAC END OF PARAGRAPH..SYRIAC CONTRACTION 070C ; Terminal_Punctuation # Po SYRIAC HARKLEAN METOBELUS @@ -156,6 +156,8 @@ FF63 ; Quotation_Mark # Pe HALFWIDTH RIGHT CORNER BRACKET 2E2E ; Terminal_Punctuation # Po REVERSED QUESTION MARK 2E3C ; Terminal_Punctuation # Po STENOGRAPHIC FULL STOP 2E41 ; Terminal_Punctuation # Po REVERSED COMMA +2E4C ; Terminal_Punctuation # Po MEDIEVAL COMMA +2E4E ; Terminal_Punctuation # Po PUNCTUS ELEVATUS MARK 3001..3002 ; Terminal_Punctuation # Po [2] IDEOGRAPHIC COMMA..IDEOGRAPHIC FULL STOP A4FE..A4FF ; Terminal_Punctuation # Po [2] LISU PUNCTUATION COMMA..LISU PUNCTUATION FULL STOP A60D..A60F ; Terminal_Punctuation # Po [3] VAI COMMA..VAI QUESTION MARK @@ -185,6 +187,7 @@ FF64 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC COMMA 10AF0..10AF5 ; Terminal_Punctuation # Po [6] MANICHAEAN PUNCTUATION STAR..MANICHAEAN PUNCTUATION TWO DOTS 10B3A..10B3F ; Terminal_Punctuation # Po [6] TINY TWO DOTS OVER ONE DOT PUNCTUATION..LARGE ONE RING OVER TWO RINGS PUNCTUATION 10B99..10B9C ; Terminal_Punctuation # Po [4] PSALTER PAHLAVI SECTION MARK..PSALTER PAHLAVI FOUR DOTS WITH DOT +10F55..10F59 ; Terminal_Punctuation # Po [5] SOGDIAN PUNCTUATION TWO VERTICAL BARS..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT 11047..1104D ; Terminal_Punctuation # Po [7] BRAHMI DANDA..BRAHMI PUNCTUATION LOTUS 110BE..110C1 ; Terminal_Punctuation # Po [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA 11141..11143 ; Terminal_Punctuation # Po [3] CHAKMA DANDA..CHAKMA QUESTION MARK @@ -204,15 +207,17 @@ FF64 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC COMMA 11AA1..11AA2 ; Terminal_Punctuation # Po [2] SOYOMBO TERMINAL MARK-1..SOYOMBO TERMINAL MARK-2 11C41..11C43 ; Terminal_Punctuation # Po [3] BHAIKSUKI DANDA..BHAIKSUKI WORD SEPARATOR 11C71 ; Terminal_Punctuation # Po MARCHEN MARK SHAD +11EF7..11EF8 ; Terminal_Punctuation # Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION 12470..12474 ; Terminal_Punctuation # Po [5] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON 16A6E..16A6F ; Terminal_Punctuation # Po [2] MRO DANDA..MRO DOUBLE DANDA 16AF5 ; Terminal_Punctuation # Po BASSA VAH FULL STOP 16B37..16B39 ; Terminal_Punctuation # Po [3] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN CIM CHEEM 16B44 ; Terminal_Punctuation # Po PAHAWH HMONG SIGN XAUS +16E97..16E98 ; Terminal_Punctuation # Po [2] MEDEFAIDRIN COMMA..MEDEFAIDRIN FULL STOP 1BC9F ; Terminal_Punctuation # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP 1DA87..1DA8A ; Terminal_Punctuation # Po [4] SIGNWRITING COMMA..SIGNWRITING COLON -# Total code points: 252 +# Total code points: 264 # ================================================ @@ -661,6 +666,7 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 10A01..10A03 ; Other_Alphabetic # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R 10A05..10A06 ; Other_Alphabetic # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O 10A0C..10A0F ; Other_Alphabetic # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA +10D24..10D27 ; Other_Alphabetic # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI 11000 ; Other_Alphabetic # Mc BRAHMI SIGN CANDRABINDU 11001 ; Other_Alphabetic # Mn BRAHMI SIGN ANUSVARA 11002 ; Other_Alphabetic # Mc BRAHMI SIGN VISARGA @@ -673,6 +679,7 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 11127..1112B ; Other_Alphabetic # Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU 1112C ; Other_Alphabetic # Mc CHAKMA VOWEL SIGN E 1112D..11132 ; Other_Alphabetic # Mn [6] CHAKMA VOWEL SIGN AI..CHAKMA AU MARK +11145..11146 ; Other_Alphabetic # Mc [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI 11180..11181 ; Other_Alphabetic # Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA 11182 ; Other_Alphabetic # Mc SHARADA SIGN VISARGA 111B3..111B5 ; Other_Alphabetic # Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II @@ -730,9 +737,10 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 11722..11725 ; Other_Alphabetic # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU 11726 ; Other_Alphabetic # Mc AHOM VOWEL SIGN E 11727..1172A ; Other_Alphabetic # Mn [4] AHOM VOWEL SIGN AW..AHOM VOWEL SIGN AM -11A01..11A06 ; Other_Alphabetic # Mn [6] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL SIGN O -11A07..11A08 ; Other_Alphabetic # Mc [2] ZANABAZAR SQUARE VOWEL SIGN AI..ZANABAZAR SQUARE VOWEL SIGN AU -11A09..11A0A ; Other_Alphabetic # Mn [2] ZANABAZAR SQUARE VOWEL SIGN REVERSED I..ZANABAZAR SQUARE VOWEL LENGTH MARK +1182C..1182E ; Other_Alphabetic # Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II +1182F..11837 ; Other_Alphabetic # Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA +11838 ; Other_Alphabetic # Mc DOGRA SIGN VISARGA +11A01..11A0A ; Other_Alphabetic # Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK 11A35..11A38 ; Other_Alphabetic # Mn [4] ZANABAZAR SQUARE SIGN CANDRABINDU..ZANABAZAR SQUARE SIGN ANUSVARA 11A39 ; Other_Alphabetic # Mc ZANABAZAR SQUARE SIGN VISARGA 11A3B..11A3E ; Other_Alphabetic # Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA @@ -758,6 +766,13 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 11D3F..11D41 ; Other_Alphabetic # Mn [3] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI SIGN VISARGA 11D43 ; Other_Alphabetic # Mn MASARAM GONDI SIGN CANDRA 11D47 ; Other_Alphabetic # Mn MASARAM GONDI RA-KARA +11D8A..11D8E ; Other_Alphabetic # Mc [5] GUNJALA GONDI VOWEL SIGN AA..GUNJALA GONDI VOWEL SIGN UU +11D90..11D91 ; Other_Alphabetic # Mn [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI +11D93..11D94 ; Other_Alphabetic # Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU +11D95 ; Other_Alphabetic # Mn GUNJALA GONDI SIGN ANUSVARA +11D96 ; Other_Alphabetic # Mc GUNJALA GONDI SIGN VISARGA +11EF3..11EF4 ; Other_Alphabetic # Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U +11EF5..11EF6 ; Other_Alphabetic # Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O 16B30..16B36 ; Other_Alphabetic # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM 16F51..16F7E ; Other_Alphabetic # Mc [46] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN NG 1BC9E ; Other_Alphabetic # Mn DUPLOYAN DOUBLE MARK @@ -771,7 +786,7 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 1F150..1F169 ; Other_Alphabetic # So [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z 1F170..1F189 ; Other_Alphabetic # So [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z -# Total code points: 1300 +# Total code points: 1334 # ================================================ @@ -780,10 +795,10 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 3021..3029 ; Ideographic # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE 3038..303A ; Ideographic # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY 3400..4DB5 ; Ideographic # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 -4E00..9FEA ; Ideographic # Lo [20971] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FEA +4E00..9FEF ; Ideographic # Lo [20976] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FEF F900..FA6D ; Ideographic # Lo [366] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA6D FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 -17000..187EC ; Ideographic # Lo [6125] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187EC +17000..187F1 ; Ideographic # Lo [6130] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F1 18800..18AF2 ; Ideographic # Lo [755] TANGUT COMPONENT-001..TANGUT COMPONENT-755 1B170..1B2FB ; Ideographic # Lo [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB 20000..2A6D6 ; Ideographic # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 @@ -793,7 +808,7 @@ FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COM 2CEB0..2EBE0 ; Ideographic # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 2F800..2FA1D ; Ideographic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D -# Total code points: 96174 +# Total code points: 96184 # ================================================ @@ -953,6 +968,9 @@ FF9E..FF9F ; Diacritic # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFW FFE3 ; Diacritic # Sk FULLWIDTH MACRON 102E0 ; Diacritic # Mn COPTIC EPACT THOUSANDS MARK 10AE5..10AE6 ; Diacritic # Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW +10D22..10D23 ; Diacritic # Lo [2] HANIFI ROHINGYA MARK SAKIN..HANIFI ROHINGYA MARK NA KHONNA +10D24..10D27 ; Diacritic # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI +10F46..10F50 ; Diacritic # Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW 110B9..110BA ; Diacritic # Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA 11133..11134 ; Diacritic # Mn [2] CHAKMA VIRAMA..CHAKMA MAAYYAA 11173 ; Diacritic # Mn MAHAJANI SIGN NUKTA @@ -973,12 +991,14 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON 116B6 ; Diacritic # Mc TAKRI SIGN VIRAMA 116B7 ; Diacritic # Mn TAKRI SIGN NUKTA 1172B ; Diacritic # Mn AHOM SIGN KILLER +11839..1183A ; Diacritic # Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA 11A34 ; Diacritic # Mn ZANABAZAR SQUARE SIGN VIRAMA 11A47 ; Diacritic # Mn ZANABAZAR SQUARE SUBJOINER 11A99 ; Diacritic # Mn SOYOMBO SUBJOINER 11C3F ; Diacritic # Mn BHAIKSUKI SIGN VIRAMA 11D42 ; Diacritic # Mn MASARAM GONDI SIGN NUKTA 11D44..11D45 ; Diacritic # Mn [2] MASARAM GONDI SIGN HALANTA..MASARAM GONDI VIRAMA +11D97 ; Diacritic # Mn GUNJALA GONDI VIRAMA 16AF0..16AF4 ; Diacritic # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE 16F8F..16F92 ; Diacritic # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW 16F93..16F9F ; Diacritic # Lm [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8 @@ -991,7 +1011,7 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON 1E944..1E946 ; Diacritic # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK 1E948..1E94A ; Diacritic # Mn [3] ADLAM CONSONANT MODIFIER..ADLAM NUKTA -# Total code points: 798 +# Total code points: 818 # ================================================ @@ -1137,7 +1157,7 @@ E0020..E007F ; Other_Grapheme_Extend # Cf [96] TAG SPACE..CANCEL TAG # ================================================ 3400..4DB5 ; Unified_Ideograph # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 -4E00..9FEA ; Unified_Ideograph # Lo [20971] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FEA +4E00..9FEF ; Unified_Ideograph # Lo [20976] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FEF FA0E..FA0F ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA0E..CJK COMPATIBILITY IDEOGRAPH-FA0F FA11 ; Unified_Ideograph # Lo CJK COMPATIBILITY IDEOGRAPH-FA11 FA13..FA14 ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA13..CJK COMPATIBILITY IDEOGRAPH-FA14 @@ -1151,7 +1171,7 @@ FA27..FA29 ; Unified_Ideograph # Lo [3] CJK COMPATIBILITY IDEOGRAPH-FA27..C 2B820..2CEA1 ; Unified_Ideograph # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 2CEB0..2EBE0 ; Unified_Ideograph # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 -# Total code points: 87882 +# Total code points: 87887 # ================================================ @@ -1255,10 +1275,13 @@ AABB..AABC ; Logical_Order_Exception # Lo [2] TAI VIET VOWEL AUE..TAI VIET 002E ; Sentence_Terminal # Po FULL STOP 003F ; Sentence_Terminal # Po QUESTION MARK 0589 ; Sentence_Terminal # Po ARMENIAN FULL STOP -061F ; Sentence_Terminal # Po ARABIC QUESTION MARK +061E..061F ; Sentence_Terminal # Po [2] ARABIC TRIPLE DOT PUNCTUATION MARK..ARABIC QUESTION MARK 06D4 ; Sentence_Terminal # Po ARABIC FULL STOP 0700..0702 ; Sentence_Terminal # Po [3] SYRIAC END OF PARAGRAPH..SYRIAC SUBLINEAR FULL STOP 07F9 ; Sentence_Terminal # Po NKO EXCLAMATION MARK +0837 ; Sentence_Terminal # Po SAMARITAN PUNCTUATION MELODIC QITSA +0839 ; Sentence_Terminal # Po SAMARITAN PUNCTUATION QITSA +083D..083E ; Sentence_Terminal # Po [2] SAMARITAN PUNCTUATION SOF MASHFAAT..SAMARITAN PUNCTUATION ANNAAU 0964..0965 ; Sentence_Terminal # Po [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA 104A..104B ; Sentence_Terminal # Po [2] MYANMAR SIGN LITTLE SECTION..MYANMAR SIGN SECTION 1362 ; Sentence_Terminal # Po ETHIOPIC FULL STOP @@ -1296,6 +1319,7 @@ FF0E ; Sentence_Terminal # Po FULLWIDTH FULL STOP FF1F ; Sentence_Terminal # Po FULLWIDTH QUESTION MARK FF61 ; Sentence_Terminal # Po HALFWIDTH IDEOGRAPHIC FULL STOP 10A56..10A57 ; Sentence_Terminal # Po [2] KHAROSHTHI PUNCTUATION DANDA..KHAROSHTHI PUNCTUATION DOUBLE DANDA +10F55..10F59 ; Sentence_Terminal # Po [5] SOGDIAN PUNCTUATION TWO VERTICAL BARS..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT 11047..11048 ; Sentence_Terminal # Po [2] BRAHMI DANDA..BRAHMI DOUBLE DANDA 110BE..110C1 ; Sentence_Terminal # Po [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA 11141..11143 ; Sentence_Terminal # Po [3] CHAKMA DANDA..CHAKMA QUESTION MARK @@ -1313,14 +1337,16 @@ FF61 ; Sentence_Terminal # Po HALFWIDTH IDEOGRAPHIC FULL STOP 11A42..11A43 ; Sentence_Terminal # Po [2] ZANABAZAR SQUARE MARK SHAD..ZANABAZAR SQUARE MARK DOUBLE SHAD 11A9B..11A9C ; Sentence_Terminal # Po [2] SOYOMBO MARK SHAD..SOYOMBO MARK DOUBLE SHAD 11C41..11C42 ; Sentence_Terminal # Po [2] BHAIKSUKI DANDA..BHAIKSUKI DOUBLE DANDA +11EF7..11EF8 ; Sentence_Terminal # Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION 16A6E..16A6F ; Sentence_Terminal # Po [2] MRO DANDA..MRO DOUBLE DANDA 16AF5 ; Sentence_Terminal # Po BASSA VAH FULL STOP 16B37..16B38 ; Sentence_Terminal # Po [2] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN VOS TSHAB CEEB 16B44 ; Sentence_Terminal # Po PAHAWH HMONG SIGN XAUS +16E98 ; Sentence_Terminal # Po MEDEFAIDRIN FULL STOP 1BC9F ; Sentence_Terminal # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP 1DA88 ; Sentence_Terminal # Po SIGNWRITING FULL STOP -# Total code points: 128 +# Total code points: 141 # ================================================ @@ -1521,14 +1547,10 @@ E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION S 2B74..2B75 ; Pattern_Syntax # Cn [2] <reserved-2B74>..<reserved-2B75> 2B76..2B95 ; Pattern_Syntax # So [32] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..RIGHTWARDS BLACK ARROW 2B96..2B97 ; Pattern_Syntax # Cn [2] <reserved-2B96>..<reserved-2B97> -2B98..2BB9 ; Pattern_Syntax # So [34] THREE-D TOP-LIGHTED LEFTWARDS EQUILATERAL ARROWHEAD..UP ARROWHEAD IN A RECTANGLE BOX -2BBA..2BBC ; Pattern_Syntax # Cn [3] <reserved-2BBA>..<reserved-2BBC> -2BBD..2BC8 ; Pattern_Syntax # So [12] BALLOT BOX WITH LIGHT X..BLACK MEDIUM RIGHT-POINTING TRIANGLE CENTRED +2B98..2BC8 ; Pattern_Syntax # So [49] THREE-D TOP-LIGHTED LEFTWARDS EQUILATERAL ARROWHEAD..BLACK MEDIUM RIGHT-POINTING TRIANGLE CENTRED 2BC9 ; Pattern_Syntax # Cn <reserved-2BC9> -2BCA..2BD2 ; Pattern_Syntax # So [9] TOP HALF BLACK CIRCLE..GROUP MARK -2BD3..2BEB ; Pattern_Syntax # Cn [25] <reserved-2BD3>..<reserved-2BEB> -2BEC..2BEF ; Pattern_Syntax # So [4] LEFTWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS..DOWNWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS -2BF0..2BFF ; Pattern_Syntax # Cn [16] <reserved-2BF0>..<reserved-2BFF> +2BCA..2BFE ; Pattern_Syntax # So [53] TOP HALF BLACK CIRCLE..REVERSED RIGHT ANGLE +2BFF ; Pattern_Syntax # Cn <reserved-2BFF> 2E00..2E01 ; Pattern_Syntax # Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER 2E02 ; Pattern_Syntax # Pi LEFT SUBSTITUTION BRACKET 2E03 ; Pattern_Syntax # Pf RIGHT SUBSTITUTION BRACKET @@ -1566,8 +1588,8 @@ E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION S 2E40 ; Pattern_Syntax # Pd DOUBLE HYPHEN 2E41 ; Pattern_Syntax # Po REVERSED COMMA 2E42 ; Pattern_Syntax # Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK -2E43..2E49 ; Pattern_Syntax # Po [7] DASH WITH LEFT UPTURN..DOUBLE STACKED COMMA -2E4A..2E7F ; Pattern_Syntax # Cn [54] <reserved-2E4A>..<reserved-2E7F> +2E43..2E4E ; Pattern_Syntax # Po [12] DASH WITH LEFT UPTURN..PUNCTUS ELEVATUS MARK +2E4F..2E7F ; Pattern_Syntax # Cn [49] <reserved-2E4F>..<reserved-2E7F> 3001..3003 ; Pattern_Syntax # Po [3] IDEOGRAPHIC COMMA..DITTO MARK 3008 ; Pattern_Syntax # Ps LEFT ANGLE BRACKET 3009 ; Pattern_Syntax # Pe RIGHT ANGLE BRACKET @@ -1606,8 +1628,9 @@ FE45..FE46 ; Pattern_Syntax # Po [2] SESAME DOT..WHITE SESAME DOT 070F ; Prepended_Concatenation_Mark # Cf SYRIAC ABBREVIATION MARK 08E2 ; Prepended_Concatenation_Mark # Cf ARABIC DISPUTED END OF AYAH 110BD ; Prepended_Concatenation_Mark # Cf KAITHI NUMBER SIGN +110CD ; Prepended_Concatenation_Mark # Cf KAITHI NUMBER SIGN ABOVE -# Total code points: 10 +# Total code points: 11 # ================================================ diff --git a/lib/stdlib/uc_spec/README-UPDATE.txt b/lib/stdlib/uc_spec/README-UPDATE.txt index d59337e1a5..e1f5c8fcd0 100644 --- a/lib/stdlib/uc_spec/README-UPDATE.txt +++ b/lib/stdlib/uc_spec/README-UPDATE.txt @@ -1,7 +1,13 @@ When updating the unicode version copy the necessary files to this directory. - And update the test files in stdlib/test/unicode_util_SUITE_data/* +Unicode 11 was updated from: +https://www.unicode.org/Public/11.0.0/ucd/ +https://www.unicode.org/Public/11.0.0/ucd/auxiliary/ +https://www.unicode.org/Public/emoji/11.0/ + Update the spec_version(..) function in the generator, gen_unicode_mod.escript + + diff --git a/lib/stdlib/uc_spec/SpecialCasing.txt b/lib/stdlib/uc_spec/SpecialCasing.txt index b9ba0d81c1..c90d09acb3 100644 --- a/lib/stdlib/uc_spec/SpecialCasing.txt +++ b/lib/stdlib/uc_spec/SpecialCasing.txt @@ -1,6 +1,6 @@ -# SpecialCasing-10.0.0.txt -# Date: 2017-04-14, 05:40:43 GMT -# © 2017 Unicode®, Inc. +# SpecialCasing-11.0.0.txt +# Date: 2018-02-22, 06:16:47 GMT +# © 2018 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -121,7 +121,7 @@ FB17; FB17; 0544 056D; 0544 053D; # ARMENIAN SMALL LIGATURE MEN XEH # The following cases are already in the UnicodeData.txt file, so are only commented here. -# 0345; 0345; 0345; 0399; # COMBINING GREEK YPOGEGRAMMENI +# 0345; 0345; 0399; 0399; # COMBINING GREEK YPOGEGRAMMENI # All letters with YPOGEGRAMMENI (iota-subscript) or PROSGEGRAMMENI (iota adscript) # have special uppercases. diff --git a/lib/stdlib/uc_spec/UnicodeData.txt b/lib/stdlib/uc_spec/UnicodeData.txt index d89c64f526..ec32fafbce 100644 --- a/lib/stdlib/uc_spec/UnicodeData.txt +++ b/lib/stdlib/uc_spec/UnicodeData.txt @@ -1362,6 +1362,7 @@ 055D;ARMENIAN COMMA;Po;0;L;;;;;N;;;;; 055E;ARMENIAN QUESTION MARK;Po;0;L;;;;;N;;;;; 055F;ARMENIAN ABBREVIATION MARK;Po;0;L;;;;;N;;;;; +0560;ARMENIAN SMALL LETTER TURNED AYB;Ll;0;L;;;;;N;;;;; 0561;ARMENIAN SMALL LETTER AYB;Ll;0;L;;;;;N;;;0531;;0531 0562;ARMENIAN SMALL LETTER BEN;Ll;0;L;;;;;N;;;0532;;0532 0563;ARMENIAN SMALL LETTER GIM;Ll;0;L;;;;;N;;;0533;;0533 @@ -1401,6 +1402,7 @@ 0585;ARMENIAN SMALL LETTER OH;Ll;0;L;;;;;N;;;0555;;0555 0586;ARMENIAN SMALL LETTER FEH;Ll;0;L;;;;;N;;;0556;;0556 0587;ARMENIAN SMALL LIGATURE ECH YIWN;Ll;0;L;<compat> 0565 0582;;;;N;;;;; +0588;ARMENIAN SMALL LETTER YI WITH STROKE;Ll;0;L;;;;;N;;;;; 0589;ARMENIAN FULL STOP;Po;0;L;;;;;N;ARMENIAN PERIOD;;;; 058A;ARMENIAN HYPHEN;Pd;0;ON;;;;;N;;;;; 058D;RIGHT-FACING ARMENIAN ETERNITY SIGN;So;0;ON;;;;;N;;;;; @@ -1488,6 +1490,7 @@ 05E8;HEBREW LETTER RESH;Lo;0;R;;;;;N;;;;; 05E9;HEBREW LETTER SHIN;Lo;0;R;;;;;N;;;;; 05EA;HEBREW LETTER TAV;Lo;0;R;;;;;N;;;;; +05EF;HEBREW YOD TRIANGLE;Lo;0;R;;;;;N;;;;; 05F0;HEBREW LIGATURE YIDDISH DOUBLE VAV;Lo;0;R;;;;;N;HEBREW LETTER DOUBLE VAV;;;; 05F1;HEBREW LIGATURE YIDDISH VAV YOD;Lo;0;R;;;;;N;HEBREW LETTER VAV YOD;;;; 05F2;HEBREW LIGATURE YIDDISH DOUBLE YOD;Lo;0;R;;;;;N;HEBREW LETTER DOUBLE YOD;;;; @@ -1982,6 +1985,9 @@ 07F8;NKO COMMA;Po;0;ON;;;;;N;;;;; 07F9;NKO EXCLAMATION MARK;Po;0;ON;;;;;N;;;;; 07FA;NKO LAJANYALAN;Lm;0;R;;;;;N;;;;; +07FD;NKO DANTAYALAN;Mn;220;NSM;;;;;N;;;;; +07FE;NKO DOROME SIGN;Sc;0;R;;;;;N;;;;; +07FF;NKO TAMAN SIGN;Sc;0;R;;;;;N;;;;; 0800;SAMARITAN LETTER ALAF;Lo;0;R;;;;;N;;;;; 0801;SAMARITAN LETTER BIT;Lo;0;R;;;;;N;;;;; 0802;SAMARITAN LETTER GAMAN;Lo;0;R;;;;;N;;;;; @@ -2112,6 +2118,7 @@ 08BB;ARABIC LETTER AFRICAN FEH;Lo;0;AL;;;;;N;;;;; 08BC;ARABIC LETTER AFRICAN QAF;Lo;0;AL;;;;;N;;;;; 08BD;ARABIC LETTER AFRICAN NOON;Lo;0;AL;;;;;N;;;;; +08D3;ARABIC SMALL LOW WAW;Mn;220;NSM;;;;;N;;;;; 08D4;ARABIC SMALL HIGH WORD AR-RUB;Mn;230;NSM;;;;;N;;;;; 08D5;ARABIC SMALL HIGH SAD;Mn;230;NSM;;;;;N;;;;; 08D6;ARABIC SMALL HIGH AIN;Mn;230;NSM;;;;;N;;;;; @@ -2379,6 +2386,7 @@ 09FB;BENGALI GANDA MARK;Sc;0;ET;;;;;N;;;;; 09FC;BENGALI LETTER VEDIC ANUSVARA;Lo;0;L;;;;;N;;;;; 09FD;BENGALI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; +09FE;BENGALI SANDHI MARK;Mn;230;NSM;;;;;N;;;;; 0A01;GURMUKHI SIGN ADAK BINDI;Mn;0;NSM;;;;;N;;;;; 0A02;GURMUKHI SIGN BINDI;Mn;0;NSM;;;;;N;;;;; 0A03;GURMUKHI SIGN VISARGA;Mc;0;L;;;;;N;;;;; @@ -2458,6 +2466,7 @@ 0A73;GURMUKHI URA;Lo;0;L;;;;;N;;;;; 0A74;GURMUKHI EK ONKAR;Lo;0;L;;;;;N;;;;; 0A75;GURMUKHI SIGN YAKASH;Mn;0;NSM;;;;;N;;;;; +0A76;GURMUKHI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; 0A81;GUJARATI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; 0A82;GUJARATI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; 0A83;GUJARATI SIGN VISARGA;Mc;0;L;;;;;N;;;;; @@ -2715,6 +2724,7 @@ 0C01;TELUGU SIGN CANDRABINDU;Mc;0;L;;;;;N;;;;; 0C02;TELUGU SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; 0C03;TELUGU SIGN VISARGA;Mc;0;L;;;;;N;;;;; +0C04;TELUGU SIGN COMBINING ANUSVARA ABOVE;Mn;0;NSM;;;;;N;;;;; 0C05;TELUGU LETTER A;Lo;0;L;;;;;N;;;;; 0C06;TELUGU LETTER AA;Lo;0;L;;;;;N;;;;; 0C07;TELUGU LETTER I;Lo;0;L;;;;;N;;;;; @@ -2811,6 +2821,7 @@ 0C81;KANNADA SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; 0C82;KANNADA SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; 0C83;KANNADA SIGN VISARGA;Mc;0;L;;;;;N;;;;; +0C84;KANNADA SIGN SIDDHAM;Po;0;L;;;;;N;;;;; 0C85;KANNADA LETTER A;Lo;0;L;;;;;N;;;;; 0C86;KANNADA LETTER AA;Lo;0;L;;;;;N;;;;; 0C87;KANNADA LETTER I;Lo;0;L;;;;;N;;;;; @@ -3667,54 +3678,54 @@ 10C5;GEORGIAN CAPITAL LETTER HOE;Lu;0;L;;;;;N;;;;2D25; 10C7;GEORGIAN CAPITAL LETTER YN;Lu;0;L;;;;;N;;;;2D27; 10CD;GEORGIAN CAPITAL LETTER AEN;Lu;0;L;;;;;N;;;;2D2D; -10D0;GEORGIAN LETTER AN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER AN;;;; -10D1;GEORGIAN LETTER BAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER BAN;;;; -10D2;GEORGIAN LETTER GAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER GAN;;;; -10D3;GEORGIAN LETTER DON;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER DON;;;; -10D4;GEORGIAN LETTER EN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER EN;;;; -10D5;GEORGIAN LETTER VIN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER VIN;;;; -10D6;GEORGIAN LETTER ZEN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER ZEN;;;; -10D7;GEORGIAN LETTER TAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER TAN;;;; -10D8;GEORGIAN LETTER IN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER IN;;;; -10D9;GEORGIAN LETTER KAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER KAN;;;; -10DA;GEORGIAN LETTER LAS;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER LAS;;;; -10DB;GEORGIAN LETTER MAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER MAN;;;; -10DC;GEORGIAN LETTER NAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER NAR;;;; -10DD;GEORGIAN LETTER ON;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER ON;;;; -10DE;GEORGIAN LETTER PAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER PAR;;;; -10DF;GEORGIAN LETTER ZHAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER ZHAR;;;; -10E0;GEORGIAN LETTER RAE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER RAE;;;; -10E1;GEORGIAN LETTER SAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER SAN;;;; -10E2;GEORGIAN LETTER TAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER TAR;;;; -10E3;GEORGIAN LETTER UN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER UN;;;; -10E4;GEORGIAN LETTER PHAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER PHAR;;;; -10E5;GEORGIAN LETTER KHAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER KHAR;;;; -10E6;GEORGIAN LETTER GHAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER GHAN;;;; -10E7;GEORGIAN LETTER QAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER QAR;;;; -10E8;GEORGIAN LETTER SHIN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER SHIN;;;; -10E9;GEORGIAN LETTER CHIN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER CHIN;;;; -10EA;GEORGIAN LETTER CAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER CAN;;;; -10EB;GEORGIAN LETTER JIL;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER JIL;;;; -10EC;GEORGIAN LETTER CIL;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER CIL;;;; -10ED;GEORGIAN LETTER CHAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER CHAR;;;; -10EE;GEORGIAN LETTER XAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER XAN;;;; -10EF;GEORGIAN LETTER JHAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER JHAN;;;; -10F0;GEORGIAN LETTER HAE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER HAE;;;; -10F1;GEORGIAN LETTER HE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER HE;;;; -10F2;GEORGIAN LETTER HIE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER HIE;;;; -10F3;GEORGIAN LETTER WE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER WE;;;; -10F4;GEORGIAN LETTER HAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER HAR;;;; -10F5;GEORGIAN LETTER HOE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER HOE;;;; -10F6;GEORGIAN LETTER FI;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER FI;;;; -10F7;GEORGIAN LETTER YN;Lo;0;L;;;;;N;;;;; -10F8;GEORGIAN LETTER ELIFI;Lo;0;L;;;;;N;;;;; -10F9;GEORGIAN LETTER TURNED GAN;Lo;0;L;;;;;N;;;;; -10FA;GEORGIAN LETTER AIN;Lo;0;L;;;;;N;;;;; +10D0;GEORGIAN LETTER AN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER AN;;1C90;;10D0 +10D1;GEORGIAN LETTER BAN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER BAN;;1C91;;10D1 +10D2;GEORGIAN LETTER GAN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER GAN;;1C92;;10D2 +10D3;GEORGIAN LETTER DON;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER DON;;1C93;;10D3 +10D4;GEORGIAN LETTER EN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER EN;;1C94;;10D4 +10D5;GEORGIAN LETTER VIN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER VIN;;1C95;;10D5 +10D6;GEORGIAN LETTER ZEN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER ZEN;;1C96;;10D6 +10D7;GEORGIAN LETTER TAN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER TAN;;1C97;;10D7 +10D8;GEORGIAN LETTER IN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER IN;;1C98;;10D8 +10D9;GEORGIAN LETTER KAN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER KAN;;1C99;;10D9 +10DA;GEORGIAN LETTER LAS;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER LAS;;1C9A;;10DA +10DB;GEORGIAN LETTER MAN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER MAN;;1C9B;;10DB +10DC;GEORGIAN LETTER NAR;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER NAR;;1C9C;;10DC +10DD;GEORGIAN LETTER ON;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER ON;;1C9D;;10DD +10DE;GEORGIAN LETTER PAR;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER PAR;;1C9E;;10DE +10DF;GEORGIAN LETTER ZHAR;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER ZHAR;;1C9F;;10DF +10E0;GEORGIAN LETTER RAE;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER RAE;;1CA0;;10E0 +10E1;GEORGIAN LETTER SAN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER SAN;;1CA1;;10E1 +10E2;GEORGIAN LETTER TAR;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER TAR;;1CA2;;10E2 +10E3;GEORGIAN LETTER UN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER UN;;1CA3;;10E3 +10E4;GEORGIAN LETTER PHAR;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER PHAR;;1CA4;;10E4 +10E5;GEORGIAN LETTER KHAR;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER KHAR;;1CA5;;10E5 +10E6;GEORGIAN LETTER GHAN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER GHAN;;1CA6;;10E6 +10E7;GEORGIAN LETTER QAR;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER QAR;;1CA7;;10E7 +10E8;GEORGIAN LETTER SHIN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER SHIN;;1CA8;;10E8 +10E9;GEORGIAN LETTER CHIN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER CHIN;;1CA9;;10E9 +10EA;GEORGIAN LETTER CAN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER CAN;;1CAA;;10EA +10EB;GEORGIAN LETTER JIL;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER JIL;;1CAB;;10EB +10EC;GEORGIAN LETTER CIL;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER CIL;;1CAC;;10EC +10ED;GEORGIAN LETTER CHAR;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER CHAR;;1CAD;;10ED +10EE;GEORGIAN LETTER XAN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER XAN;;1CAE;;10EE +10EF;GEORGIAN LETTER JHAN;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER JHAN;;1CAF;;10EF +10F0;GEORGIAN LETTER HAE;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER HAE;;1CB0;;10F0 +10F1;GEORGIAN LETTER HE;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER HE;;1CB1;;10F1 +10F2;GEORGIAN LETTER HIE;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER HIE;;1CB2;;10F2 +10F3;GEORGIAN LETTER WE;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER WE;;1CB3;;10F3 +10F4;GEORGIAN LETTER HAR;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER HAR;;1CB4;;10F4 +10F5;GEORGIAN LETTER HOE;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER HOE;;1CB5;;10F5 +10F6;GEORGIAN LETTER FI;Ll;0;L;;;;;N;GEORGIAN SMALL LETTER FI;;1CB6;;10F6 +10F7;GEORGIAN LETTER YN;Ll;0;L;;;;;N;;;1CB7;;10F7 +10F8;GEORGIAN LETTER ELIFI;Ll;0;L;;;;;N;;;1CB8;;10F8 +10F9;GEORGIAN LETTER TURNED GAN;Ll;0;L;;;;;N;;;1CB9;;10F9 +10FA;GEORGIAN LETTER AIN;Ll;0;L;;;;;N;;;1CBA;;10FA 10FB;GEORGIAN PARAGRAPH SEPARATOR;Po;0;L;;;;;N;;;;; 10FC;MODIFIER LETTER GEORGIAN NAR;Lm;0;L;<super> 10DC;;;;N;;;;; -10FD;GEORGIAN LETTER AEN;Lo;0;L;;;;;N;;;;; -10FE;GEORGIAN LETTER HARD SIGN;Lo;0;L;;;;;N;;;;; -10FF;GEORGIAN LETTER LABIAL SIGN;Lo;0;L;;;;;N;;;;; +10FD;GEORGIAN LETTER AEN;Ll;0;L;;;;;N;;;1CBD;;10FD +10FE;GEORGIAN LETTER HARD SIGN;Ll;0;L;;;;;N;;;1CBE;;10FE +10FF;GEORGIAN LETTER LABIAL SIGN;Ll;0;L;;;;;N;;;1CBF;;10FF 1100;HANGUL CHOSEONG KIYEOK;Lo;0;L;;;;;N;;;;; 1101;HANGUL CHOSEONG SSANGKIYEOK;Lo;0;L;;;;;N;;;;; 1102;HANGUL CHOSEONG NIEUN;Lo;0;L;;;;;N;;;;; @@ -5513,6 +5524,7 @@ 1875;MONGOLIAN LETTER MANCHU RA;Lo;0;L;;;;;N;;;;; 1876;MONGOLIAN LETTER MANCHU FA;Lo;0;L;;;;;N;;;;; 1877;MONGOLIAN LETTER MANCHU ZHA;Lo;0;L;;;;;N;;;;; +1878;MONGOLIAN LETTER CHA WITH TWO DOTS;Lo;0;L;;;;;N;;;;; 1880;MONGOLIAN LETTER ALI GALI ANUSVARA ONE;Lo;0;L;;;;;N;;;;; 1881;MONGOLIAN LETTER ALI GALI VISARGA ONE;Lo;0;L;;;;;N;;;;; 1882;MONGOLIAN LETTER ALI GALI DAMARU;Lo;0;L;;;;;N;;;;; @@ -6388,6 +6400,52 @@ 1C86;CYRILLIC SMALL LETTER TALL HARD SIGN;Ll;0;L;;;;;N;;;042A;;042A 1C87;CYRILLIC SMALL LETTER TALL YAT;Ll;0;L;;;;;N;;;0462;;0462 1C88;CYRILLIC SMALL LETTER UNBLENDED UK;Ll;0;L;;;;;N;;;A64A;;A64A +1C90;GEORGIAN MTAVRULI CAPITAL LETTER AN;Lu;0;L;;;;;N;;;;10D0; +1C91;GEORGIAN MTAVRULI CAPITAL LETTER BAN;Lu;0;L;;;;;N;;;;10D1; +1C92;GEORGIAN MTAVRULI CAPITAL LETTER GAN;Lu;0;L;;;;;N;;;;10D2; +1C93;GEORGIAN MTAVRULI CAPITAL LETTER DON;Lu;0;L;;;;;N;;;;10D3; +1C94;GEORGIAN MTAVRULI CAPITAL LETTER EN;Lu;0;L;;;;;N;;;;10D4; +1C95;GEORGIAN MTAVRULI CAPITAL LETTER VIN;Lu;0;L;;;;;N;;;;10D5; +1C96;GEORGIAN MTAVRULI CAPITAL LETTER ZEN;Lu;0;L;;;;;N;;;;10D6; +1C97;GEORGIAN MTAVRULI CAPITAL LETTER TAN;Lu;0;L;;;;;N;;;;10D7; +1C98;GEORGIAN MTAVRULI CAPITAL LETTER IN;Lu;0;L;;;;;N;;;;10D8; +1C99;GEORGIAN MTAVRULI CAPITAL LETTER KAN;Lu;0;L;;;;;N;;;;10D9; +1C9A;GEORGIAN MTAVRULI CAPITAL LETTER LAS;Lu;0;L;;;;;N;;;;10DA; +1C9B;GEORGIAN MTAVRULI CAPITAL LETTER MAN;Lu;0;L;;;;;N;;;;10DB; +1C9C;GEORGIAN MTAVRULI CAPITAL LETTER NAR;Lu;0;L;;;;;N;;;;10DC; +1C9D;GEORGIAN MTAVRULI CAPITAL LETTER ON;Lu;0;L;;;;;N;;;;10DD; +1C9E;GEORGIAN MTAVRULI CAPITAL LETTER PAR;Lu;0;L;;;;;N;;;;10DE; +1C9F;GEORGIAN MTAVRULI CAPITAL LETTER ZHAR;Lu;0;L;;;;;N;;;;10DF; +1CA0;GEORGIAN MTAVRULI CAPITAL LETTER RAE;Lu;0;L;;;;;N;;;;10E0; +1CA1;GEORGIAN MTAVRULI CAPITAL LETTER SAN;Lu;0;L;;;;;N;;;;10E1; +1CA2;GEORGIAN MTAVRULI CAPITAL LETTER TAR;Lu;0;L;;;;;N;;;;10E2; +1CA3;GEORGIAN MTAVRULI CAPITAL LETTER UN;Lu;0;L;;;;;N;;;;10E3; +1CA4;GEORGIAN MTAVRULI CAPITAL LETTER PHAR;Lu;0;L;;;;;N;;;;10E4; +1CA5;GEORGIAN MTAVRULI CAPITAL LETTER KHAR;Lu;0;L;;;;;N;;;;10E5; +1CA6;GEORGIAN MTAVRULI CAPITAL LETTER GHAN;Lu;0;L;;;;;N;;;;10E6; +1CA7;GEORGIAN MTAVRULI CAPITAL LETTER QAR;Lu;0;L;;;;;N;;;;10E7; +1CA8;GEORGIAN MTAVRULI CAPITAL LETTER SHIN;Lu;0;L;;;;;N;;;;10E8; +1CA9;GEORGIAN MTAVRULI CAPITAL LETTER CHIN;Lu;0;L;;;;;N;;;;10E9; +1CAA;GEORGIAN MTAVRULI CAPITAL LETTER CAN;Lu;0;L;;;;;N;;;;10EA; +1CAB;GEORGIAN MTAVRULI CAPITAL LETTER JIL;Lu;0;L;;;;;N;;;;10EB; +1CAC;GEORGIAN MTAVRULI CAPITAL LETTER CIL;Lu;0;L;;;;;N;;;;10EC; +1CAD;GEORGIAN MTAVRULI CAPITAL LETTER CHAR;Lu;0;L;;;;;N;;;;10ED; +1CAE;GEORGIAN MTAVRULI CAPITAL LETTER XAN;Lu;0;L;;;;;N;;;;10EE; +1CAF;GEORGIAN MTAVRULI CAPITAL LETTER JHAN;Lu;0;L;;;;;N;;;;10EF; +1CB0;GEORGIAN MTAVRULI CAPITAL LETTER HAE;Lu;0;L;;;;;N;;;;10F0; +1CB1;GEORGIAN MTAVRULI CAPITAL LETTER HE;Lu;0;L;;;;;N;;;;10F1; +1CB2;GEORGIAN MTAVRULI CAPITAL LETTER HIE;Lu;0;L;;;;;N;;;;10F2; +1CB3;GEORGIAN MTAVRULI CAPITAL LETTER WE;Lu;0;L;;;;;N;;;;10F3; +1CB4;GEORGIAN MTAVRULI CAPITAL LETTER HAR;Lu;0;L;;;;;N;;;;10F4; +1CB5;GEORGIAN MTAVRULI CAPITAL LETTER HOE;Lu;0;L;;;;;N;;;;10F5; +1CB6;GEORGIAN MTAVRULI CAPITAL LETTER FI;Lu;0;L;;;;;N;;;;10F6; +1CB7;GEORGIAN MTAVRULI CAPITAL LETTER YN;Lu;0;L;;;;;N;;;;10F7; +1CB8;GEORGIAN MTAVRULI CAPITAL LETTER ELIFI;Lu;0;L;;;;;N;;;;10F8; +1CB9;GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN;Lu;0;L;;;;;N;;;;10F9; +1CBA;GEORGIAN MTAVRULI CAPITAL LETTER AIN;Lu;0;L;;;;;N;;;;10FA; +1CBD;GEORGIAN MTAVRULI CAPITAL LETTER AEN;Lu;0;L;;;;;N;;;;10FD; +1CBE;GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN;Lu;0;L;;;;;N;;;;10FE; +1CBF;GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN;Lu;0;L;;;;;N;;;;10FF; 1CC0;SUNDANESE PUNCTUATION BINDU SURYA;Po;0;L;;;;;N;;;;; 1CC1;SUNDANESE PUNCTUATION BINDU PANGLONG;Po;0;L;;;;;N;;;;; 1CC2;SUNDANESE PUNCTUATION BINDU PURNAMA;Po;0;L;;;;;N;;;;; @@ -9559,7 +9617,7 @@ 299E;ANGLE WITH S INSIDE;Sm;0;ON;;;;;Y;;;;; 299F;ACUTE ANGLE;Sm;0;ON;;;;;Y;;;;; 29A0;SPHERICAL ANGLE OPENING LEFT;Sm;0;ON;;;;;Y;;;;; -29A1;SPHERICAL ANGLE OPENING UP;Sm;0;ON;;;;;Y;;;;; +29A1;SPHERICAL ANGLE OPENING UP;Sm;0;ON;;;;;N;;;;; 29A2;TURNED ANGLE;Sm;0;ON;;;;;Y;;;;; 29A3;REVERSED ANGLE;Sm;0;ON;;;;;Y;;;;; 29A4;ANGLE WITH UNDERBAR;Sm;0;ON;;;;;Y;;;;; @@ -10092,6 +10150,9 @@ 2BB7;RIBBON ARROW RIGHT DOWN;So;0;ON;;;;;N;;;;; 2BB8;UPWARDS WHITE ARROW FROM BAR WITH HORIZONTAL BAR;So;0;ON;;;;;N;;;;; 2BB9;UP ARROWHEAD IN A RECTANGLE BOX;So;0;ON;;;;;N;;;;; +2BBA;OVERLAPPING WHITE SQUARES;So;0;ON;;;;;N;;;;; +2BBB;OVERLAPPING WHITE AND BLACK SQUARES;So;0;ON;;;;;N;;;;; +2BBC;OVERLAPPING BLACK SQUARES;So;0;ON;;;;;N;;;;; 2BBD;BALLOT BOX WITH LIGHT X;So;0;ON;;;;;N;;;;; 2BBE;CIRCLED X;So;0;ON;;;;;N;;;;; 2BBF;CIRCLED BOLD X;So;0;ON;;;;;N;;;;; @@ -10113,10 +10174,50 @@ 2BD0;SQUARE POSITION INDICATOR;So;0;ON;;;;;N;;;;; 2BD1;UNCERTAINTY SIGN;So;0;ON;;;;;N;;;;; 2BD2;GROUP MARK;So;0;ON;;;;;N;;;;; +2BD3;PLUTO FORM TWO;So;0;ON;;;;;N;;;;; +2BD4;PLUTO FORM THREE;So;0;ON;;;;;N;;;;; +2BD5;PLUTO FORM FOUR;So;0;ON;;;;;N;;;;; +2BD6;PLUTO FORM FIVE;So;0;ON;;;;;N;;;;; +2BD7;TRANSPLUTO;So;0;ON;;;;;N;;;;; +2BD8;PROSERPINA;So;0;ON;;;;;N;;;;; +2BD9;ASTRAEA;So;0;ON;;;;;N;;;;; +2BDA;HYGIEA;So;0;ON;;;;;N;;;;; +2BDB;PHOLUS;So;0;ON;;;;;N;;;;; +2BDC;NESSUS;So;0;ON;;;;;N;;;;; +2BDD;WHITE MOON SELENA;So;0;ON;;;;;N;;;;; +2BDE;BLACK DIAMOND ON CROSS;So;0;ON;;;;;N;;;;; +2BDF;TRUE LIGHT MOON ARTA;So;0;ON;;;;;N;;;;; +2BE0;CUPIDO;So;0;ON;;;;;N;;;;; +2BE1;HADES;So;0;ON;;;;;N;;;;; +2BE2;ZEUS;So;0;ON;;;;;N;;;;; +2BE3;KRONOS;So;0;ON;;;;;N;;;;; +2BE4;APOLLON;So;0;ON;;;;;N;;;;; +2BE5;ADMETOS;So;0;ON;;;;;N;;;;; +2BE6;VULCANUS;So;0;ON;;;;;N;;;;; +2BE7;POSEIDON;So;0;ON;;;;;N;;;;; +2BE8;LEFT HALF BLACK STAR;So;0;ON;;;;;N;;;;; +2BE9;RIGHT HALF BLACK STAR;So;0;ON;;;;;N;;;;; +2BEA;STAR WITH LEFT HALF BLACK;So;0;ON;;;;;N;;;;; +2BEB;STAR WITH RIGHT HALF BLACK;So;0;ON;;;;;N;;;;; 2BEC;LEFTWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS;So;0;ON;;;;;N;;;;; 2BED;UPWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS;So;0;ON;;;;;N;;;;; 2BEE;RIGHTWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS;So;0;ON;;;;;N;;;;; 2BEF;DOWNWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS;So;0;ON;;;;;N;;;;; +2BF0;ERIS FORM ONE;So;0;ON;;;;;N;;;;; +2BF1;ERIS FORM TWO;So;0;ON;;;;;N;;;;; +2BF2;SEDNA;So;0;ON;;;;;N;;;;; +2BF3;RUSSIAN ASTROLOGICAL SYMBOL VIGINTILE;So;0;ON;;;;;N;;;;; +2BF4;RUSSIAN ASTROLOGICAL SYMBOL NOVILE;So;0;ON;;;;;N;;;;; +2BF5;RUSSIAN ASTROLOGICAL SYMBOL QUINTILE;So;0;ON;;;;;N;;;;; +2BF6;RUSSIAN ASTROLOGICAL SYMBOL BINOVILE;So;0;ON;;;;;N;;;;; +2BF7;RUSSIAN ASTROLOGICAL SYMBOL SENTAGON;So;0;ON;;;;;N;;;;; +2BF8;RUSSIAN ASTROLOGICAL SYMBOL TREDECILE;So;0;ON;;;;;N;;;;; +2BF9;EQUALS SIGN WITH INFINITY BELOW;So;0;ON;;;;;N;;;;; +2BFA;UNITED SYMBOL;So;0;ON;;;;;N;;;;; +2BFB;SEPARATED SYMBOL;So;0;ON;;;;;N;;;;; +2BFC;DOUBLED SYMBOL;So;0;ON;;;;;N;;;;; +2BFD;PASSED SYMBOL;So;0;ON;;;;;N;;;;; +2BFE;REVERSED RIGHT ANGLE;So;0;ON;;;;;Y;;;;; 2C00;GLAGOLITIC CAPITAL LETTER AZU;Lu;0;L;;;;;N;;;;2C30; 2C01;GLAGOLITIC CAPITAL LETTER BUKY;Lu;0;L;;;;;N;;;;2C31; 2C02;GLAGOLITIC CAPITAL LETTER VEDE;Lu;0;L;;;;;N;;;;2C32; @@ -10650,6 +10751,11 @@ 2E47;LOW KAVYKA;Po;0;ON;;;;;N;;;;; 2E48;LOW KAVYKA WITH DOT;Po;0;ON;;;;;N;;;;; 2E49;DOUBLE STACKED COMMA;Po;0;ON;;;;;N;;;;; +2E4A;DOTTED SOLIDUS;Po;0;ON;;;;;N;;;;; +2E4B;TRIPLE DAGGER;Po;0;ON;;;;;N;;;;; +2E4C;MEDIEVAL COMMA;Po;0;ON;;;;;N;;;;; +2E4D;PARAGRAPHUS MARK;Po;0;ON;;;;;N;;;;; +2E4E;PUNCTUS ELEVATUS MARK;Po;0;ON;;;;;N;;;;; 2E80;CJK RADICAL REPEAT;So;0;ON;;;;;N;;;;; 2E81;CJK RADICAL CLIFF;So;0;ON;;;;;N;;;;; 2E82;CJK RADICAL SECOND ONE;So;0;ON;;;;;N;;;;; @@ -11286,6 +11392,7 @@ 312C;BOPOMOFO LETTER GN;Lo;0;L;;;;;N;;;;; 312D;BOPOMOFO LETTER IH;Lo;0;L;;;;;N;;;;; 312E;BOPOMOFO LETTER O WITH DOT ABOVE;Lo;0;L;;;;;N;;;;; +312F;BOPOMOFO LETTER NN;Lo;0;L;;;;;N;;;;; 3131;HANGUL LETTER KIYEOK;Lo;0;L;<compat> 1100;;;;N;HANGUL LETTER GIYEOG;;;; 3132;HANGUL LETTER SSANGKIYEOK;Lo;0;L;<compat> 1101;;;;N;HANGUL LETTER SSANG GIYEOG;;;; 3133;HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;; @@ -12052,7 +12159,7 @@ 4DFE;HEXAGRAM FOR AFTER COMPLETION;So;0;ON;;;;;N;;;;; 4DFF;HEXAGRAM FOR BEFORE COMPLETION;So;0;ON;;;;;N;;;;; 4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;; -9FEA;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;; +9FEF;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;; A000;YI SYLLABLE IT;Lo;0;L;;;;;N;;;;; A001;YI SYLLABLE IX;Lo;0;L;;;;;N;;;;; A002;YI SYLLABLE I;Lo;0;L;;;;;N;;;;; @@ -13980,6 +14087,7 @@ A7AB;LATIN CAPITAL LETTER REVERSED OPEN E;Lu;0;L;;;;;N;;;;025C; A7AC;LATIN CAPITAL LETTER SCRIPT G;Lu;0;L;;;;;N;;;;0261; A7AD;LATIN CAPITAL LETTER L WITH BELT;Lu;0;L;;;;;N;;;;026C; A7AE;LATIN CAPITAL LETTER SMALL CAPITAL I;Lu;0;L;;;;;N;;;;026A; +A7AF;LATIN LETTER SMALL CAPITAL Q;Ll;0;L;;;;;N;;;;; A7B0;LATIN CAPITAL LETTER TURNED K;Lu;0;L;;;;;N;;;;029E; A7B1;LATIN CAPITAL LETTER TURNED T;Lu;0;L;;;;;N;;;;0287; A7B2;LATIN CAPITAL LETTER J WITH CROSSED-TAIL;Lu;0;L;;;;;N;;;;029D; @@ -13988,6 +14096,8 @@ A7B4;LATIN CAPITAL LETTER BETA;Lu;0;L;;;;;N;;;;A7B5; A7B5;LATIN SMALL LETTER BETA;Ll;0;L;;;;;N;;;A7B4;;A7B4 A7B6;LATIN CAPITAL LETTER OMEGA;Lu;0;L;;;;;N;;;;A7B7; A7B7;LATIN SMALL LETTER OMEGA;Ll;0;L;;;;;N;;;A7B6;;A7B6 +A7B8;LATIN CAPITAL LETTER U WITH STROKE;Lu;0;L;;;;;N;;;;A7B9; +A7B9;LATIN SMALL LETTER U WITH STROKE;Ll;0;L;;;;;N;;;A7B8;;A7B8 A7F7;LATIN EPIGRAPHIC LETTER SIDEWAYS I;Lo;0;L;;;;;N;;;;; A7F8;MODIFIER LETTER CAPITAL H WITH STROKE;Lm;0;L;<super> 0126;;;;N;;;;; A7F9;MODIFIER LETTER SMALL LIGATURE OE;Lm;0;L;<super> 0153;;;;N;;;;; @@ -14219,6 +14329,8 @@ A8FA;DEVANAGARI CARET;Po;0;L;;;;;N;;;;; A8FB;DEVANAGARI HEADSTROKE;Lo;0;L;;;;;N;;;;; A8FC;DEVANAGARI SIGN SIDDHAM;Po;0;L;;;;;N;;;;; A8FD;DEVANAGARI JAIN OM;Lo;0;L;;;;;N;;;;; +A8FE;DEVANAGARI LETTER AY;Lo;0;L;;;;;N;;;;; +A8FF;DEVANAGARI VOWEL SIGN AY;Mn;0;NSM;;;;;N;;;;; A900;KAYAH LI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; A901;KAYAH LI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; A902;KAYAH LI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; @@ -18363,6 +18475,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10A31;KHAROSHTHI LETTER HA;Lo;0;R;;;;;N;;;;; 10A32;KHAROSHTHI LETTER KKA;Lo;0;R;;;;;N;;;;; 10A33;KHAROSHTHI LETTER TTTHA;Lo;0;R;;;;;N;;;;; +10A34;KHAROSHTHI LETTER TTTA;Lo;0;R;;;;;N;;;;; +10A35;KHAROSHTHI LETTER VHA;Lo;0;R;;;;;N;;;;; 10A38;KHAROSHTHI SIGN BAR ABOVE;Mn;230;NSM;;;;;N;;;;; 10A39;KHAROSHTHI SIGN CAUDA;Mn;1;NSM;;;;;N;;;;; 10A3A;KHAROSHTHI SIGN DOT BELOW;Mn;220;NSM;;;;;N;;;;; @@ -18375,6 +18489,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10A45;KHAROSHTHI NUMBER TWENTY;No;0;R;;;;20;N;;;;; 10A46;KHAROSHTHI NUMBER ONE HUNDRED;No;0;R;;;;100;N;;;;; 10A47;KHAROSHTHI NUMBER ONE THOUSAND;No;0;R;;;;1000;N;;;;; +10A48;KHAROSHTHI FRACTION ONE HALF;No;0;R;;;;1/2;N;;;;; 10A50;KHAROSHTHI PUNCTUATION DOT;Po;0;R;;;;;N;;;;; 10A51;KHAROSHTHI PUNCTUATION SMALL CIRCLE;Po;0;R;;;;;N;;;;; 10A52;KHAROSHTHI PUNCTUATION CIRCLE;Po;0;R;;;;;N;;;;; @@ -18827,6 +18942,56 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10CFD;OLD HUNGARIAN NUMBER FIFTY;No;0;R;;;;50;N;;;;; 10CFE;OLD HUNGARIAN NUMBER ONE HUNDRED;No;0;R;;;;100;N;;;;; 10CFF;OLD HUNGARIAN NUMBER ONE THOUSAND;No;0;R;;;;1000;N;;;;; +10D00;HANIFI ROHINGYA LETTER A;Lo;0;AL;;;;;N;;;;; +10D01;HANIFI ROHINGYA LETTER BA;Lo;0;AL;;;;;N;;;;; +10D02;HANIFI ROHINGYA LETTER PA;Lo;0;AL;;;;;N;;;;; +10D03;HANIFI ROHINGYA LETTER TA;Lo;0;AL;;;;;N;;;;; +10D04;HANIFI ROHINGYA LETTER TTA;Lo;0;AL;;;;;N;;;;; +10D05;HANIFI ROHINGYA LETTER JA;Lo;0;AL;;;;;N;;;;; +10D06;HANIFI ROHINGYA LETTER CA;Lo;0;AL;;;;;N;;;;; +10D07;HANIFI ROHINGYA LETTER HA;Lo;0;AL;;;;;N;;;;; +10D08;HANIFI ROHINGYA LETTER KHA;Lo;0;AL;;;;;N;;;;; +10D09;HANIFI ROHINGYA LETTER FA;Lo;0;AL;;;;;N;;;;; +10D0A;HANIFI ROHINGYA LETTER DA;Lo;0;AL;;;;;N;;;;; +10D0B;HANIFI ROHINGYA LETTER DDA;Lo;0;AL;;;;;N;;;;; +10D0C;HANIFI ROHINGYA LETTER RA;Lo;0;AL;;;;;N;;;;; +10D0D;HANIFI ROHINGYA LETTER RRA;Lo;0;AL;;;;;N;;;;; +10D0E;HANIFI ROHINGYA LETTER ZA;Lo;0;AL;;;;;N;;;;; +10D0F;HANIFI ROHINGYA LETTER SA;Lo;0;AL;;;;;N;;;;; +10D10;HANIFI ROHINGYA LETTER SHA;Lo;0;AL;;;;;N;;;;; +10D11;HANIFI ROHINGYA LETTER KA;Lo;0;AL;;;;;N;;;;; +10D12;HANIFI ROHINGYA LETTER GA;Lo;0;AL;;;;;N;;;;; +10D13;HANIFI ROHINGYA LETTER LA;Lo;0;AL;;;;;N;;;;; +10D14;HANIFI ROHINGYA LETTER MA;Lo;0;AL;;;;;N;;;;; +10D15;HANIFI ROHINGYA LETTER NA;Lo;0;AL;;;;;N;;;;; +10D16;HANIFI ROHINGYA LETTER WA;Lo;0;AL;;;;;N;;;;; +10D17;HANIFI ROHINGYA LETTER KINNA WA;Lo;0;AL;;;;;N;;;;; +10D18;HANIFI ROHINGYA LETTER YA;Lo;0;AL;;;;;N;;;;; +10D19;HANIFI ROHINGYA LETTER KINNA YA;Lo;0;AL;;;;;N;;;;; +10D1A;HANIFI ROHINGYA LETTER NGA;Lo;0;AL;;;;;N;;;;; +10D1B;HANIFI ROHINGYA LETTER NYA;Lo;0;AL;;;;;N;;;;; +10D1C;HANIFI ROHINGYA LETTER VA;Lo;0;AL;;;;;N;;;;; +10D1D;HANIFI ROHINGYA VOWEL A;Lo;0;AL;;;;;N;;;;; +10D1E;HANIFI ROHINGYA VOWEL I;Lo;0;AL;;;;;N;;;;; +10D1F;HANIFI ROHINGYA VOWEL U;Lo;0;AL;;;;;N;;;;; +10D20;HANIFI ROHINGYA VOWEL E;Lo;0;AL;;;;;N;;;;; +10D21;HANIFI ROHINGYA VOWEL O;Lo;0;AL;;;;;N;;;;; +10D22;HANIFI ROHINGYA MARK SAKIN;Lo;0;AL;;;;;N;;;;; +10D23;HANIFI ROHINGYA MARK NA KHONNA;Lo;0;AL;;;;;N;;;;; +10D24;HANIFI ROHINGYA SIGN HARBAHAY;Mn;230;NSM;;;;;N;;;;; +10D25;HANIFI ROHINGYA SIGN TAHALA;Mn;230;NSM;;;;;N;;;;; +10D26;HANIFI ROHINGYA SIGN TANA;Mn;230;NSM;;;;;N;;;;; +10D27;HANIFI ROHINGYA SIGN TASSI;Mn;230;NSM;;;;;N;;;;; +10D30;HANIFI ROHINGYA DIGIT ZERO;Nd;0;AN;;0;0;0;N;;;;; +10D31;HANIFI ROHINGYA DIGIT ONE;Nd;0;AN;;1;1;1;N;;;;; +10D32;HANIFI ROHINGYA DIGIT TWO;Nd;0;AN;;2;2;2;N;;;;; +10D33;HANIFI ROHINGYA DIGIT THREE;Nd;0;AN;;3;3;3;N;;;;; +10D34;HANIFI ROHINGYA DIGIT FOUR;Nd;0;AN;;4;4;4;N;;;;; +10D35;HANIFI ROHINGYA DIGIT FIVE;Nd;0;AN;;5;5;5;N;;;;; +10D36;HANIFI ROHINGYA DIGIT SIX;Nd;0;AN;;6;6;6;N;;;;; +10D37;HANIFI ROHINGYA DIGIT SEVEN;Nd;0;AN;;7;7;7;N;;;;; +10D38;HANIFI ROHINGYA DIGIT EIGHT;Nd;0;AN;;8;8;8;N;;;;; +10D39;HANIFI ROHINGYA DIGIT NINE;Nd;0;AN;;9;9;9;N;;;;; 10E60;RUMI DIGIT ONE;No;0;AN;;;1;1;N;;;;; 10E61;RUMI DIGIT TWO;No;0;AN;;;2;2;N;;;;; 10E62;RUMI DIGIT THREE;No;0;AN;;;3;3;N;;;;; @@ -18858,6 +19023,88 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10E7C;RUMI FRACTION ONE QUARTER;No;0;AN;;;;1/4;N;;;;; 10E7D;RUMI FRACTION ONE THIRD;No;0;AN;;;;1/3;N;;;;; 10E7E;RUMI FRACTION TWO THIRDS;No;0;AN;;;;2/3;N;;;;; +10F00;OLD SOGDIAN LETTER ALEPH;Lo;0;R;;;;;N;;;;; +10F01;OLD SOGDIAN LETTER FINAL ALEPH;Lo;0;R;;;;;N;;;;; +10F02;OLD SOGDIAN LETTER BETH;Lo;0;R;;;;;N;;;;; +10F03;OLD SOGDIAN LETTER FINAL BETH;Lo;0;R;;;;;N;;;;; +10F04;OLD SOGDIAN LETTER GIMEL;Lo;0;R;;;;;N;;;;; +10F05;OLD SOGDIAN LETTER HE;Lo;0;R;;;;;N;;;;; +10F06;OLD SOGDIAN LETTER FINAL HE;Lo;0;R;;;;;N;;;;; +10F07;OLD SOGDIAN LETTER WAW;Lo;0;R;;;;;N;;;;; +10F08;OLD SOGDIAN LETTER ZAYIN;Lo;0;R;;;;;N;;;;; +10F09;OLD SOGDIAN LETTER HETH;Lo;0;R;;;;;N;;;;; +10F0A;OLD SOGDIAN LETTER YODH;Lo;0;R;;;;;N;;;;; +10F0B;OLD SOGDIAN LETTER KAPH;Lo;0;R;;;;;N;;;;; +10F0C;OLD SOGDIAN LETTER LAMEDH;Lo;0;R;;;;;N;;;;; +10F0D;OLD SOGDIAN LETTER MEM;Lo;0;R;;;;;N;;;;; +10F0E;OLD SOGDIAN LETTER NUN;Lo;0;R;;;;;N;;;;; +10F0F;OLD SOGDIAN LETTER FINAL NUN;Lo;0;R;;;;;N;;;;; +10F10;OLD SOGDIAN LETTER FINAL NUN WITH VERTICAL TAIL;Lo;0;R;;;;;N;;;;; +10F11;OLD SOGDIAN LETTER SAMEKH;Lo;0;R;;;;;N;;;;; +10F12;OLD SOGDIAN LETTER AYIN;Lo;0;R;;;;;N;;;;; +10F13;OLD SOGDIAN LETTER ALTERNATE AYIN;Lo;0;R;;;;;N;;;;; +10F14;OLD SOGDIAN LETTER PE;Lo;0;R;;;;;N;;;;; +10F15;OLD SOGDIAN LETTER SADHE;Lo;0;R;;;;;N;;;;; +10F16;OLD SOGDIAN LETTER FINAL SADHE;Lo;0;R;;;;;N;;;;; +10F17;OLD SOGDIAN LETTER FINAL SADHE WITH VERTICAL TAIL;Lo;0;R;;;;;N;;;;; +10F18;OLD SOGDIAN LETTER RESH-AYIN-DALETH;Lo;0;R;;;;;N;;;;; +10F19;OLD SOGDIAN LETTER SHIN;Lo;0;R;;;;;N;;;;; +10F1A;OLD SOGDIAN LETTER TAW;Lo;0;R;;;;;N;;;;; +10F1B;OLD SOGDIAN LETTER FINAL TAW;Lo;0;R;;;;;N;;;;; +10F1C;OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL;Lo;0;R;;;;;N;;;;; +10F1D;OLD SOGDIAN NUMBER ONE;No;0;R;;;;1;N;;;;; +10F1E;OLD SOGDIAN NUMBER TWO;No;0;R;;;;2;N;;;;; +10F1F;OLD SOGDIAN NUMBER THREE;No;0;R;;;;3;N;;;;; +10F20;OLD SOGDIAN NUMBER FOUR;No;0;R;;;;4;N;;;;; +10F21;OLD SOGDIAN NUMBER FIVE;No;0;R;;;;5;N;;;;; +10F22;OLD SOGDIAN NUMBER TEN;No;0;R;;;;10;N;;;;; +10F23;OLD SOGDIAN NUMBER TWENTY;No;0;R;;;;20;N;;;;; +10F24;OLD SOGDIAN NUMBER THIRTY;No;0;R;;;;30;N;;;;; +10F25;OLD SOGDIAN NUMBER ONE HUNDRED;No;0;R;;;;100;N;;;;; +10F26;OLD SOGDIAN FRACTION ONE HALF;No;0;R;;;;1/2;N;;;;; +10F27;OLD SOGDIAN LIGATURE AYIN-DALETH;Lo;0;R;;;;;N;;;;; +10F30;SOGDIAN LETTER ALEPH;Lo;0;AL;;;;;N;;;;; +10F31;SOGDIAN LETTER BETH;Lo;0;AL;;;;;N;;;;; +10F32;SOGDIAN LETTER GIMEL;Lo;0;AL;;;;;N;;;;; +10F33;SOGDIAN LETTER HE;Lo;0;AL;;;;;N;;;;; +10F34;SOGDIAN LETTER WAW;Lo;0;AL;;;;;N;;;;; +10F35;SOGDIAN LETTER ZAYIN;Lo;0;AL;;;;;N;;;;; +10F36;SOGDIAN LETTER HETH;Lo;0;AL;;;;;N;;;;; +10F37;SOGDIAN LETTER YODH;Lo;0;AL;;;;;N;;;;; +10F38;SOGDIAN LETTER KAPH;Lo;0;AL;;;;;N;;;;; +10F39;SOGDIAN LETTER LAMEDH;Lo;0;AL;;;;;N;;;;; +10F3A;SOGDIAN LETTER MEM;Lo;0;AL;;;;;N;;;;; +10F3B;SOGDIAN LETTER NUN;Lo;0;AL;;;;;N;;;;; +10F3C;SOGDIAN LETTER SAMEKH;Lo;0;AL;;;;;N;;;;; +10F3D;SOGDIAN LETTER AYIN;Lo;0;AL;;;;;N;;;;; +10F3E;SOGDIAN LETTER PE;Lo;0;AL;;;;;N;;;;; +10F3F;SOGDIAN LETTER SADHE;Lo;0;AL;;;;;N;;;;; +10F40;SOGDIAN LETTER RESH-AYIN;Lo;0;AL;;;;;N;;;;; +10F41;SOGDIAN LETTER SHIN;Lo;0;AL;;;;;N;;;;; +10F42;SOGDIAN LETTER TAW;Lo;0;AL;;;;;N;;;;; +10F43;SOGDIAN LETTER FETH;Lo;0;AL;;;;;N;;;;; +10F44;SOGDIAN LETTER LESH;Lo;0;AL;;;;;N;;;;; +10F45;SOGDIAN INDEPENDENT SHIN;Lo;0;AL;;;;;N;;;;; +10F46;SOGDIAN COMBINING DOT BELOW;Mn;220;NSM;;;;;N;;;;; +10F47;SOGDIAN COMBINING TWO DOTS BELOW;Mn;220;NSM;;;;;N;;;;; +10F48;SOGDIAN COMBINING DOT ABOVE;Mn;230;NSM;;;;;N;;;;; +10F49;SOGDIAN COMBINING TWO DOTS ABOVE;Mn;230;NSM;;;;;N;;;;; +10F4A;SOGDIAN COMBINING CURVE ABOVE;Mn;230;NSM;;;;;N;;;;; +10F4B;SOGDIAN COMBINING CURVE BELOW;Mn;220;NSM;;;;;N;;;;; +10F4C;SOGDIAN COMBINING HOOK ABOVE;Mn;230;NSM;;;;;N;;;;; +10F4D;SOGDIAN COMBINING HOOK BELOW;Mn;220;NSM;;;;;N;;;;; +10F4E;SOGDIAN COMBINING LONG HOOK BELOW;Mn;220;NSM;;;;;N;;;;; +10F4F;SOGDIAN COMBINING RESH BELOW;Mn;220;NSM;;;;;N;;;;; +10F50;SOGDIAN COMBINING STROKE BELOW;Mn;220;NSM;;;;;N;;;;; +10F51;SOGDIAN NUMBER ONE;No;0;AL;;;;1;N;;;;; +10F52;SOGDIAN NUMBER TEN;No;0;AL;;;;10;N;;;;; +10F53;SOGDIAN NUMBER TWENTY;No;0;AL;;;;20;N;;;;; +10F54;SOGDIAN NUMBER ONE HUNDRED;No;0;AL;;;;100;N;;;;; +10F55;SOGDIAN PUNCTUATION TWO VERTICAL BARS;Po;0;AL;;;;;N;;;;; +10F56;SOGDIAN PUNCTUATION TWO VERTICAL BARS WITH DOTS;Po;0;AL;;;;;N;;;;; +10F57;SOGDIAN PUNCTUATION CIRCLE WITH DOT;Po;0;AL;;;;;N;;;;; +10F58;SOGDIAN PUNCTUATION TWO CIRCLES WITH DOTS;Po;0;AL;;;;;N;;;;; +10F59;SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT;Po;0;AL;;;;;N;;;;; 11000;BRAHMI SIGN CANDRABINDU;Mc;0;L;;;;;N;;;;; 11001;BRAHMI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; 11002;BRAHMI SIGN VISARGA;Mc;0;L;;;;;N;;;;; @@ -19033,6 +19280,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 110BF;KAITHI DOUBLE SECTION MARK;Po;0;L;;;;;N;;;;; 110C0;KAITHI DANDA;Po;0;L;;;;;N;;;;; 110C1;KAITHI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +110CD;KAITHI NUMBER SIGN ABOVE;Cf;0;L;;;;;N;;;;; 110D0;SORA SOMPENG LETTER SAH;Lo;0;L;;;;;N;;;;; 110D1;SORA SOMPENG LETTER TAH;Lo;0;L;;;;;N;;;;; 110D2;SORA SOMPENG LETTER BAH;Lo;0;L;;;;;N;;;;; @@ -19135,6 +19383,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11141;CHAKMA DANDA;Po;0;L;;;;;N;;;;; 11142;CHAKMA DOUBLE DANDA;Po;0;L;;;;;N;;;;; 11143;CHAKMA QUESTION MARK;Po;0;L;;;;;N;;;;; +11144;CHAKMA LETTER LHAA;Lo;0;L;;;;;N;;;;; +11145;CHAKMA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +11146;CHAKMA VOWEL SIGN EI;Mc;0;L;;;;;N;;;;; 11150;MAHAJANI LETTER A;Lo;0;L;;;;;N;;;;; 11151;MAHAJANI LETTER I;Lo;0;L;;;;;N;;;;; 11152;MAHAJANI LETTER U;Lo;0;L;;;;;N;;;;; @@ -19247,7 +19498,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 111C6;SHARADA DOUBLE DANDA;Po;0;L;;;;;N;;;;; 111C7;SHARADA ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; 111C8;SHARADA SEPARATOR;Po;0;L;;;;;N;;;;; -111C9;SHARADA SANDHI MARK;Po;0;L;;;;;N;;;;; +111C9;SHARADA SANDHI MARK;Mn;0;NSM;;;;;N;;;;; 111CA;SHARADA SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; 111CB;SHARADA VOWEL MODIFIER MARK;Mn;0;NSM;;;;;N;;;;; 111CC;SHARADA EXTRA SHORT VOWEL MARK;Mn;0;NSM;;;;;N;;;;; @@ -19507,6 +19758,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11337;GRANTHA LETTER SSA;Lo;0;L;;;;;N;;;;; 11338;GRANTHA LETTER SA;Lo;0;L;;;;;N;;;;; 11339;GRANTHA LETTER HA;Lo;0;L;;;;;N;;;;; +1133B;COMBINING BINDU BELOW;Mn;7;NSM;;;;;N;;;;; 1133C;GRANTHA SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; 1133D;GRANTHA SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; 1133E;GRANTHA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; @@ -19634,6 +19886,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11459;NEWA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1145B;NEWA PLACEHOLDER MARK;Po;0;L;;;;;N;;;;; 1145D;NEWA INSERTION SIGN;Po;0;L;;;;;N;;;;; +1145E;NEWA SANDHI MARK;Mn;230;NSM;;;;;N;;;;; 11480;TIRHUTA ANJI;Lo;0;L;;;;;N;;;;; 11481;TIRHUTA LETTER A;Lo;0;L;;;;;N;;;;; 11482;TIRHUTA LETTER AA;Lo;0;L;;;;;N;;;;; @@ -19992,6 +20245,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11717;AHOM LETTER GHA;Lo;0;L;;;;;N;;;;; 11718;AHOM LETTER BHA;Lo;0;L;;;;;N;;;;; 11719;AHOM LETTER JHA;Lo;0;L;;;;;N;;;;; +1171A;AHOM LETTER ALTERNATE BA;Lo;0;L;;;;;N;;;;; 1171D;AHOM CONSONANT SIGN MEDIAL LA;Mn;0;NSM;;;;;N;;;;; 1171E;AHOM CONSONANT SIGN MEDIAL RA;Mn;0;NSM;;;;;N;;;;; 1171F;AHOM CONSONANT SIGN MEDIAL LIGATING RA;Mn;0;NSM;;;;;N;;;;; @@ -20023,6 +20277,66 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1173D;AHOM SIGN SECTION;Po;0;L;;;;;N;;;;; 1173E;AHOM SIGN RULAI;Po;0;L;;;;;N;;;;; 1173F;AHOM SYMBOL VI;So;0;L;;;;;N;;;;; +11800;DOGRA LETTER A;Lo;0;L;;;;;N;;;;; +11801;DOGRA LETTER AA;Lo;0;L;;;;;N;;;;; +11802;DOGRA LETTER I;Lo;0;L;;;;;N;;;;; +11803;DOGRA LETTER II;Lo;0;L;;;;;N;;;;; +11804;DOGRA LETTER U;Lo;0;L;;;;;N;;;;; +11805;DOGRA LETTER UU;Lo;0;L;;;;;N;;;;; +11806;DOGRA LETTER E;Lo;0;L;;;;;N;;;;; +11807;DOGRA LETTER AI;Lo;0;L;;;;;N;;;;; +11808;DOGRA LETTER O;Lo;0;L;;;;;N;;;;; +11809;DOGRA LETTER AU;Lo;0;L;;;;;N;;;;; +1180A;DOGRA LETTER KA;Lo;0;L;;;;;N;;;;; +1180B;DOGRA LETTER KHA;Lo;0;L;;;;;N;;;;; +1180C;DOGRA LETTER GA;Lo;0;L;;;;;N;;;;; +1180D;DOGRA LETTER GHA;Lo;0;L;;;;;N;;;;; +1180E;DOGRA LETTER NGA;Lo;0;L;;;;;N;;;;; +1180F;DOGRA LETTER CA;Lo;0;L;;;;;N;;;;; +11810;DOGRA LETTER CHA;Lo;0;L;;;;;N;;;;; +11811;DOGRA LETTER JA;Lo;0;L;;;;;N;;;;; +11812;DOGRA LETTER JHA;Lo;0;L;;;;;N;;;;; +11813;DOGRA LETTER NYA;Lo;0;L;;;;;N;;;;; +11814;DOGRA LETTER TTA;Lo;0;L;;;;;N;;;;; +11815;DOGRA LETTER TTHA;Lo;0;L;;;;;N;;;;; +11816;DOGRA LETTER DDA;Lo;0;L;;;;;N;;;;; +11817;DOGRA LETTER DDHA;Lo;0;L;;;;;N;;;;; +11818;DOGRA LETTER NNA;Lo;0;L;;;;;N;;;;; +11819;DOGRA LETTER TA;Lo;0;L;;;;;N;;;;; +1181A;DOGRA LETTER THA;Lo;0;L;;;;;N;;;;; +1181B;DOGRA LETTER DA;Lo;0;L;;;;;N;;;;; +1181C;DOGRA LETTER DHA;Lo;0;L;;;;;N;;;;; +1181D;DOGRA LETTER NA;Lo;0;L;;;;;N;;;;; +1181E;DOGRA LETTER PA;Lo;0;L;;;;;N;;;;; +1181F;DOGRA LETTER PHA;Lo;0;L;;;;;N;;;;; +11820;DOGRA LETTER BA;Lo;0;L;;;;;N;;;;; +11821;DOGRA LETTER BHA;Lo;0;L;;;;;N;;;;; +11822;DOGRA LETTER MA;Lo;0;L;;;;;N;;;;; +11823;DOGRA LETTER YA;Lo;0;L;;;;;N;;;;; +11824;DOGRA LETTER RA;Lo;0;L;;;;;N;;;;; +11825;DOGRA LETTER LA;Lo;0;L;;;;;N;;;;; +11826;DOGRA LETTER VA;Lo;0;L;;;;;N;;;;; +11827;DOGRA LETTER SHA;Lo;0;L;;;;;N;;;;; +11828;DOGRA LETTER SSA;Lo;0;L;;;;;N;;;;; +11829;DOGRA LETTER SA;Lo;0;L;;;;;N;;;;; +1182A;DOGRA LETTER HA;Lo;0;L;;;;;N;;;;; +1182B;DOGRA LETTER RRA;Lo;0;L;;;;;N;;;;; +1182C;DOGRA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +1182D;DOGRA VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +1182E;DOGRA VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +1182F;DOGRA VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +11830;DOGRA VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +11831;DOGRA VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +11832;DOGRA VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;; +11833;DOGRA VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +11834;DOGRA VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +11835;DOGRA VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +11836;DOGRA VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;; +11837;DOGRA SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11838;DOGRA SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11839;DOGRA SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +1183A;DOGRA SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +1183B;DOGRA ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; 118A0;WARANG CITI CAPITAL LETTER NGAA;Lu;0;L;;;;;N;;;;118C0; 118A1;WARANG CITI CAPITAL LETTER A;Lu;0;L;;;;;N;;;;118C1; 118A2;WARANG CITI CAPITAL LETTER WI;Lu;0;L;;;;;N;;;;118C2; @@ -20114,8 +20428,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11A04;ZANABAZAR SQUARE VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; 11A05;ZANABAZAR SQUARE VOWEL SIGN OE;Mn;0;NSM;;;;;N;;;;; 11A06;ZANABAZAR SQUARE VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; -11A07;ZANABAZAR SQUARE VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; -11A08;ZANABAZAR SQUARE VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +11A07;ZANABAZAR SQUARE VOWEL SIGN AI;Mn;0;L;;;;;N;;;;; +11A08;ZANABAZAR SQUARE VOWEL SIGN AU;Mn;0;L;;;;;N;;;;; 11A09;ZANABAZAR SQUARE VOWEL SIGN REVERSED I;Mn;0;NSM;;;;;N;;;;; 11A0A;ZANABAZAR SQUARE VOWEL LENGTH MARK;Mn;0;NSM;;;;;N;;;;; 11A0B;ZANABAZAR SQUARE LETTER KA;Lo;0;L;;;;;N;;;;; @@ -20254,6 +20568,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11A9A;SOYOMBO MARK TSHEG;Po;0;L;;;;;N;;;;; 11A9B;SOYOMBO MARK SHAD;Po;0;L;;;;;N;;;;; 11A9C;SOYOMBO MARK DOUBLE SHAD;Po;0;L;;;;;N;;;;; +11A9D;SOYOMBO MARK PLUTA;Lo;0;L;;;;;N;;;;; 11A9E;SOYOMBO HEAD MARK WITH MOON AND SUN AND TRIPLE FLAME;Po;0;L;;;;;N;;;;; 11A9F;SOYOMBO HEAD MARK WITH MOON AND SUN AND FLAME;Po;0;L;;;;;N;;;;; 11AA0;SOYOMBO HEAD MARK WITH MOON AND SUN;Po;0;L;;;;;N;;;;; @@ -20556,6 +20871,94 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11D57;MASARAM GONDI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; 11D58;MASARAM GONDI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 11D59;MASARAM GONDI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +11D60;GUNJALA GONDI LETTER A;Lo;0;L;;;;;N;;;;; +11D61;GUNJALA GONDI LETTER AA;Lo;0;L;;;;;N;;;;; +11D62;GUNJALA GONDI LETTER I;Lo;0;L;;;;;N;;;;; +11D63;GUNJALA GONDI LETTER II;Lo;0;L;;;;;N;;;;; +11D64;GUNJALA GONDI LETTER U;Lo;0;L;;;;;N;;;;; +11D65;GUNJALA GONDI LETTER UU;Lo;0;L;;;;;N;;;;; +11D67;GUNJALA GONDI LETTER EE;Lo;0;L;;;;;N;;;;; +11D68;GUNJALA GONDI LETTER AI;Lo;0;L;;;;;N;;;;; +11D6A;GUNJALA GONDI LETTER OO;Lo;0;L;;;;;N;;;;; +11D6B;GUNJALA GONDI LETTER AU;Lo;0;L;;;;;N;;;;; +11D6C;GUNJALA GONDI LETTER YA;Lo;0;L;;;;;N;;;;; +11D6D;GUNJALA GONDI LETTER VA;Lo;0;L;;;;;N;;;;; +11D6E;GUNJALA GONDI LETTER BA;Lo;0;L;;;;;N;;;;; +11D6F;GUNJALA GONDI LETTER BHA;Lo;0;L;;;;;N;;;;; +11D70;GUNJALA GONDI LETTER MA;Lo;0;L;;;;;N;;;;; +11D71;GUNJALA GONDI LETTER KA;Lo;0;L;;;;;N;;;;; +11D72;GUNJALA GONDI LETTER KHA;Lo;0;L;;;;;N;;;;; +11D73;GUNJALA GONDI LETTER TA;Lo;0;L;;;;;N;;;;; +11D74;GUNJALA GONDI LETTER THA;Lo;0;L;;;;;N;;;;; +11D75;GUNJALA GONDI LETTER LA;Lo;0;L;;;;;N;;;;; +11D76;GUNJALA GONDI LETTER GA;Lo;0;L;;;;;N;;;;; +11D77;GUNJALA GONDI LETTER GHA;Lo;0;L;;;;;N;;;;; +11D78;GUNJALA GONDI LETTER DA;Lo;0;L;;;;;N;;;;; +11D79;GUNJALA GONDI LETTER DHA;Lo;0;L;;;;;N;;;;; +11D7A;GUNJALA GONDI LETTER NA;Lo;0;L;;;;;N;;;;; +11D7B;GUNJALA GONDI LETTER CA;Lo;0;L;;;;;N;;;;; +11D7C;GUNJALA GONDI LETTER CHA;Lo;0;L;;;;;N;;;;; +11D7D;GUNJALA GONDI LETTER TTA;Lo;0;L;;;;;N;;;;; +11D7E;GUNJALA GONDI LETTER TTHA;Lo;0;L;;;;;N;;;;; +11D7F;GUNJALA GONDI LETTER LLA;Lo;0;L;;;;;N;;;;; +11D80;GUNJALA GONDI LETTER JA;Lo;0;L;;;;;N;;;;; +11D81;GUNJALA GONDI LETTER JHA;Lo;0;L;;;;;N;;;;; +11D82;GUNJALA GONDI LETTER DDA;Lo;0;L;;;;;N;;;;; +11D83;GUNJALA GONDI LETTER DDHA;Lo;0;L;;;;;N;;;;; +11D84;GUNJALA GONDI LETTER NGA;Lo;0;L;;;;;N;;;;; +11D85;GUNJALA GONDI LETTER PA;Lo;0;L;;;;;N;;;;; +11D86;GUNJALA GONDI LETTER PHA;Lo;0;L;;;;;N;;;;; +11D87;GUNJALA GONDI LETTER HA;Lo;0;L;;;;;N;;;;; +11D88;GUNJALA GONDI LETTER RA;Lo;0;L;;;;;N;;;;; +11D89;GUNJALA GONDI LETTER SA;Lo;0;L;;;;;N;;;;; +11D8A;GUNJALA GONDI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +11D8B;GUNJALA GONDI VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +11D8C;GUNJALA GONDI VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +11D8D;GUNJALA GONDI VOWEL SIGN U;Mc;0;L;;;;;N;;;;; +11D8E;GUNJALA GONDI VOWEL SIGN UU;Mc;0;L;;;;;N;;;;; +11D90;GUNJALA GONDI VOWEL SIGN EE;Mn;0;NSM;;;;;N;;;;; +11D91;GUNJALA GONDI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +11D93;GUNJALA GONDI VOWEL SIGN OO;Mc;0;L;;;;;N;;;;; +11D94;GUNJALA GONDI VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +11D95;GUNJALA GONDI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11D96;GUNJALA GONDI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11D97;GUNJALA GONDI VIRAMA;Mn;9;NSM;;;;;N;;;;; +11D98;GUNJALA GONDI OM;Lo;0;L;;;;;N;;;;; +11DA0;GUNJALA GONDI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11DA1;GUNJALA GONDI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11DA2;GUNJALA GONDI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11DA3;GUNJALA GONDI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +11DA4;GUNJALA GONDI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +11DA5;GUNJALA GONDI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +11DA6;GUNJALA GONDI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +11DA7;GUNJALA GONDI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +11DA8;GUNJALA GONDI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +11DA9;GUNJALA GONDI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +11EE0;MAKASAR LETTER KA;Lo;0;L;;;;;N;;;;; +11EE1;MAKASAR LETTER GA;Lo;0;L;;;;;N;;;;; +11EE2;MAKASAR LETTER NGA;Lo;0;L;;;;;N;;;;; +11EE3;MAKASAR LETTER PA;Lo;0;L;;;;;N;;;;; +11EE4;MAKASAR LETTER BA;Lo;0;L;;;;;N;;;;; +11EE5;MAKASAR LETTER MA;Lo;0;L;;;;;N;;;;; +11EE6;MAKASAR LETTER TA;Lo;0;L;;;;;N;;;;; +11EE7;MAKASAR LETTER DA;Lo;0;L;;;;;N;;;;; +11EE8;MAKASAR LETTER NA;Lo;0;L;;;;;N;;;;; +11EE9;MAKASAR LETTER CA;Lo;0;L;;;;;N;;;;; +11EEA;MAKASAR LETTER JA;Lo;0;L;;;;;N;;;;; +11EEB;MAKASAR LETTER NYA;Lo;0;L;;;;;N;;;;; +11EEC;MAKASAR LETTER YA;Lo;0;L;;;;;N;;;;; +11EED;MAKASAR LETTER RA;Lo;0;L;;;;;N;;;;; +11EEE;MAKASAR LETTER LA;Lo;0;L;;;;;N;;;;; +11EEF;MAKASAR LETTER VA;Lo;0;L;;;;;N;;;;; +11EF0;MAKASAR LETTER SA;Lo;0;L;;;;;N;;;;; +11EF1;MAKASAR LETTER A;Lo;0;L;;;;;N;;;;; +11EF2;MAKASAR ANGKA;Lo;0;L;;;;;N;;;;; +11EF3;MAKASAR VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +11EF4;MAKASAR VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +11EF5;MAKASAR VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +11EF6;MAKASAR VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +11EF7;MAKASAR PASSIMBANG;Po;0;L;;;;;N;;;;; +11EF8;MAKASAR END OF SECTION;Po;0;L;;;;;N;;;;; 12000;CUNEIFORM SIGN A;Lo;0;L;;;;;N;;;;; 12001;CUNEIFORM SIGN A TIMES A;Lo;0;L;;;;;N;;;;; 12002;CUNEIFORM SIGN A TIMES BAD;Lo;0;L;;;;;N;;;;; @@ -24219,6 +24622,97 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16B8D;PAHAWH HMONG CLAN SIGN TSWB;Lo;0;L;;;;;N;;;;; 16B8E;PAHAWH HMONG CLAN SIGN KWM;Lo;0;L;;;;;N;;;;; 16B8F;PAHAWH HMONG CLAN SIGN VWJ;Lo;0;L;;;;;N;;;;; +16E40;MEDEFAIDRIN CAPITAL LETTER M;Lu;0;L;;;;;N;;;;16E60; +16E41;MEDEFAIDRIN CAPITAL LETTER S;Lu;0;L;;;;;N;;;;16E61; +16E42;MEDEFAIDRIN CAPITAL LETTER V;Lu;0;L;;;;;N;;;;16E62; +16E43;MEDEFAIDRIN CAPITAL LETTER W;Lu;0;L;;;;;N;;;;16E63; +16E44;MEDEFAIDRIN CAPITAL LETTER ATIU;Lu;0;L;;;;;N;;;;16E64; +16E45;MEDEFAIDRIN CAPITAL LETTER Z;Lu;0;L;;;;;N;;;;16E65; +16E46;MEDEFAIDRIN CAPITAL LETTER KP;Lu;0;L;;;;;N;;;;16E66; +16E47;MEDEFAIDRIN CAPITAL LETTER P;Lu;0;L;;;;;N;;;;16E67; +16E48;MEDEFAIDRIN CAPITAL LETTER T;Lu;0;L;;;;;N;;;;16E68; +16E49;MEDEFAIDRIN CAPITAL LETTER G;Lu;0;L;;;;;N;;;;16E69; +16E4A;MEDEFAIDRIN CAPITAL LETTER F;Lu;0;L;;;;;N;;;;16E6A; +16E4B;MEDEFAIDRIN CAPITAL LETTER I;Lu;0;L;;;;;N;;;;16E6B; +16E4C;MEDEFAIDRIN CAPITAL LETTER K;Lu;0;L;;;;;N;;;;16E6C; +16E4D;MEDEFAIDRIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;16E6D; +16E4E;MEDEFAIDRIN CAPITAL LETTER J;Lu;0;L;;;;;N;;;;16E6E; +16E4F;MEDEFAIDRIN CAPITAL LETTER E;Lu;0;L;;;;;N;;;;16E6F; +16E50;MEDEFAIDRIN CAPITAL LETTER B;Lu;0;L;;;;;N;;;;16E70; +16E51;MEDEFAIDRIN CAPITAL LETTER C;Lu;0;L;;;;;N;;;;16E71; +16E52;MEDEFAIDRIN CAPITAL LETTER U;Lu;0;L;;;;;N;;;;16E72; +16E53;MEDEFAIDRIN CAPITAL LETTER YU;Lu;0;L;;;;;N;;;;16E73; +16E54;MEDEFAIDRIN CAPITAL LETTER L;Lu;0;L;;;;;N;;;;16E74; +16E55;MEDEFAIDRIN CAPITAL LETTER Q;Lu;0;L;;;;;N;;;;16E75; +16E56;MEDEFAIDRIN CAPITAL LETTER HP;Lu;0;L;;;;;N;;;;16E76; +16E57;MEDEFAIDRIN CAPITAL LETTER NY;Lu;0;L;;;;;N;;;;16E77; +16E58;MEDEFAIDRIN CAPITAL LETTER X;Lu;0;L;;;;;N;;;;16E78; +16E59;MEDEFAIDRIN CAPITAL LETTER D;Lu;0;L;;;;;N;;;;16E79; +16E5A;MEDEFAIDRIN CAPITAL LETTER OE;Lu;0;L;;;;;N;;;;16E7A; +16E5B;MEDEFAIDRIN CAPITAL LETTER N;Lu;0;L;;;;;N;;;;16E7B; +16E5C;MEDEFAIDRIN CAPITAL LETTER R;Lu;0;L;;;;;N;;;;16E7C; +16E5D;MEDEFAIDRIN CAPITAL LETTER O;Lu;0;L;;;;;N;;;;16E7D; +16E5E;MEDEFAIDRIN CAPITAL LETTER AI;Lu;0;L;;;;;N;;;;16E7E; +16E5F;MEDEFAIDRIN CAPITAL LETTER Y;Lu;0;L;;;;;N;;;;16E7F; +16E60;MEDEFAIDRIN SMALL LETTER M;Ll;0;L;;;;;N;;;16E40;;16E40 +16E61;MEDEFAIDRIN SMALL LETTER S;Ll;0;L;;;;;N;;;16E41;;16E41 +16E62;MEDEFAIDRIN SMALL LETTER V;Ll;0;L;;;;;N;;;16E42;;16E42 +16E63;MEDEFAIDRIN SMALL LETTER W;Ll;0;L;;;;;N;;;16E43;;16E43 +16E64;MEDEFAIDRIN SMALL LETTER ATIU;Ll;0;L;;;;;N;;;16E44;;16E44 +16E65;MEDEFAIDRIN SMALL LETTER Z;Ll;0;L;;;;;N;;;16E45;;16E45 +16E66;MEDEFAIDRIN SMALL LETTER KP;Ll;0;L;;;;;N;;;16E46;;16E46 +16E67;MEDEFAIDRIN SMALL LETTER P;Ll;0;L;;;;;N;;;16E47;;16E47 +16E68;MEDEFAIDRIN SMALL LETTER T;Ll;0;L;;;;;N;;;16E48;;16E48 +16E69;MEDEFAIDRIN SMALL LETTER G;Ll;0;L;;;;;N;;;16E49;;16E49 +16E6A;MEDEFAIDRIN SMALL LETTER F;Ll;0;L;;;;;N;;;16E4A;;16E4A +16E6B;MEDEFAIDRIN SMALL LETTER I;Ll;0;L;;;;;N;;;16E4B;;16E4B +16E6C;MEDEFAIDRIN SMALL LETTER K;Ll;0;L;;;;;N;;;16E4C;;16E4C +16E6D;MEDEFAIDRIN SMALL LETTER A;Ll;0;L;;;;;N;;;16E4D;;16E4D +16E6E;MEDEFAIDRIN SMALL LETTER J;Ll;0;L;;;;;N;;;16E4E;;16E4E +16E6F;MEDEFAIDRIN SMALL LETTER E;Ll;0;L;;;;;N;;;16E4F;;16E4F +16E70;MEDEFAIDRIN SMALL LETTER B;Ll;0;L;;;;;N;;;16E50;;16E50 +16E71;MEDEFAIDRIN SMALL LETTER C;Ll;0;L;;;;;N;;;16E51;;16E51 +16E72;MEDEFAIDRIN SMALL LETTER U;Ll;0;L;;;;;N;;;16E52;;16E52 +16E73;MEDEFAIDRIN SMALL LETTER YU;Ll;0;L;;;;;N;;;16E53;;16E53 +16E74;MEDEFAIDRIN SMALL LETTER L;Ll;0;L;;;;;N;;;16E54;;16E54 +16E75;MEDEFAIDRIN SMALL LETTER Q;Ll;0;L;;;;;N;;;16E55;;16E55 +16E76;MEDEFAIDRIN SMALL LETTER HP;Ll;0;L;;;;;N;;;16E56;;16E56 +16E77;MEDEFAIDRIN SMALL LETTER NY;Ll;0;L;;;;;N;;;16E57;;16E57 +16E78;MEDEFAIDRIN SMALL LETTER X;Ll;0;L;;;;;N;;;16E58;;16E58 +16E79;MEDEFAIDRIN SMALL LETTER D;Ll;0;L;;;;;N;;;16E59;;16E59 +16E7A;MEDEFAIDRIN SMALL LETTER OE;Ll;0;L;;;;;N;;;16E5A;;16E5A +16E7B;MEDEFAIDRIN SMALL LETTER N;Ll;0;L;;;;;N;;;16E5B;;16E5B +16E7C;MEDEFAIDRIN SMALL LETTER R;Ll;0;L;;;;;N;;;16E5C;;16E5C +16E7D;MEDEFAIDRIN SMALL LETTER O;Ll;0;L;;;;;N;;;16E5D;;16E5D +16E7E;MEDEFAIDRIN SMALL LETTER AI;Ll;0;L;;;;;N;;;16E5E;;16E5E +16E7F;MEDEFAIDRIN SMALL LETTER Y;Ll;0;L;;;;;N;;;16E5F;;16E5F +16E80;MEDEFAIDRIN DIGIT ZERO;No;0;L;;;;0;N;;;;; +16E81;MEDEFAIDRIN DIGIT ONE;No;0;L;;;;1;N;;;;; +16E82;MEDEFAIDRIN DIGIT TWO;No;0;L;;;;2;N;;;;; +16E83;MEDEFAIDRIN DIGIT THREE;No;0;L;;;;3;N;;;;; +16E84;MEDEFAIDRIN DIGIT FOUR;No;0;L;;;;4;N;;;;; +16E85;MEDEFAIDRIN DIGIT FIVE;No;0;L;;;;5;N;;;;; +16E86;MEDEFAIDRIN DIGIT SIX;No;0;L;;;;6;N;;;;; +16E87;MEDEFAIDRIN DIGIT SEVEN;No;0;L;;;;7;N;;;;; +16E88;MEDEFAIDRIN DIGIT EIGHT;No;0;L;;;;8;N;;;;; +16E89;MEDEFAIDRIN DIGIT NINE;No;0;L;;;;9;N;;;;; +16E8A;MEDEFAIDRIN NUMBER TEN;No;0;L;;;;10;N;;;;; +16E8B;MEDEFAIDRIN NUMBER ELEVEN;No;0;L;;;;11;N;;;;; +16E8C;MEDEFAIDRIN NUMBER TWELVE;No;0;L;;;;12;N;;;;; +16E8D;MEDEFAIDRIN NUMBER THIRTEEN;No;0;L;;;;13;N;;;;; +16E8E;MEDEFAIDRIN NUMBER FOURTEEN;No;0;L;;;;14;N;;;;; +16E8F;MEDEFAIDRIN NUMBER FIFTEEN;No;0;L;;;;15;N;;;;; +16E90;MEDEFAIDRIN NUMBER SIXTEEN;No;0;L;;;;16;N;;;;; +16E91;MEDEFAIDRIN NUMBER SEVENTEEN;No;0;L;;;;17;N;;;;; +16E92;MEDEFAIDRIN NUMBER EIGHTEEN;No;0;L;;;;18;N;;;;; +16E93;MEDEFAIDRIN NUMBER NINETEEN;No;0;L;;;;19;N;;;;; +16E94;MEDEFAIDRIN DIGIT ONE ALTERNATE FORM;No;0;L;;;;1;N;;;;; +16E95;MEDEFAIDRIN DIGIT TWO ALTERNATE FORM;No;0;L;;;;2;N;;;;; +16E96;MEDEFAIDRIN DIGIT THREE ALTERNATE FORM;No;0;L;;;;3;N;;;;; +16E97;MEDEFAIDRIN COMMA;Po;0;L;;;;;N;;;;; +16E98;MEDEFAIDRIN FULL STOP;Po;0;L;;;;;N;;;;; +16E99;MEDEFAIDRIN SYMBOL AIVA;Po;0;L;;;;;N;;;;; +16E9A;MEDEFAIDRIN EXCLAMATION OH;Po;0;L;;;;;N;;;;; 16F00;MIAO LETTER PA;Lo;0;L;;;;;N;;;;; 16F01;MIAO LETTER BA;Lo;0;L;;;;;N;;;;; 16F02;MIAO LETTER YI PA;Lo;0;L;;;;;N;;;;; @@ -24355,7 +24849,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16FE0;TANGUT ITERATION MARK;Lm;0;L;;;;;N;;;;; 16FE1;NUSHU ITERATION MARK;Lm;0;L;;;;;N;;;;; 17000;<Tangut Ideograph, First>;Lo;0;L;;;;;N;;;;; -187EC;<Tangut Ideograph, Last>;Lo;0;L;;;;;N;;;;; +187F1;<Tangut Ideograph, Last>;Lo;0;L;;;;;N;;;;; 18800;TANGUT COMPONENT-001;Lo;0;L;;;;;N;;;;; 18801;TANGUT COMPONENT-002;Lo;0;L;;;;;N;;;;; 18802;TANGUT COMPONENT-003;Lo;0;L;;;;;N;;;;; @@ -26488,6 +26982,26 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D243;COMBINING GREEK MUSICAL TETRASEME;Mn;230;NSM;;;;;N;;;;; 1D244;COMBINING GREEK MUSICAL PENTASEME;Mn;230;NSM;;;;;N;;;;; 1D245;GREEK MUSICAL LEIMMA;So;0;ON;;;;;N;;;;; +1D2E0;MAYAN NUMERAL ZERO;No;0;L;;;;0;N;;;;; +1D2E1;MAYAN NUMERAL ONE;No;0;L;;;;1;N;;;;; +1D2E2;MAYAN NUMERAL TWO;No;0;L;;;;2;N;;;;; +1D2E3;MAYAN NUMERAL THREE;No;0;L;;;;3;N;;;;; +1D2E4;MAYAN NUMERAL FOUR;No;0;L;;;;4;N;;;;; +1D2E5;MAYAN NUMERAL FIVE;No;0;L;;;;5;N;;;;; +1D2E6;MAYAN NUMERAL SIX;No;0;L;;;;6;N;;;;; +1D2E7;MAYAN NUMERAL SEVEN;No;0;L;;;;7;N;;;;; +1D2E8;MAYAN NUMERAL EIGHT;No;0;L;;;;8;N;;;;; +1D2E9;MAYAN NUMERAL NINE;No;0;L;;;;9;N;;;;; +1D2EA;MAYAN NUMERAL TEN;No;0;L;;;;10;N;;;;; +1D2EB;MAYAN NUMERAL ELEVEN;No;0;L;;;;11;N;;;;; +1D2EC;MAYAN NUMERAL TWELVE;No;0;L;;;;12;N;;;;; +1D2ED;MAYAN NUMERAL THIRTEEN;No;0;L;;;;13;N;;;;; +1D2EE;MAYAN NUMERAL FOURTEEN;No;0;L;;;;14;N;;;;; +1D2EF;MAYAN NUMERAL FIFTEEN;No;0;L;;;;15;N;;;;; +1D2F0;MAYAN NUMERAL SIXTEEN;No;0;L;;;;16;N;;;;; +1D2F1;MAYAN NUMERAL SEVENTEEN;No;0;L;;;;17;N;;;;; +1D2F2;MAYAN NUMERAL EIGHTEEN;No;0;L;;;;18;N;;;;; +1D2F3;MAYAN NUMERAL NINETEEN;No;0;L;;;;19;N;;;;; 1D300;MONOGRAM FOR EARTH;So;0;ON;;;;;N;;;;; 1D301;DIGRAM FOR HEAVENLY EARTH;So;0;ON;;;;;N;;;;; 1D302;DIGRAM FOR HUMAN EARTH;So;0;ON;;;;;N;;;;; @@ -26593,6 +27107,13 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D36F;COUNTING ROD TENS DIGIT SEVEN;No;0;L;;;;70;N;;;;; 1D370;COUNTING ROD TENS DIGIT EIGHT;No;0;L;;;;80;N;;;;; 1D371;COUNTING ROD TENS DIGIT NINE;No;0;L;;;;90;N;;;;; +1D372;IDEOGRAPHIC TALLY MARK ONE;No;0;L;;;;1;N;;;;; +1D373;IDEOGRAPHIC TALLY MARK TWO;No;0;L;;;;2;N;;;;; +1D374;IDEOGRAPHIC TALLY MARK THREE;No;0;L;;;;3;N;;;;; +1D375;IDEOGRAPHIC TALLY MARK FOUR;No;0;L;;;;4;N;;;;; +1D376;IDEOGRAPHIC TALLY MARK FIVE;No;0;L;;;;5;N;;;;; +1D377;TALLY MARK ONE;No;0;L;;;;1;N;;;;; +1D378;TALLY MARK FIVE;No;0;L;;;;5;N;;;;; 1D400;MATHEMATICAL BOLD CAPITAL A;Lu;0;L;<font> 0041;;;;N;;;;; 1D401;MATHEMATICAL BOLD CAPITAL B;Lu;0;L;<font> 0042;;;;N;;;;; 1D402;MATHEMATICAL BOLD CAPITAL C;Lu;0;L;<font> 0043;;;;N;;;;; @@ -28599,6 +29120,74 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E959;ADLAM DIGIT NINE;Nd;0;R;;9;9;9;N;;;;; 1E95E;ADLAM INITIAL EXCLAMATION MARK;Po;0;R;;;;;N;;;;; 1E95F;ADLAM INITIAL QUESTION MARK;Po;0;R;;;;;N;;;;; +1EC71;INDIC SIYAQ NUMBER ONE;No;0;AL;;;;1;N;;;;; +1EC72;INDIC SIYAQ NUMBER TWO;No;0;AL;;;;2;N;;;;; +1EC73;INDIC SIYAQ NUMBER THREE;No;0;AL;;;;3;N;;;;; +1EC74;INDIC SIYAQ NUMBER FOUR;No;0;AL;;;;4;N;;;;; +1EC75;INDIC SIYAQ NUMBER FIVE;No;0;AL;;;;5;N;;;;; +1EC76;INDIC SIYAQ NUMBER SIX;No;0;AL;;;;6;N;;;;; +1EC77;INDIC SIYAQ NUMBER SEVEN;No;0;AL;;;;7;N;;;;; +1EC78;INDIC SIYAQ NUMBER EIGHT;No;0;AL;;;;8;N;;;;; +1EC79;INDIC SIYAQ NUMBER NINE;No;0;AL;;;;9;N;;;;; +1EC7A;INDIC SIYAQ NUMBER TEN;No;0;AL;;;;10;N;;;;; +1EC7B;INDIC SIYAQ NUMBER TWENTY;No;0;AL;;;;20;N;;;;; +1EC7C;INDIC SIYAQ NUMBER THIRTY;No;0;AL;;;;30;N;;;;; +1EC7D;INDIC SIYAQ NUMBER FORTY;No;0;AL;;;;40;N;;;;; +1EC7E;INDIC SIYAQ NUMBER FIFTY;No;0;AL;;;;50;N;;;;; +1EC7F;INDIC SIYAQ NUMBER SIXTY;No;0;AL;;;;60;N;;;;; +1EC80;INDIC SIYAQ NUMBER SEVENTY;No;0;AL;;;;70;N;;;;; +1EC81;INDIC SIYAQ NUMBER EIGHTY;No;0;AL;;;;80;N;;;;; +1EC82;INDIC SIYAQ NUMBER NINETY;No;0;AL;;;;90;N;;;;; +1EC83;INDIC SIYAQ NUMBER ONE HUNDRED;No;0;AL;;;;100;N;;;;; +1EC84;INDIC SIYAQ NUMBER TWO HUNDRED;No;0;AL;;;;200;N;;;;; +1EC85;INDIC SIYAQ NUMBER THREE HUNDRED;No;0;AL;;;;300;N;;;;; +1EC86;INDIC SIYAQ NUMBER FOUR HUNDRED;No;0;AL;;;;400;N;;;;; +1EC87;INDIC SIYAQ NUMBER FIVE HUNDRED;No;0;AL;;;;500;N;;;;; +1EC88;INDIC SIYAQ NUMBER SIX HUNDRED;No;0;AL;;;;600;N;;;;; +1EC89;INDIC SIYAQ NUMBER SEVEN HUNDRED;No;0;AL;;;;700;N;;;;; +1EC8A;INDIC SIYAQ NUMBER EIGHT HUNDRED;No;0;AL;;;;800;N;;;;; +1EC8B;INDIC SIYAQ NUMBER NINE HUNDRED;No;0;AL;;;;900;N;;;;; +1EC8C;INDIC SIYAQ NUMBER ONE THOUSAND;No;0;AL;;;;1000;N;;;;; +1EC8D;INDIC SIYAQ NUMBER TWO THOUSAND;No;0;AL;;;;2000;N;;;;; +1EC8E;INDIC SIYAQ NUMBER THREE THOUSAND;No;0;AL;;;;3000;N;;;;; +1EC8F;INDIC SIYAQ NUMBER FOUR THOUSAND;No;0;AL;;;;4000;N;;;;; +1EC90;INDIC SIYAQ NUMBER FIVE THOUSAND;No;0;AL;;;;5000;N;;;;; +1EC91;INDIC SIYAQ NUMBER SIX THOUSAND;No;0;AL;;;;6000;N;;;;; +1EC92;INDIC SIYAQ NUMBER SEVEN THOUSAND;No;0;AL;;;;7000;N;;;;; +1EC93;INDIC SIYAQ NUMBER EIGHT THOUSAND;No;0;AL;;;;8000;N;;;;; +1EC94;INDIC SIYAQ NUMBER NINE THOUSAND;No;0;AL;;;;9000;N;;;;; +1EC95;INDIC SIYAQ NUMBER TEN THOUSAND;No;0;AL;;;;10000;N;;;;; +1EC96;INDIC SIYAQ NUMBER TWENTY THOUSAND;No;0;AL;;;;20000;N;;;;; +1EC97;INDIC SIYAQ NUMBER THIRTY THOUSAND;No;0;AL;;;;30000;N;;;;; +1EC98;INDIC SIYAQ NUMBER FORTY THOUSAND;No;0;AL;;;;40000;N;;;;; +1EC99;INDIC SIYAQ NUMBER FIFTY THOUSAND;No;0;AL;;;;50000;N;;;;; +1EC9A;INDIC SIYAQ NUMBER SIXTY THOUSAND;No;0;AL;;;;60000;N;;;;; +1EC9B;INDIC SIYAQ NUMBER SEVENTY THOUSAND;No;0;AL;;;;70000;N;;;;; +1EC9C;INDIC SIYAQ NUMBER EIGHTY THOUSAND;No;0;AL;;;;80000;N;;;;; +1EC9D;INDIC SIYAQ NUMBER NINETY THOUSAND;No;0;AL;;;;90000;N;;;;; +1EC9E;INDIC SIYAQ NUMBER LAKH;No;0;AL;;;;100000;N;;;;; +1EC9F;INDIC SIYAQ NUMBER LAKHAN;No;0;AL;;;;200000;N;;;;; +1ECA0;INDIC SIYAQ LAKH MARK;No;0;AL;;;;100000;N;;;;; +1ECA1;INDIC SIYAQ NUMBER KAROR;No;0;AL;;;;10000000;N;;;;; +1ECA2;INDIC SIYAQ NUMBER KARORAN;No;0;AL;;;;20000000;N;;;;; +1ECA3;INDIC SIYAQ NUMBER PREFIXED ONE;No;0;AL;;;;1;N;;;;; +1ECA4;INDIC SIYAQ NUMBER PREFIXED TWO;No;0;AL;;;;2;N;;;;; +1ECA5;INDIC SIYAQ NUMBER PREFIXED THREE;No;0;AL;;;;3;N;;;;; +1ECA6;INDIC SIYAQ NUMBER PREFIXED FOUR;No;0;AL;;;;4;N;;;;; +1ECA7;INDIC SIYAQ NUMBER PREFIXED FIVE;No;0;AL;;;;5;N;;;;; +1ECA8;INDIC SIYAQ NUMBER PREFIXED SIX;No;0;AL;;;;6;N;;;;; +1ECA9;INDIC SIYAQ NUMBER PREFIXED SEVEN;No;0;AL;;;;7;N;;;;; +1ECAA;INDIC SIYAQ NUMBER PREFIXED EIGHT;No;0;AL;;;;8;N;;;;; +1ECAB;INDIC SIYAQ NUMBER PREFIXED NINE;No;0;AL;;;;9;N;;;;; +1ECAC;INDIC SIYAQ PLACEHOLDER;So;0;AL;;;;;N;;;;; +1ECAD;INDIC SIYAQ FRACTION ONE QUARTER;No;0;AL;;;;1/4;N;;;;; +1ECAE;INDIC SIYAQ FRACTION ONE HALF;No;0;AL;;;;1/2;N;;;;; +1ECAF;INDIC SIYAQ FRACTION THREE QUARTERS;No;0;AL;;;;3/4;N;;;;; +1ECB0;INDIC SIYAQ RUPEE MARK;Sc;0;AL;;;;;N;;;;; +1ECB1;INDIC SIYAQ NUMBER ALTERNATE ONE;No;0;AL;;;;1;N;;;;; +1ECB2;INDIC SIYAQ NUMBER ALTERNATE TWO;No;0;AL;;;;2;N;;;;; +1ECB3;INDIC SIYAQ NUMBER ALTERNATE TEN THOUSAND;No;0;AL;;;;10000;N;;;;; +1ECB4;INDIC SIYAQ ALTERNATE LAKH MARK;No;0;AL;;;;100000;N;;;;; 1EE00;ARABIC MATHEMATICAL ALEF;Lo;0;AL;<font> 0627;;;;N;;;;; 1EE01;ARABIC MATHEMATICAL BEH;Lo;0;AL;<font> 0628;;;;N;;;;; 1EE02;ARABIC MATHEMATICAL JEEM;Lo;0;AL;<font> 062C;;;;N;;;;; @@ -29012,6 +29601,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F12C;CIRCLED ITALIC LATIN CAPITAL LETTER R;So;0;L;<circle> 0052;;;;N;;;;; 1F12D;CIRCLED CD;So;0;L;<circle> 0043 0044;;;;N;;;;; 1F12E;CIRCLED WZ;So;0;L;<circle> 0057 005A;;;;N;;;;; +1F12F;COPYLEFT SYMBOL;So;0;ON;;;;;N;;;;; 1F130;SQUARED LATIN CAPITAL LETTER A;So;0;L;<square> 0041;;;;N;;;;; 1F131;SQUARED LATIN CAPITAL LETTER B;So;0;L;<square> 0042;;;;N;;;;; 1F132;SQUARED LATIN CAPITAL LETTER C;So;0;L;<square> 0043;;;;N;;;;; @@ -30226,6 +30816,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6F6;CANOE;So;0;ON;;;;;N;;;;; 1F6F7;SLED;So;0;ON;;;;;N;;;;; 1F6F8;FLYING SAUCER;So;0;ON;;;;;N;;;;; +1F6F9;SKATEBOARD;So;0;ON;;;;;N;;;;; 1F700;ALCHEMICAL SYMBOL FOR QUINTESSENCE;So;0;ON;;;;;N;;;;; 1F701;ALCHEMICAL SYMBOL FOR AIR;So;0;ON;;;;;N;;;;; 1F702;ALCHEMICAL SYMBOL FOR FIRE;So;0;ON;;;;;N;;;;; @@ -30427,6 +31018,10 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F7D2;LIGHT TWELVE POINTED BLACK STAR;So;0;ON;;;;;N;;;;; 1F7D3;HEAVY TWELVE POINTED BLACK STAR;So;0;ON;;;;;N;;;;; 1F7D4;HEAVY TWELVE POINTED PINWHEEL STAR;So;0;ON;;;;;N;;;;; +1F7D5;CIRCLED TRIANGLE;So;0;ON;;;;;N;;;;; +1F7D6;NEGATIVE CIRCLED TRIANGLE;So;0;ON;;;;;N;;;;; +1F7D7;CIRCLED SQUARE;So;0;ON;;;;;N;;;;; +1F7D8;NEGATIVE CIRCLED SQUARE;So;0;ON;;;;;N;;;;; 1F800;LEFTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD;So;0;ON;;;;;N;;;;; 1F801;UPWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD;So;0;ON;;;;;N;;;;; 1F802;RIGHTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD;So;0;ON;;;;;N;;;;; @@ -30647,6 +31242,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F94A;BOXING GLOVE;So;0;ON;;;;;N;;;;; 1F94B;MARTIAL ARTS UNIFORM;So;0;ON;;;;;N;;;;; 1F94C;CURLING STONE;So;0;ON;;;;;N;;;;; +1F94D;LACROSSE STICK AND BALL;So;0;ON;;;;;N;;;;; +1F94E;SOFTBALL;So;0;ON;;;;;N;;;;; +1F94F;FLYING DISC;So;0;ON;;;;;N;;;;; 1F950;CROISSANT;So;0;ON;;;;;N;;;;; 1F951;AVOCADO;So;0;ON;;;;;N;;;;; 1F952;CUCUMBER;So;0;ON;;;;;N;;;;; @@ -30675,6 +31273,20 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F969;CUT OF MEAT;So;0;ON;;;;;N;;;;; 1F96A;SANDWICH;So;0;ON;;;;;N;;;;; 1F96B;CANNED FOOD;So;0;ON;;;;;N;;;;; +1F96C;LEAFY GREEN;So;0;ON;;;;;N;;;;; +1F96D;MANGO;So;0;ON;;;;;N;;;;; +1F96E;MOON CAKE;So;0;ON;;;;;N;;;;; +1F96F;BAGEL;So;0;ON;;;;;N;;;;; +1F970;SMILING FACE WITH SMILING EYES AND THREE HEARTS;So;0;ON;;;;;N;;;;; +1F973;FACE WITH PARTY HORN AND PARTY HAT;So;0;ON;;;;;N;;;;; +1F974;FACE WITH UNEVEN EYES AND WAVY MOUTH;So;0;ON;;;;;N;;;;; +1F975;OVERHEATED FACE;So;0;ON;;;;;N;;;;; +1F976;FREEZING FACE;So;0;ON;;;;;N;;;;; +1F97A;FACE WITH PLEADING EYES;So;0;ON;;;;;N;;;;; +1F97C;LAB COAT;So;0;ON;;;;;N;;;;; +1F97D;GOGGLES;So;0;ON;;;;;N;;;;; +1F97E;HIKING BOOT;So;0;ON;;;;;N;;;;; +1F97F;FLAT SHOE;So;0;ON;;;;;N;;;;; 1F980;CRAB;So;0;ON;;;;;N;;;;; 1F981;LION FACE;So;0;ON;;;;;N;;;;; 1F982;SCORPION;So;0;ON;;;;;N;;;;; @@ -30699,7 +31311,30 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F995;SAUROPOD;So;0;ON;;;;;N;;;;; 1F996;T-REX;So;0;ON;;;;;N;;;;; 1F997;CRICKET;So;0;ON;;;;;N;;;;; +1F998;KANGAROO;So;0;ON;;;;;N;;;;; +1F999;LLAMA;So;0;ON;;;;;N;;;;; +1F99A;PEACOCK;So;0;ON;;;;;N;;;;; +1F99B;HIPPOPOTAMUS;So;0;ON;;;;;N;;;;; +1F99C;PARROT;So;0;ON;;;;;N;;;;; +1F99D;RACCOON;So;0;ON;;;;;N;;;;; +1F99E;LOBSTER;So;0;ON;;;;;N;;;;; +1F99F;MOSQUITO;So;0;ON;;;;;N;;;;; +1F9A0;MICROBE;So;0;ON;;;;;N;;;;; +1F9A1;BADGER;So;0;ON;;;;;N;;;;; +1F9A2;SWAN;So;0;ON;;;;;N;;;;; +1F9B0;EMOJI COMPONENT RED HAIR;So;0;ON;;;;;N;;;;; +1F9B1;EMOJI COMPONENT CURLY HAIR;So;0;ON;;;;;N;;;;; +1F9B2;EMOJI COMPONENT BALD;So;0;ON;;;;;N;;;;; +1F9B3;EMOJI COMPONENT WHITE HAIR;So;0;ON;;;;;N;;;;; +1F9B4;BONE;So;0;ON;;;;;N;;;;; +1F9B5;LEG;So;0;ON;;;;;N;;;;; +1F9B6;FOOT;So;0;ON;;;;;N;;;;; +1F9B7;TOOTH;So;0;ON;;;;;N;;;;; +1F9B8;SUPERHERO;So;0;ON;;;;;N;;;;; +1F9B9;SUPERVILLAIN;So;0;ON;;;;;N;;;;; 1F9C0;CHEESE WEDGE;So;0;ON;;;;;N;;;;; +1F9C1;CUPCAKE;So;0;ON;;;;;N;;;;; +1F9C2;SALT SHAKER;So;0;ON;;;;;N;;;;; 1F9D0;FACE WITH MONOCLE;So;0;ON;;;;;N;;;;; 1F9D1;ADULT;So;0;ON;;;;;N;;;;; 1F9D2;CHILD;So;0;ON;;;;;N;;;;; @@ -30723,6 +31358,45 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F9E4;GLOVES;So;0;ON;;;;;N;;;;; 1F9E5;COAT;So;0;ON;;;;;N;;;;; 1F9E6;SOCKS;So;0;ON;;;;;N;;;;; +1F9E7;RED GIFT ENVELOPE;So;0;ON;;;;;N;;;;; +1F9E8;FIRECRACKER;So;0;ON;;;;;N;;;;; +1F9E9;JIGSAW PUZZLE PIECE;So;0;ON;;;;;N;;;;; +1F9EA;TEST TUBE;So;0;ON;;;;;N;;;;; +1F9EB;PETRI DISH;So;0;ON;;;;;N;;;;; +1F9EC;DNA DOUBLE HELIX;So;0;ON;;;;;N;;;;; +1F9ED;COMPASS;So;0;ON;;;;;N;;;;; +1F9EE;ABACUS;So;0;ON;;;;;N;;;;; +1F9EF;FIRE EXTINGUISHER;So;0;ON;;;;;N;;;;; +1F9F0;TOOLBOX;So;0;ON;;;;;N;;;;; +1F9F1;BRICK;So;0;ON;;;;;N;;;;; +1F9F2;MAGNET;So;0;ON;;;;;N;;;;; +1F9F3;LUGGAGE;So;0;ON;;;;;N;;;;; +1F9F4;LOTION BOTTLE;So;0;ON;;;;;N;;;;; +1F9F5;SPOOL OF THREAD;So;0;ON;;;;;N;;;;; +1F9F6;BALL OF YARN;So;0;ON;;;;;N;;;;; +1F9F7;SAFETY PIN;So;0;ON;;;;;N;;;;; +1F9F8;TEDDY BEAR;So;0;ON;;;;;N;;;;; +1F9F9;BROOM;So;0;ON;;;;;N;;;;; +1F9FA;BASKET;So;0;ON;;;;;N;;;;; +1F9FB;ROLL OF PAPER;So;0;ON;;;;;N;;;;; +1F9FC;BAR OF SOAP;So;0;ON;;;;;N;;;;; +1F9FD;SPONGE;So;0;ON;;;;;N;;;;; +1F9FE;RECEIPT;So;0;ON;;;;;N;;;;; +1F9FF;NAZAR AMULET;So;0;ON;;;;;N;;;;; +1FA60;XIANGQI RED GENERAL;So;0;ON;;;;;N;;;;; +1FA61;XIANGQI RED MANDARIN;So;0;ON;;;;;N;;;;; +1FA62;XIANGQI RED ELEPHANT;So;0;ON;;;;;N;;;;; +1FA63;XIANGQI RED HORSE;So;0;ON;;;;;N;;;;; +1FA64;XIANGQI RED CHARIOT;So;0;ON;;;;;N;;;;; +1FA65;XIANGQI RED CANNON;So;0;ON;;;;;N;;;;; +1FA66;XIANGQI RED SOLDIER;So;0;ON;;;;;N;;;;; +1FA67;XIANGQI BLACK GENERAL;So;0;ON;;;;;N;;;;; +1FA68;XIANGQI BLACK MANDARIN;So;0;ON;;;;;N;;;;; +1FA69;XIANGQI BLACK ELEPHANT;So;0;ON;;;;;N;;;;; +1FA6A;XIANGQI BLACK HORSE;So;0;ON;;;;;N;;;;; +1FA6B;XIANGQI BLACK CHARIOT;So;0;ON;;;;;N;;;;; +1FA6C;XIANGQI BLACK CANNON;So;0;ON;;;;;N;;;;; +1FA6D;XIANGQI BLACK SOLDIER;So;0;ON;;;;;N;;;;; 20000;<CJK Ideograph Extension B, First>;Lo;0;L;;;;;N;;;;; 2A6D6;<CJK Ideograph Extension B, Last>;Lo;0;L;;;;;N;;;;; 2A700;<CJK Ideograph Extension C, First>;Lo;0;L;;;;;N;;;;; diff --git a/lib/stdlib/uc_spec/emoji-data.txt b/lib/stdlib/uc_spec/emoji-data.txt new file mode 100644 index 0000000000..6e66455252 --- /dev/null +++ b/lib/stdlib/uc_spec/emoji-data.txt @@ -0,0 +1,714 @@ +# emoji-data.txt +# Date: 2018-02-07, 07:55:18 GMT +# © 2018 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Emoji Data for UTS #51 +# Version: 11.0 +# +# For documentation and usage, see http://www.unicode.org/reports/tr51 +# +# Format: +# <codepoint(s)> ; <property> # <comments> +# Note: there is no guarantee as to the structure of whitespace or comments +# +# Characters and sequences are listed in code point order. Users should be shown a more natural order. +# See the CLDR collation order for Emoji. + + +# ================================================ + +# All omitted code points have Emoji=No +# @missing: 0000..10FFFF ; Emoji ; No + +0023 ; Emoji # 1.1 [1] (#️) number sign +002A ; Emoji # 1.1 [1] (*️) asterisk +0030..0039 ; Emoji # 1.1 [10] (0️..9️) digit zero..digit nine +00A9 ; Emoji # 1.1 [1] (©️) copyright +00AE ; Emoji # 1.1 [1] (®️) registered +203C ; Emoji # 1.1 [1] (‼️) double exclamation mark +2049 ; Emoji # 3.0 [1] (⁉️) exclamation question mark +2122 ; Emoji # 1.1 [1] (™️) trade mark +2139 ; Emoji # 3.0 [1] (ℹ️) information +2194..2199 ; Emoji # 1.1 [6] (↔️..↙️) left-right arrow..down-left arrow +21A9..21AA ; Emoji # 1.1 [2] (↩️..↪️) right arrow curving left..left arrow curving right +231A..231B ; Emoji # 1.1 [2] (⌚..⌛) watch..hourglass done +2328 ; Emoji # 1.1 [1] (⌨️) keyboard +23CF ; Emoji # 4.0 [1] (⏏️) eject button +23E9..23F3 ; Emoji # 6.0 [11] (⏩..⏳) fast-forward button..hourglass not done +23F8..23FA ; Emoji # 7.0 [3] (⏸️..⏺️) pause button..record button +24C2 ; Emoji # 1.1 [1] (Ⓜ️) circled M +25AA..25AB ; Emoji # 1.1 [2] (▪️..▫️) black small square..white small square +25B6 ; Emoji # 1.1 [1] (▶️) play button +25C0 ; Emoji # 1.1 [1] (◀️) reverse button +25FB..25FE ; Emoji # 3.2 [4] (◻️..◾) white medium square..black medium-small square +2600..2604 ; Emoji # 1.1 [5] (☀️..☄️) sun..comet +260E ; Emoji # 1.1 [1] (☎️) telephone +2611 ; Emoji # 1.1 [1] (☑️) ballot box with check +2614..2615 ; Emoji # 4.0 [2] (☔..☕) umbrella with rain drops..hot beverage +2618 ; Emoji # 4.1 [1] (☘️) shamrock +261D ; Emoji # 1.1 [1] (☝️) index pointing up +2620 ; Emoji # 1.1 [1] (☠️) skull and crossbones +2622..2623 ; Emoji # 1.1 [2] (☢️..☣️) radioactive..biohazard +2626 ; Emoji # 1.1 [1] (☦️) orthodox cross +262A ; Emoji # 1.1 [1] (☪️) star and crescent +262E..262F ; Emoji # 1.1 [2] (☮️..☯️) peace symbol..yin yang +2638..263A ; Emoji # 1.1 [3] (☸️..☺️) wheel of dharma..smiling face +2640 ; Emoji # 1.1 [1] (♀️) female sign +2642 ; Emoji # 1.1 [1] (♂️) male sign +2648..2653 ; Emoji # 1.1 [12] (♈..♓) Aries..Pisces +265F..2660 ; Emoji # 1.1 [2] (♟️..♠️) chess pawn..spade suit +2663 ; Emoji # 1.1 [1] (♣️) club suit +2665..2666 ; Emoji # 1.1 [2] (♥️..♦️) heart suit..diamond suit +2668 ; Emoji # 1.1 [1] (♨️) hot springs +267B ; Emoji # 3.2 [1] (♻️) recycling symbol +267E..267F ; Emoji # 4.1 [2] (♾️..♿) infinity..wheelchair symbol +2692..2697 ; Emoji # 4.1 [6] (⚒️..⚗️) hammer and pick..alembic +2699 ; Emoji # 4.1 [1] (⚙️) gear +269B..269C ; Emoji # 4.1 [2] (⚛️..⚜️) atom symbol..fleur-de-lis +26A0..26A1 ; Emoji # 4.0 [2] (⚠️..⚡) warning..high voltage +26AA..26AB ; Emoji # 4.1 [2] (⚪..⚫) white circle..black circle +26B0..26B1 ; Emoji # 4.1 [2] (⚰️..⚱️) coffin..funeral urn +26BD..26BE ; Emoji # 5.2 [2] (⚽..⚾) soccer ball..baseball +26C4..26C5 ; Emoji # 5.2 [2] (⛄..⛅) snowman without snow..sun behind cloud +26C8 ; Emoji # 5.2 [1] (⛈️) cloud with lightning and rain +26CE ; Emoji # 6.0 [1] (⛎) Ophiuchus +26CF ; Emoji # 5.2 [1] (⛏️) pick +26D1 ; Emoji # 5.2 [1] (⛑️) rescue worker’s helmet +26D3..26D4 ; Emoji # 5.2 [2] (⛓️..⛔) chains..no entry +26E9..26EA ; Emoji # 5.2 [2] (⛩️..⛪) shinto shrine..church +26F0..26F5 ; Emoji # 5.2 [6] (⛰️..⛵) mountain..sailboat +26F7..26FA ; Emoji # 5.2 [4] (⛷️..⛺) skier..tent +26FD ; Emoji # 5.2 [1] (⛽) fuel pump +2702 ; Emoji # 1.1 [1] (✂️) scissors +2705 ; Emoji # 6.0 [1] (✅) white heavy check mark +2708..2709 ; Emoji # 1.1 [2] (✈️..✉️) airplane..envelope +270A..270B ; Emoji # 6.0 [2] (✊..✋) raised fist..raised hand +270C..270D ; Emoji # 1.1 [2] (✌️..✍️) victory hand..writing hand +270F ; Emoji # 1.1 [1] (✏️) pencil +2712 ; Emoji # 1.1 [1] (✒️) black nib +2714 ; Emoji # 1.1 [1] (✔️) heavy check mark +2716 ; Emoji # 1.1 [1] (✖️) heavy multiplication x +271D ; Emoji # 1.1 [1] (✝️) latin cross +2721 ; Emoji # 1.1 [1] (✡️) star of David +2728 ; Emoji # 6.0 [1] (✨) sparkles +2733..2734 ; Emoji # 1.1 [2] (✳️..✴️) eight-spoked asterisk..eight-pointed star +2744 ; Emoji # 1.1 [1] (❄️) snowflake +2747 ; Emoji # 1.1 [1] (❇️) sparkle +274C ; Emoji # 6.0 [1] (❌) cross mark +274E ; Emoji # 6.0 [1] (❎) cross mark button +2753..2755 ; Emoji # 6.0 [3] (❓..❕) question mark..white exclamation mark +2757 ; Emoji # 5.2 [1] (❗) exclamation mark +2763..2764 ; Emoji # 1.1 [2] (❣️..❤️) heavy heart exclamation..red heart +2795..2797 ; Emoji # 6.0 [3] (➕..➗) heavy plus sign..heavy division sign +27A1 ; Emoji # 1.1 [1] (➡️) right arrow +27B0 ; Emoji # 6.0 [1] (➰) curly loop +27BF ; Emoji # 6.0 [1] (➿) double curly loop +2934..2935 ; Emoji # 3.2 [2] (⤴️..⤵️) right arrow curving up..right arrow curving down +2B05..2B07 ; Emoji # 4.0 [3] (⬅️..⬇️) left arrow..down arrow +2B1B..2B1C ; Emoji # 5.1 [2] (⬛..⬜) black large square..white large square +2B50 ; Emoji # 5.1 [1] (⭐) star +2B55 ; Emoji # 5.2 [1] (⭕) heavy large circle +3030 ; Emoji # 1.1 [1] (〰️) wavy dash +303D ; Emoji # 3.2 [1] (〽️) part alternation mark +3297 ; Emoji # 1.1 [1] (㊗️) Japanese “congratulations” button +3299 ; Emoji # 1.1 [1] (㊙️) Japanese “secret” button +1F004 ; Emoji # 5.1 [1] (🀄) mahjong red dragon +1F0CF ; Emoji # 6.0 [1] (🃏) joker +1F170..1F171 ; Emoji # 6.0 [2] (🅰️..🅱️) A button (blood type)..B button (blood type) +1F17E ; Emoji # 6.0 [1] (🅾️) O button (blood type) +1F17F ; Emoji # 5.2 [1] (🅿️) P button +1F18E ; Emoji # 6.0 [1] (🆎) AB button (blood type) +1F191..1F19A ; Emoji # 6.0 [10] (🆑..🆚) CL button..VS button +1F1E6..1F1FF ; Emoji # 6.0 [26] (🇦..🇿) regional indicator symbol letter a..regional indicator symbol letter z +1F201..1F202 ; Emoji # 6.0 [2] (🈁..🈂️) Japanese “here” button..Japanese “service charge” button +1F21A ; Emoji # 5.2 [1] (🈚) Japanese “free of charge” button +1F22F ; Emoji # 5.2 [1] (🈯) Japanese “reserved” button +1F232..1F23A ; Emoji # 6.0 [9] (🈲..🈺) Japanese “prohibited” button..Japanese “open for business” button +1F250..1F251 ; Emoji # 6.0 [2] (🉐..🉑) Japanese “bargain” button..Japanese “acceptable” button +1F300..1F320 ; Emoji # 6.0 [33] (🌀..🌠) cyclone..shooting star +1F321 ; Emoji # 7.0 [1] (🌡️) thermometer +1F324..1F32C ; Emoji # 7.0 [9] (🌤️..🌬️) sun behind small cloud..wind face +1F32D..1F32F ; Emoji # 8.0 [3] (🌭..🌯) hot dog..burrito +1F330..1F335 ; Emoji # 6.0 [6] (🌰..🌵) chestnut..cactus +1F336 ; Emoji # 7.0 [1] (🌶️) hot pepper +1F337..1F37C ; Emoji # 6.0 [70] (🌷..🍼) tulip..baby bottle +1F37D ; Emoji # 7.0 [1] (🍽️) fork and knife with plate +1F37E..1F37F ; Emoji # 8.0 [2] (🍾..🍿) bottle with popping cork..popcorn +1F380..1F393 ; Emoji # 6.0 [20] (🎀..🎓) ribbon..graduation cap +1F396..1F397 ; Emoji # 7.0 [2] (🎖️..🎗️) military medal..reminder ribbon +1F399..1F39B ; Emoji # 7.0 [3] (🎙️..🎛️) studio microphone..control knobs +1F39E..1F39F ; Emoji # 7.0 [2] (🎞️..🎟️) film frames..admission tickets +1F3A0..1F3C4 ; Emoji # 6.0 [37] (🎠..🏄) carousel horse..person surfing +1F3C5 ; Emoji # 7.0 [1] (🏅) sports medal +1F3C6..1F3CA ; Emoji # 6.0 [5] (🏆..🏊) trophy..person swimming +1F3CB..1F3CE ; Emoji # 7.0 [4] (🏋️..🏎️) person lifting weights..racing car +1F3CF..1F3D3 ; Emoji # 8.0 [5] (🏏..🏓) cricket game..ping pong +1F3D4..1F3DF ; Emoji # 7.0 [12] (🏔️..🏟️) snow-capped mountain..stadium +1F3E0..1F3F0 ; Emoji # 6.0 [17] (🏠..🏰) house..castle +1F3F3..1F3F5 ; Emoji # 7.0 [3] (🏳️..🏵️) white flag..rosette +1F3F7 ; Emoji # 7.0 [1] (🏷️) label +1F3F8..1F3FF ; Emoji # 8.0 [8] (🏸..🏿) badminton..dark skin tone +1F400..1F43E ; Emoji # 6.0 [63] (🐀..🐾) rat..paw prints +1F43F ; Emoji # 7.0 [1] (🐿️) chipmunk +1F440 ; Emoji # 6.0 [1] (👀) eyes +1F441 ; Emoji # 7.0 [1] (👁️) eye +1F442..1F4F7 ; Emoji # 6.0[182] (👂..📷) ear..camera +1F4F8 ; Emoji # 7.0 [1] (📸) camera with flash +1F4F9..1F4FC ; Emoji # 6.0 [4] (📹..📼) video camera..videocassette +1F4FD ; Emoji # 7.0 [1] (📽️) film projector +1F4FF ; Emoji # 8.0 [1] (📿) prayer beads +1F500..1F53D ; Emoji # 6.0 [62] (🔀..🔽) shuffle tracks button..downwards button +1F549..1F54A ; Emoji # 7.0 [2] (🕉️..🕊️) om..dove +1F54B..1F54E ; Emoji # 8.0 [4] (🕋..🕎) kaaba..menorah +1F550..1F567 ; Emoji # 6.0 [24] (🕐..🕧) one o’clock..twelve-thirty +1F56F..1F570 ; Emoji # 7.0 [2] (🕯️..🕰️) candle..mantelpiece clock +1F573..1F579 ; Emoji # 7.0 [7] (🕳️..🕹️) hole..joystick +1F57A ; Emoji # 9.0 [1] (🕺) man dancing +1F587 ; Emoji # 7.0 [1] (🖇️) linked paperclips +1F58A..1F58D ; Emoji # 7.0 [4] (🖊️..🖍️) pen..crayon +1F590 ; Emoji # 7.0 [1] (🖐️) hand with fingers splayed +1F595..1F596 ; Emoji # 7.0 [2] (🖕..🖖) middle finger..vulcan salute +1F5A4 ; Emoji # 9.0 [1] (🖤) black heart +1F5A5 ; Emoji # 7.0 [1] (🖥️) desktop computer +1F5A8 ; Emoji # 7.0 [1] (🖨️) printer +1F5B1..1F5B2 ; Emoji # 7.0 [2] (🖱️..🖲️) computer mouse..trackball +1F5BC ; Emoji # 7.0 [1] (🖼️) framed picture +1F5C2..1F5C4 ; Emoji # 7.0 [3] (🗂️..🗄️) card index dividers..file cabinet +1F5D1..1F5D3 ; Emoji # 7.0 [3] (🗑️..🗓️) wastebasket..spiral calendar +1F5DC..1F5DE ; Emoji # 7.0 [3] (🗜️..🗞️) clamp..rolled-up newspaper +1F5E1 ; Emoji # 7.0 [1] (🗡️) dagger +1F5E3 ; Emoji # 7.0 [1] (🗣️) speaking head +1F5E8 ; Emoji # 7.0 [1] (🗨️) left speech bubble +1F5EF ; Emoji # 7.0 [1] (🗯️) right anger bubble +1F5F3 ; Emoji # 7.0 [1] (🗳️) ballot box with ballot +1F5FA ; Emoji # 7.0 [1] (🗺️) world map +1F5FB..1F5FF ; Emoji # 6.0 [5] (🗻..🗿) mount fuji..moai +1F600 ; Emoji # 6.1 [1] (😀) grinning face +1F601..1F610 ; Emoji # 6.0 [16] (😁..😐) beaming face with smiling eyes..neutral face +1F611 ; Emoji # 6.1 [1] (😑) expressionless face +1F612..1F614 ; Emoji # 6.0 [3] (😒..😔) unamused face..pensive face +1F615 ; Emoji # 6.1 [1] (😕) confused face +1F616 ; Emoji # 6.0 [1] (😖) confounded face +1F617 ; Emoji # 6.1 [1] (😗) kissing face +1F618 ; Emoji # 6.0 [1] (😘) face blowing a kiss +1F619 ; Emoji # 6.1 [1] (😙) kissing face with smiling eyes +1F61A ; Emoji # 6.0 [1] (😚) kissing face with closed eyes +1F61B ; Emoji # 6.1 [1] (😛) face with tongue +1F61C..1F61E ; Emoji # 6.0 [3] (😜..😞) winking face with tongue..disappointed face +1F61F ; Emoji # 6.1 [1] (😟) worried face +1F620..1F625 ; Emoji # 6.0 [6] (😠..😥) angry face..sad but relieved face +1F626..1F627 ; Emoji # 6.1 [2] (😦..😧) frowning face with open mouth..anguished face +1F628..1F62B ; Emoji # 6.0 [4] (😨..😫) fearful face..tired face +1F62C ; Emoji # 6.1 [1] (😬) grimacing face +1F62D ; Emoji # 6.0 [1] (😭) loudly crying face +1F62E..1F62F ; Emoji # 6.1 [2] (😮..😯) face with open mouth..hushed face +1F630..1F633 ; Emoji # 6.0 [4] (😰..😳) anxious face with sweat..flushed face +1F634 ; Emoji # 6.1 [1] (😴) sleeping face +1F635..1F640 ; Emoji # 6.0 [12] (😵..🙀) dizzy face..weary cat face +1F641..1F642 ; Emoji # 7.0 [2] (🙁..🙂) slightly frowning face..slightly smiling face +1F643..1F644 ; Emoji # 8.0 [2] (🙃..🙄) upside-down face..face with rolling eyes +1F645..1F64F ; Emoji # 6.0 [11] (🙅..🙏) person gesturing NO..folded hands +1F680..1F6C5 ; Emoji # 6.0 [70] (🚀..🛅) rocket..left luggage +1F6CB..1F6CF ; Emoji # 7.0 [5] (🛋️..🛏️) couch and lamp..bed +1F6D0 ; Emoji # 8.0 [1] (🛐) place of worship +1F6D1..1F6D2 ; Emoji # 9.0 [2] (🛑..🛒) stop sign..shopping cart +1F6E0..1F6E5 ; Emoji # 7.0 [6] (🛠️..🛥️) hammer and wrench..motor boat +1F6E9 ; Emoji # 7.0 [1] (🛩️) small airplane +1F6EB..1F6EC ; Emoji # 7.0 [2] (🛫..🛬) airplane departure..airplane arrival +1F6F0 ; Emoji # 7.0 [1] (🛰️) satellite +1F6F3 ; Emoji # 7.0 [1] (🛳️) passenger ship +1F6F4..1F6F6 ; Emoji # 9.0 [3] (🛴..🛶) kick scooter..canoe +1F6F7..1F6F8 ; Emoji # 10.0 [2] (🛷..🛸) sled..flying saucer +1F6F9 ; Emoji # 11.0 [1] (🛹) skateboard +1F910..1F918 ; Emoji # 8.0 [9] (🤐..🤘) zipper-mouth face..sign of the horns +1F919..1F91E ; Emoji # 9.0 [6] (🤙..🤞) call me hand..crossed fingers +1F91F ; Emoji # 10.0 [1] (🤟) love-you gesture +1F920..1F927 ; Emoji # 9.0 [8] (🤠..🤧) cowboy hat face..sneezing face +1F928..1F92F ; Emoji # 10.0 [8] (🤨..🤯) face with raised eyebrow..exploding head +1F930 ; Emoji # 9.0 [1] (🤰) pregnant woman +1F931..1F932 ; Emoji # 10.0 [2] (🤱..🤲) breast-feeding..palms up together +1F933..1F93A ; Emoji # 9.0 [8] (🤳..🤺) selfie..person fencing +1F93C..1F93E ; Emoji # 9.0 [3] (🤼..🤾) people wrestling..person playing handball +1F940..1F945 ; Emoji # 9.0 [6] (🥀..🥅) wilted flower..goal net +1F947..1F94B ; Emoji # 9.0 [5] (🥇..🥋) 1st place medal..martial arts uniform +1F94C ; Emoji # 10.0 [1] (🥌) curling stone +1F94D..1F94F ; Emoji # 11.0 [3] (🥍..🥏) lacrosse..flying disc +1F950..1F95E ; Emoji # 9.0 [15] (🥐..🥞) croissant..pancakes +1F95F..1F96B ; Emoji # 10.0 [13] (🥟..🥫) dumpling..canned food +1F96C..1F970 ; Emoji # 11.0 [5] (🥬..🥰) leafy green..smiling face with 3 hearts +1F973..1F976 ; Emoji # 11.0 [4] (🥳..🥶) partying face..cold face +1F97A ; Emoji # 11.0 [1] (🥺) pleading face +1F97C..1F97F ; Emoji # 11.0 [4] (🥼..🥿) lab coat..woman’s flat shoe +1F980..1F984 ; Emoji # 8.0 [5] (🦀..🦄) crab..unicorn face +1F985..1F991 ; Emoji # 9.0 [13] (🦅..🦑) eagle..squid +1F992..1F997 ; Emoji # 10.0 [6] (🦒..🦗) giraffe..cricket +1F998..1F9A2 ; Emoji # 11.0 [11] (🦘..🦢) kangaroo..swan +1F9B0..1F9B9 ; Emoji # 11.0 [10] (🦰..🦹) red-haired..supervillain +1F9C0 ; Emoji # 8.0 [1] (🧀) cheese wedge +1F9C1..1F9C2 ; Emoji # 11.0 [2] (🧁..🧂) cupcake..salt +1F9D0..1F9E6 ; Emoji # 10.0 [23] (🧐..🧦) face with monocle..socks +1F9E7..1F9FF ; Emoji # 11.0 [25] (🧧..🧿) red envelope..nazar amulet + +# Total elements: 1250 + +# ================================================ + +# All omitted code points have Emoji_Presentation=No +# @missing: 0000..10FFFF ; Emoji_Presentation ; No + +231A..231B ; Emoji_Presentation # 1.1 [2] (⌚..⌛) watch..hourglass done +23E9..23EC ; Emoji_Presentation # 6.0 [4] (⏩..⏬) fast-forward button..fast down button +23F0 ; Emoji_Presentation # 6.0 [1] (⏰) alarm clock +23F3 ; Emoji_Presentation # 6.0 [1] (⏳) hourglass not done +25FD..25FE ; Emoji_Presentation # 3.2 [2] (◽..◾) white medium-small square..black medium-small square +2614..2615 ; Emoji_Presentation # 4.0 [2] (☔..☕) umbrella with rain drops..hot beverage +2648..2653 ; Emoji_Presentation # 1.1 [12] (♈..♓) Aries..Pisces +267F ; Emoji_Presentation # 4.1 [1] (♿) wheelchair symbol +2693 ; Emoji_Presentation # 4.1 [1] (⚓) anchor +26A1 ; Emoji_Presentation # 4.0 [1] (⚡) high voltage +26AA..26AB ; Emoji_Presentation # 4.1 [2] (⚪..⚫) white circle..black circle +26BD..26BE ; Emoji_Presentation # 5.2 [2] (⚽..⚾) soccer ball..baseball +26C4..26C5 ; Emoji_Presentation # 5.2 [2] (⛄..⛅) snowman without snow..sun behind cloud +26CE ; Emoji_Presentation # 6.0 [1] (⛎) Ophiuchus +26D4 ; Emoji_Presentation # 5.2 [1] (⛔) no entry +26EA ; Emoji_Presentation # 5.2 [1] (⛪) church +26F2..26F3 ; Emoji_Presentation # 5.2 [2] (⛲..⛳) fountain..flag in hole +26F5 ; Emoji_Presentation # 5.2 [1] (⛵) sailboat +26FA ; Emoji_Presentation # 5.2 [1] (⛺) tent +26FD ; Emoji_Presentation # 5.2 [1] (⛽) fuel pump +2705 ; Emoji_Presentation # 6.0 [1] (✅) white heavy check mark +270A..270B ; Emoji_Presentation # 6.0 [2] (✊..✋) raised fist..raised hand +2728 ; Emoji_Presentation # 6.0 [1] (✨) sparkles +274C ; Emoji_Presentation # 6.0 [1] (❌) cross mark +274E ; Emoji_Presentation # 6.0 [1] (❎) cross mark button +2753..2755 ; Emoji_Presentation # 6.0 [3] (❓..❕) question mark..white exclamation mark +2757 ; Emoji_Presentation # 5.2 [1] (❗) exclamation mark +2795..2797 ; Emoji_Presentation # 6.0 [3] (➕..➗) heavy plus sign..heavy division sign +27B0 ; Emoji_Presentation # 6.0 [1] (➰) curly loop +27BF ; Emoji_Presentation # 6.0 [1] (➿) double curly loop +2B1B..2B1C ; Emoji_Presentation # 5.1 [2] (⬛..⬜) black large square..white large square +2B50 ; Emoji_Presentation # 5.1 [1] (⭐) star +2B55 ; Emoji_Presentation # 5.2 [1] (⭕) heavy large circle +1F004 ; Emoji_Presentation # 5.1 [1] (🀄) mahjong red dragon +1F0CF ; Emoji_Presentation # 6.0 [1] (🃏) joker +1F18E ; Emoji_Presentation # 6.0 [1] (🆎) AB button (blood type) +1F191..1F19A ; Emoji_Presentation # 6.0 [10] (🆑..🆚) CL button..VS button +1F1E6..1F1FF ; Emoji_Presentation # 6.0 [26] (🇦..🇿) regional indicator symbol letter a..regional indicator symbol letter z +1F201 ; Emoji_Presentation # 6.0 [1] (🈁) Japanese “here” button +1F21A ; Emoji_Presentation # 5.2 [1] (🈚) Japanese “free of charge” button +1F22F ; Emoji_Presentation # 5.2 [1] (🈯) Japanese “reserved” button +1F232..1F236 ; Emoji_Presentation # 6.0 [5] (🈲..🈶) Japanese “prohibited” button..Japanese “not free of charge” button +1F238..1F23A ; Emoji_Presentation # 6.0 [3] (🈸..🈺) Japanese “application” button..Japanese “open for business” button +1F250..1F251 ; Emoji_Presentation # 6.0 [2] (🉐..🉑) Japanese “bargain” button..Japanese “acceptable” button +1F300..1F320 ; Emoji_Presentation # 6.0 [33] (🌀..🌠) cyclone..shooting star +1F32D..1F32F ; Emoji_Presentation # 8.0 [3] (🌭..🌯) hot dog..burrito +1F330..1F335 ; Emoji_Presentation # 6.0 [6] (🌰..🌵) chestnut..cactus +1F337..1F37C ; Emoji_Presentation # 6.0 [70] (🌷..🍼) tulip..baby bottle +1F37E..1F37F ; Emoji_Presentation # 8.0 [2] (🍾..🍿) bottle with popping cork..popcorn +1F380..1F393 ; Emoji_Presentation # 6.0 [20] (🎀..🎓) ribbon..graduation cap +1F3A0..1F3C4 ; Emoji_Presentation # 6.0 [37] (🎠..🏄) carousel horse..person surfing +1F3C5 ; Emoji_Presentation # 7.0 [1] (🏅) sports medal +1F3C6..1F3CA ; Emoji_Presentation # 6.0 [5] (🏆..🏊) trophy..person swimming +1F3CF..1F3D3 ; Emoji_Presentation # 8.0 [5] (🏏..🏓) cricket game..ping pong +1F3E0..1F3F0 ; Emoji_Presentation # 6.0 [17] (🏠..🏰) house..castle +1F3F4 ; Emoji_Presentation # 7.0 [1] (🏴) black flag +1F3F8..1F3FF ; Emoji_Presentation # 8.0 [8] (🏸..🏿) badminton..dark skin tone +1F400..1F43E ; Emoji_Presentation # 6.0 [63] (🐀..🐾) rat..paw prints +1F440 ; Emoji_Presentation # 6.0 [1] (👀) eyes +1F442..1F4F7 ; Emoji_Presentation # 6.0[182] (👂..📷) ear..camera +1F4F8 ; Emoji_Presentation # 7.0 [1] (📸) camera with flash +1F4F9..1F4FC ; Emoji_Presentation # 6.0 [4] (📹..📼) video camera..videocassette +1F4FF ; Emoji_Presentation # 8.0 [1] (📿) prayer beads +1F500..1F53D ; Emoji_Presentation # 6.0 [62] (🔀..🔽) shuffle tracks button..downwards button +1F54B..1F54E ; Emoji_Presentation # 8.0 [4] (🕋..🕎) kaaba..menorah +1F550..1F567 ; Emoji_Presentation # 6.0 [24] (🕐..🕧) one o’clock..twelve-thirty +1F57A ; Emoji_Presentation # 9.0 [1] (🕺) man dancing +1F595..1F596 ; Emoji_Presentation # 7.0 [2] (🖕..🖖) middle finger..vulcan salute +1F5A4 ; Emoji_Presentation # 9.0 [1] (🖤) black heart +1F5FB..1F5FF ; Emoji_Presentation # 6.0 [5] (🗻..🗿) mount fuji..moai +1F600 ; Emoji_Presentation # 6.1 [1] (😀) grinning face +1F601..1F610 ; Emoji_Presentation # 6.0 [16] (😁..😐) beaming face with smiling eyes..neutral face +1F611 ; Emoji_Presentation # 6.1 [1] (😑) expressionless face +1F612..1F614 ; Emoji_Presentation # 6.0 [3] (😒..😔) unamused face..pensive face +1F615 ; Emoji_Presentation # 6.1 [1] (😕) confused face +1F616 ; Emoji_Presentation # 6.0 [1] (😖) confounded face +1F617 ; Emoji_Presentation # 6.1 [1] (😗) kissing face +1F618 ; Emoji_Presentation # 6.0 [1] (😘) face blowing a kiss +1F619 ; Emoji_Presentation # 6.1 [1] (😙) kissing face with smiling eyes +1F61A ; Emoji_Presentation # 6.0 [1] (😚) kissing face with closed eyes +1F61B ; Emoji_Presentation # 6.1 [1] (😛) face with tongue +1F61C..1F61E ; Emoji_Presentation # 6.0 [3] (😜..😞) winking face with tongue..disappointed face +1F61F ; Emoji_Presentation # 6.1 [1] (😟) worried face +1F620..1F625 ; Emoji_Presentation # 6.0 [6] (😠..😥) angry face..sad but relieved face +1F626..1F627 ; Emoji_Presentation # 6.1 [2] (😦..😧) frowning face with open mouth..anguished face +1F628..1F62B ; Emoji_Presentation # 6.0 [4] (😨..😫) fearful face..tired face +1F62C ; Emoji_Presentation # 6.1 [1] (😬) grimacing face +1F62D ; Emoji_Presentation # 6.0 [1] (😭) loudly crying face +1F62E..1F62F ; Emoji_Presentation # 6.1 [2] (😮..😯) face with open mouth..hushed face +1F630..1F633 ; Emoji_Presentation # 6.0 [4] (😰..😳) anxious face with sweat..flushed face +1F634 ; Emoji_Presentation # 6.1 [1] (😴) sleeping face +1F635..1F640 ; Emoji_Presentation # 6.0 [12] (😵..🙀) dizzy face..weary cat face +1F641..1F642 ; Emoji_Presentation # 7.0 [2] (🙁..🙂) slightly frowning face..slightly smiling face +1F643..1F644 ; Emoji_Presentation # 8.0 [2] (🙃..🙄) upside-down face..face with rolling eyes +1F645..1F64F ; Emoji_Presentation # 6.0 [11] (🙅..🙏) person gesturing NO..folded hands +1F680..1F6C5 ; Emoji_Presentation # 6.0 [70] (🚀..🛅) rocket..left luggage +1F6CC ; Emoji_Presentation # 7.0 [1] (🛌) person in bed +1F6D0 ; Emoji_Presentation # 8.0 [1] (🛐) place of worship +1F6D1..1F6D2 ; Emoji_Presentation # 9.0 [2] (🛑..🛒) stop sign..shopping cart +1F6EB..1F6EC ; Emoji_Presentation # 7.0 [2] (🛫..🛬) airplane departure..airplane arrival +1F6F4..1F6F6 ; Emoji_Presentation # 9.0 [3] (🛴..🛶) kick scooter..canoe +1F6F7..1F6F8 ; Emoji_Presentation # 10.0 [2] (🛷..🛸) sled..flying saucer +1F6F9 ; Emoji_Presentation # 11.0 [1] (🛹) skateboard +1F910..1F918 ; Emoji_Presentation # 8.0 [9] (🤐..🤘) zipper-mouth face..sign of the horns +1F919..1F91E ; Emoji_Presentation # 9.0 [6] (🤙..🤞) call me hand..crossed fingers +1F91F ; Emoji_Presentation # 10.0 [1] (🤟) love-you gesture +1F920..1F927 ; Emoji_Presentation # 9.0 [8] (🤠..🤧) cowboy hat face..sneezing face +1F928..1F92F ; Emoji_Presentation # 10.0 [8] (🤨..🤯) face with raised eyebrow..exploding head +1F930 ; Emoji_Presentation # 9.0 [1] (🤰) pregnant woman +1F931..1F932 ; Emoji_Presentation # 10.0 [2] (🤱..🤲) breast-feeding..palms up together +1F933..1F93A ; Emoji_Presentation # 9.0 [8] (🤳..🤺) selfie..person fencing +1F93C..1F93E ; Emoji_Presentation # 9.0 [3] (🤼..🤾) people wrestling..person playing handball +1F940..1F945 ; Emoji_Presentation # 9.0 [6] (🥀..🥅) wilted flower..goal net +1F947..1F94B ; Emoji_Presentation # 9.0 [5] (🥇..🥋) 1st place medal..martial arts uniform +1F94C ; Emoji_Presentation # 10.0 [1] (🥌) curling stone +1F94D..1F94F ; Emoji_Presentation # 11.0 [3] (🥍..🥏) lacrosse..flying disc +1F950..1F95E ; Emoji_Presentation # 9.0 [15] (🥐..🥞) croissant..pancakes +1F95F..1F96B ; Emoji_Presentation # 10.0 [13] (🥟..🥫) dumpling..canned food +1F96C..1F970 ; Emoji_Presentation # 11.0 [5] (🥬..🥰) leafy green..smiling face with 3 hearts +1F973..1F976 ; Emoji_Presentation # 11.0 [4] (🥳..🥶) partying face..cold face +1F97A ; Emoji_Presentation # 11.0 [1] (🥺) pleading face +1F97C..1F97F ; Emoji_Presentation # 11.0 [4] (🥼..🥿) lab coat..woman’s flat shoe +1F980..1F984 ; Emoji_Presentation # 8.0 [5] (🦀..🦄) crab..unicorn face +1F985..1F991 ; Emoji_Presentation # 9.0 [13] (🦅..🦑) eagle..squid +1F992..1F997 ; Emoji_Presentation # 10.0 [6] (🦒..🦗) giraffe..cricket +1F998..1F9A2 ; Emoji_Presentation # 11.0 [11] (🦘..🦢) kangaroo..swan +1F9B0..1F9B9 ; Emoji_Presentation # 11.0 [10] (🦰..🦹) red-haired..supervillain +1F9C0 ; Emoji_Presentation # 8.0 [1] (🧀) cheese wedge +1F9C1..1F9C2 ; Emoji_Presentation # 11.0 [2] (🧁..🧂) cupcake..salt +1F9D0..1F9E6 ; Emoji_Presentation # 10.0 [23] (🧐..🧦) face with monocle..socks +1F9E7..1F9FF ; Emoji_Presentation # 11.0 [25] (🧧..🧿) red envelope..nazar amulet + +# Total elements: 1032 + +# ================================================ + +# All omitted code points have Emoji_Modifier=No +# @missing: 0000..10FFFF ; Emoji_Modifier ; No + +1F3FB..1F3FF ; Emoji_Modifier # 8.0 [5] (🏻..🏿) light skin tone..dark skin tone + +# Total elements: 5 + +# ================================================ + +# All omitted code points have Emoji_Modifier_Base=No +# @missing: 0000..10FFFF ; Emoji_Modifier_Base ; No + +261D ; Emoji_Modifier_Base # 1.1 [1] (☝️) index pointing up +26F9 ; Emoji_Modifier_Base # 5.2 [1] (⛹️) person bouncing ball +270A..270B ; Emoji_Modifier_Base # 6.0 [2] (✊..✋) raised fist..raised hand +270C..270D ; Emoji_Modifier_Base # 1.1 [2] (✌️..✍️) victory hand..writing hand +1F385 ; Emoji_Modifier_Base # 6.0 [1] (🎅) Santa Claus +1F3C2..1F3C4 ; Emoji_Modifier_Base # 6.0 [3] (🏂..🏄) snowboarder..person surfing +1F3C7 ; Emoji_Modifier_Base # 6.0 [1] (🏇) horse racing +1F3CA ; Emoji_Modifier_Base # 6.0 [1] (🏊) person swimming +1F3CB..1F3CC ; Emoji_Modifier_Base # 7.0 [2] (🏋️..🏌️) person lifting weights..person golfing +1F442..1F443 ; Emoji_Modifier_Base # 6.0 [2] (👂..👃) ear..nose +1F446..1F450 ; Emoji_Modifier_Base # 6.0 [11] (👆..👐) backhand index pointing up..open hands +1F466..1F469 ; Emoji_Modifier_Base # 6.0 [4] (👦..👩) boy..woman +1F46E ; Emoji_Modifier_Base # 6.0 [1] (👮) police officer +1F470..1F478 ; Emoji_Modifier_Base # 6.0 [9] (👰..👸) bride with veil..princess +1F47C ; Emoji_Modifier_Base # 6.0 [1] (👼) baby angel +1F481..1F483 ; Emoji_Modifier_Base # 6.0 [3] (💁..💃) person tipping hand..woman dancing +1F485..1F487 ; Emoji_Modifier_Base # 6.0 [3] (💅..💇) nail polish..person getting haircut +1F4AA ; Emoji_Modifier_Base # 6.0 [1] (💪) flexed biceps +1F574..1F575 ; Emoji_Modifier_Base # 7.0 [2] (🕴️..🕵️) man in suit levitating..detective +1F57A ; Emoji_Modifier_Base # 9.0 [1] (🕺) man dancing +1F590 ; Emoji_Modifier_Base # 7.0 [1] (🖐️) hand with fingers splayed +1F595..1F596 ; Emoji_Modifier_Base # 7.0 [2] (🖕..🖖) middle finger..vulcan salute +1F645..1F647 ; Emoji_Modifier_Base # 6.0 [3] (🙅..🙇) person gesturing NO..person bowing +1F64B..1F64F ; Emoji_Modifier_Base # 6.0 [5] (🙋..🙏) person raising hand..folded hands +1F6A3 ; Emoji_Modifier_Base # 6.0 [1] (🚣) person rowing boat +1F6B4..1F6B6 ; Emoji_Modifier_Base # 6.0 [3] (🚴..🚶) person biking..person walking +1F6C0 ; Emoji_Modifier_Base # 6.0 [1] (🛀) person taking bath +1F6CC ; Emoji_Modifier_Base # 7.0 [1] (🛌) person in bed +1F918 ; Emoji_Modifier_Base # 8.0 [1] (🤘) sign of the horns +1F919..1F91C ; Emoji_Modifier_Base # 9.0 [4] (🤙..🤜) call me hand..right-facing fist +1F91E ; Emoji_Modifier_Base # 9.0 [1] (🤞) crossed fingers +1F91F ; Emoji_Modifier_Base # 10.0 [1] (🤟) love-you gesture +1F926 ; Emoji_Modifier_Base # 9.0 [1] (🤦) person facepalming +1F930 ; Emoji_Modifier_Base # 9.0 [1] (🤰) pregnant woman +1F931..1F932 ; Emoji_Modifier_Base # 10.0 [2] (🤱..🤲) breast-feeding..palms up together +1F933..1F939 ; Emoji_Modifier_Base # 9.0 [7] (🤳..🤹) selfie..person juggling +1F93D..1F93E ; Emoji_Modifier_Base # 9.0 [2] (🤽..🤾) person playing water polo..person playing handball +1F9B5..1F9B6 ; Emoji_Modifier_Base # 11.0 [2] (🦵..🦶) leg..foot +1F9B8..1F9B9 ; Emoji_Modifier_Base # 11.0 [2] (🦸..🦹) superhero..supervillain +1F9D1..1F9DD ; Emoji_Modifier_Base # 10.0 [13] (🧑..🧝) adult..elf + +# Total elements: 106 + +# ================================================ + +# All omitted code points have Emoji_Component=No +# @missing: 0000..10FFFF ; Emoji_Component ; No + +0023 ; Emoji_Component # 1.1 [1] (#️) number sign +002A ; Emoji_Component # 1.1 [1] (*️) asterisk +0030..0039 ; Emoji_Component # 1.1 [10] (0️..9️) digit zero..digit nine +200D ; Emoji_Component # 1.1 [1] () zero width joiner +20E3 ; Emoji_Component # 3.0 [1] (⃣) combining enclosing keycap +FE0F ; Emoji_Component # 3.2 [1] () VARIATION SELECTOR-16 +1F1E6..1F1FF ; Emoji_Component # 6.0 [26] (🇦..🇿) regional indicator symbol letter a..regional indicator symbol letter z +1F3FB..1F3FF ; Emoji_Component # 8.0 [5] (🏻..🏿) light skin tone..dark skin tone +1F9B0..1F9B3 ; Emoji_Component # 11.0 [4] (🦰..🦳) red-haired..white-haired +E0020..E007F ; Emoji_Component # 3.1 [96] (..) tag space..cancel tag + +# Total elements: 146 + +# ================================================ + +# All omitted code points have Extended_Pictographic=No +# @missing: 0000..10FFFF ; Extended_Pictographic ; No + +00A9 ; Extended_Pictographic# 1.1 [1] (©️) copyright +00AE ; Extended_Pictographic# 1.1 [1] (®️) registered +203C ; Extended_Pictographic# 1.1 [1] (‼️) double exclamation mark +2049 ; Extended_Pictographic# 3.0 [1] (⁉️) exclamation question mark +2122 ; Extended_Pictographic# 1.1 [1] (™️) trade mark +2139 ; Extended_Pictographic# 3.0 [1] (ℹ️) information +2194..2199 ; Extended_Pictographic# 1.1 [6] (↔️..↙️) left-right arrow..down-left arrow +21A9..21AA ; Extended_Pictographic# 1.1 [2] (↩️..↪️) right arrow curving left..left arrow curving right +231A..231B ; Extended_Pictographic# 1.1 [2] (⌚..⌛) watch..hourglass done +2328 ; Extended_Pictographic# 1.1 [1] (⌨️) keyboard +2388 ; Extended_Pictographic# 3.0 [1] (⎈️) HELM SYMBOL +23CF ; Extended_Pictographic# 4.0 [1] (⏏️) eject button +23E9..23F3 ; Extended_Pictographic# 6.0 [11] (⏩..⏳) fast-forward button..hourglass not done +23F8..23FA ; Extended_Pictographic# 7.0 [3] (⏸️..⏺️) pause button..record button +24C2 ; Extended_Pictographic# 1.1 [1] (Ⓜ️) circled M +25AA..25AB ; Extended_Pictographic# 1.1 [2] (▪️..▫️) black small square..white small square +25B6 ; Extended_Pictographic# 1.1 [1] (▶️) play button +25C0 ; Extended_Pictographic# 1.1 [1] (◀️) reverse button +25FB..25FE ; Extended_Pictographic# 3.2 [4] (◻️..◾) white medium square..black medium-small square +2600..2605 ; Extended_Pictographic# 1.1 [6] (☀️..★️) sun..BLACK STAR +2607..2612 ; Extended_Pictographic# 1.1 [12] (☇️..☒️) LIGHTNING..BALLOT BOX WITH X +2614..2615 ; Extended_Pictographic# 4.0 [2] (☔..☕) umbrella with rain drops..hot beverage +2616..2617 ; Extended_Pictographic# 3.2 [2] (☖️..☗️) WHITE SHOGI PIECE..BLACK SHOGI PIECE +2618 ; Extended_Pictographic# 4.1 [1] (☘️) shamrock +2619 ; Extended_Pictographic# 3.0 [1] (☙️) REVERSED ROTATED FLORAL HEART BULLET +261A..266F ; Extended_Pictographic# 1.1 [86] (☚️..♯️) BLACK LEFT POINTING INDEX..MUSIC SHARP SIGN +2670..2671 ; Extended_Pictographic# 3.0 [2] (♰️..♱️) WEST SYRIAC CROSS..EAST SYRIAC CROSS +2672..267D ; Extended_Pictographic# 3.2 [12] (♲️..♽️) UNIVERSAL RECYCLING SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL +267E..267F ; Extended_Pictographic# 4.1 [2] (♾️..♿) infinity..wheelchair symbol +2680..2685 ; Extended_Pictographic# 3.2 [6] (⚀️..⚅️) DIE FACE-1..DIE FACE-6 +2690..2691 ; Extended_Pictographic# 4.0 [2] (⚐️..⚑️) WHITE FLAG..BLACK FLAG +2692..269C ; Extended_Pictographic# 4.1 [11] (⚒️..⚜️) hammer and pick..fleur-de-lis +269D ; Extended_Pictographic# 5.1 [1] (⚝️) OUTLINED WHITE STAR +269E..269F ; Extended_Pictographic# 5.2 [2] (⚞️..⚟️) THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT +26A0..26A1 ; Extended_Pictographic# 4.0 [2] (⚠️..⚡) warning..high voltage +26A2..26B1 ; Extended_Pictographic# 4.1 [16] (⚢️..⚱️) DOUBLED FEMALE SIGN..funeral urn +26B2 ; Extended_Pictographic# 5.0 [1] (⚲️) NEUTER +26B3..26BC ; Extended_Pictographic# 5.1 [10] (⚳️..⚼️) CERES..SESQUIQUADRATE +26BD..26BF ; Extended_Pictographic# 5.2 [3] (⚽..⚿️) soccer ball..SQUARED KEY +26C0..26C3 ; Extended_Pictographic# 5.1 [4] (⛀️..⛃️) WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING +26C4..26CD ; Extended_Pictographic# 5.2 [10] (⛄..⛍️) snowman without snow..DISABLED CAR +26CE ; Extended_Pictographic# 6.0 [1] (⛎) Ophiuchus +26CF..26E1 ; Extended_Pictographic# 5.2 [19] (⛏️..⛡️) pick..RESTRICTED LEFT ENTRY-2 +26E2 ; Extended_Pictographic# 6.0 [1] (⛢️) ASTRONOMICAL SYMBOL FOR URANUS +26E3 ; Extended_Pictographic# 5.2 [1] (⛣️) HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE +26E4..26E7 ; Extended_Pictographic# 6.0 [4] (⛤️..⛧️) PENTAGRAM..INVERTED PENTAGRAM +26E8..26FF ; Extended_Pictographic# 5.2 [24] (⛨️..⛿️) BLACK CROSS ON SHIELD..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE +2700 ; Extended_Pictographic# 7.0 [1] (✀️) BLACK SAFETY SCISSORS +2701..2704 ; Extended_Pictographic# 1.1 [4] (✁️..✄️) UPPER BLADE SCISSORS..WHITE SCISSORS +2705 ; Extended_Pictographic# 6.0 [1] (✅) white heavy check mark +2708..2709 ; Extended_Pictographic# 1.1 [2] (✈️..✉️) airplane..envelope +270A..270B ; Extended_Pictographic# 6.0 [2] (✊..✋) raised fist..raised hand +270C..2712 ; Extended_Pictographic# 1.1 [7] (✌️..✒️) victory hand..black nib +2714 ; Extended_Pictographic# 1.1 [1] (✔️) heavy check mark +2716 ; Extended_Pictographic# 1.1 [1] (✖️) heavy multiplication x +271D ; Extended_Pictographic# 1.1 [1] (✝️) latin cross +2721 ; Extended_Pictographic# 1.1 [1] (✡️) star of David +2728 ; Extended_Pictographic# 6.0 [1] (✨) sparkles +2733..2734 ; Extended_Pictographic# 1.1 [2] (✳️..✴️) eight-spoked asterisk..eight-pointed star +2744 ; Extended_Pictographic# 1.1 [1] (❄️) snowflake +2747 ; Extended_Pictographic# 1.1 [1] (❇️) sparkle +274C ; Extended_Pictographic# 6.0 [1] (❌) cross mark +274E ; Extended_Pictographic# 6.0 [1] (❎) cross mark button +2753..2755 ; Extended_Pictographic# 6.0 [3] (❓..❕) question mark..white exclamation mark +2757 ; Extended_Pictographic# 5.2 [1] (❗) exclamation mark +2763..2767 ; Extended_Pictographic# 1.1 [5] (❣️..❧️) heavy heart exclamation..ROTATED FLORAL HEART BULLET +2795..2797 ; Extended_Pictographic# 6.0 [3] (➕..➗) heavy plus sign..heavy division sign +27A1 ; Extended_Pictographic# 1.1 [1] (➡️) right arrow +27B0 ; Extended_Pictographic# 6.0 [1] (➰) curly loop +27BF ; Extended_Pictographic# 6.0 [1] (➿) double curly loop +2934..2935 ; Extended_Pictographic# 3.2 [2] (⤴️..⤵️) right arrow curving up..right arrow curving down +2B05..2B07 ; Extended_Pictographic# 4.0 [3] (⬅️..⬇️) left arrow..down arrow +2B1B..2B1C ; Extended_Pictographic# 5.1 [2] (⬛..⬜) black large square..white large square +2B50 ; Extended_Pictographic# 5.1 [1] (⭐) star +2B55 ; Extended_Pictographic# 5.2 [1] (⭕) heavy large circle +3030 ; Extended_Pictographic# 1.1 [1] (〰️) wavy dash +303D ; Extended_Pictographic# 3.2 [1] (〽️) part alternation mark +3297 ; Extended_Pictographic# 1.1 [1] (㊗️) Japanese “congratulations” button +3299 ; Extended_Pictographic# 1.1 [1] (㊙️) Japanese “secret” button +1F000..1F02B ; Extended_Pictographic# 5.1 [44] (🀀️..🀫️) MAHJONG TILE EAST WIND..MAHJONG TILE BACK +1F02C..1F02F ; Extended_Pictographic# NA [4] (️..️) <reserved-1F02C>..<reserved-1F02F> +1F030..1F093 ; Extended_Pictographic# 5.1[100] (🀰️..🂓️) DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 +1F094..1F09F ; Extended_Pictographic# NA [12] (️..️) <reserved-1F094>..<reserved-1F09F> +1F0A0..1F0AE ; Extended_Pictographic# 6.0 [15] (🂠️..🂮️) PLAYING CARD BACK..PLAYING CARD KING OF SPADES +1F0AF..1F0B0 ; Extended_Pictographic# NA [2] (️..️) <reserved-1F0AF>..<reserved-1F0B0> +1F0B1..1F0BE ; Extended_Pictographic# 6.0 [14] (🂱️..🂾️) PLAYING CARD ACE OF HEARTS..PLAYING CARD KING OF HEARTS +1F0BF ; Extended_Pictographic# 7.0 [1] (🂿️) PLAYING CARD RED JOKER +1F0C0 ; Extended_Pictographic# NA [1] (️) <reserved-1F0C0> +1F0C1..1F0CF ; Extended_Pictographic# 6.0 [15] (🃁️..🃏) PLAYING CARD ACE OF DIAMONDS..joker +1F0D0 ; Extended_Pictographic# NA [1] (️) <reserved-1F0D0> +1F0D1..1F0DF ; Extended_Pictographic# 6.0 [15] (🃑️..🃟️) PLAYING CARD ACE OF CLUBS..PLAYING CARD WHITE JOKER +1F0E0..1F0F5 ; Extended_Pictographic# 7.0 [22] (🃠️..🃵️) PLAYING CARD FOOL..PLAYING CARD TRUMP-21 +1F0F6..1F0FF ; Extended_Pictographic# NA [10] (️..️) <reserved-1F0F6>..<reserved-1F0FF> +1F10D..1F10F ; Extended_Pictographic# NA [3] (🄍️..🄏️) <reserved-1F10D>..<reserved-1F10F> +1F12F ; Extended_Pictographic# 11.0 [1] (🄯️) COPYLEFT SYMBOL +1F16C..1F16F ; Extended_Pictographic# NA [4] (🅬️..🅯️) <reserved-1F16C>..<reserved-1F16F> +1F170..1F171 ; Extended_Pictographic# 6.0 [2] (🅰️..🅱️) A button (blood type)..B button (blood type) +1F17E ; Extended_Pictographic# 6.0 [1] (🅾️) O button (blood type) +1F17F ; Extended_Pictographic# 5.2 [1] (🅿️) P button +1F18E ; Extended_Pictographic# 6.0 [1] (🆎) AB button (blood type) +1F191..1F19A ; Extended_Pictographic# 6.0 [10] (🆑..🆚) CL button..VS button +1F1AD..1F1E5 ; Extended_Pictographic# NA [57] (🆭️..️) <reserved-1F1AD>..<reserved-1F1E5> +1F201..1F202 ; Extended_Pictographic# 6.0 [2] (🈁..🈂️) Japanese “here” button..Japanese “service charge” button +1F203..1F20F ; Extended_Pictographic# NA [13] (️..️) <reserved-1F203>..<reserved-1F20F> +1F21A ; Extended_Pictographic# 5.2 [1] (🈚) Japanese “free of charge” button +1F22F ; Extended_Pictographic# 5.2 [1] (🈯) Japanese “reserved” button +1F232..1F23A ; Extended_Pictographic# 6.0 [9] (🈲..🈺) Japanese “prohibited” button..Japanese “open for business” button +1F23C..1F23F ; Extended_Pictographic# NA [4] (️..️) <reserved-1F23C>..<reserved-1F23F> +1F249..1F24F ; Extended_Pictographic# NA [7] (️..️) <reserved-1F249>..<reserved-1F24F> +1F250..1F251 ; Extended_Pictographic# 6.0 [2] (🉐..🉑) Japanese “bargain” button..Japanese “acceptable” button +1F252..1F25F ; Extended_Pictographic# NA [14] (️..️) <reserved-1F252>..<reserved-1F25F> +1F260..1F265 ; Extended_Pictographic# 10.0 [6] (🉠️..🉥️) ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI +1F266..1F2FF ; Extended_Pictographic# NA[154] (️..️) <reserved-1F266>..<reserved-1F2FF> +1F300..1F320 ; Extended_Pictographic# 6.0 [33] (🌀..🌠) cyclone..shooting star +1F321..1F32C ; Extended_Pictographic# 7.0 [12] (🌡️..🌬️) thermometer..wind face +1F32D..1F32F ; Extended_Pictographic# 8.0 [3] (🌭..🌯) hot dog..burrito +1F330..1F335 ; Extended_Pictographic# 6.0 [6] (🌰..🌵) chestnut..cactus +1F336 ; Extended_Pictographic# 7.0 [1] (🌶️) hot pepper +1F337..1F37C ; Extended_Pictographic# 6.0 [70] (🌷..🍼) tulip..baby bottle +1F37D ; Extended_Pictographic# 7.0 [1] (🍽️) fork and knife with plate +1F37E..1F37F ; Extended_Pictographic# 8.0 [2] (🍾..🍿) bottle with popping cork..popcorn +1F380..1F393 ; Extended_Pictographic# 6.0 [20] (🎀..🎓) ribbon..graduation cap +1F394..1F39F ; Extended_Pictographic# 7.0 [12] (🎔️..🎟️) HEART WITH TIP ON THE LEFT..admission tickets +1F3A0..1F3C4 ; Extended_Pictographic# 6.0 [37] (🎠..🏄) carousel horse..person surfing +1F3C5 ; Extended_Pictographic# 7.0 [1] (🏅) sports medal +1F3C6..1F3CA ; Extended_Pictographic# 6.0 [5] (🏆..🏊) trophy..person swimming +1F3CB..1F3CE ; Extended_Pictographic# 7.0 [4] (🏋️..🏎️) person lifting weights..racing car +1F3CF..1F3D3 ; Extended_Pictographic# 8.0 [5] (🏏..🏓) cricket game..ping pong +1F3D4..1F3DF ; Extended_Pictographic# 7.0 [12] (🏔️..🏟️) snow-capped mountain..stadium +1F3E0..1F3F0 ; Extended_Pictographic# 6.0 [17] (🏠..🏰) house..castle +1F3F1..1F3F7 ; Extended_Pictographic# 7.0 [7] (🏱️..🏷️) WHITE PENNANT..label +1F3F8..1F3FA ; Extended_Pictographic# 8.0 [3] (🏸..🏺) badminton..amphora +1F400..1F43E ; Extended_Pictographic# 6.0 [63] (🐀..🐾) rat..paw prints +1F43F ; Extended_Pictographic# 7.0 [1] (🐿️) chipmunk +1F440 ; Extended_Pictographic# 6.0 [1] (👀) eyes +1F441 ; Extended_Pictographic# 7.0 [1] (👁️) eye +1F442..1F4F7 ; Extended_Pictographic# 6.0[182] (👂..📷) ear..camera +1F4F8 ; Extended_Pictographic# 7.0 [1] (📸) camera with flash +1F4F9..1F4FC ; Extended_Pictographic# 6.0 [4] (📹..📼) video camera..videocassette +1F4FD..1F4FE ; Extended_Pictographic# 7.0 [2] (📽️..📾️) film projector..PORTABLE STEREO +1F4FF ; Extended_Pictographic# 8.0 [1] (📿) prayer beads +1F500..1F53D ; Extended_Pictographic# 6.0 [62] (🔀..🔽) shuffle tracks button..downwards button +1F546..1F54A ; Extended_Pictographic# 7.0 [5] (🕆️..🕊️) WHITE LATIN CROSS..dove +1F54B..1F54F ; Extended_Pictographic# 8.0 [5] (🕋..🕏️) kaaba..BOWL OF HYGIEIA +1F550..1F567 ; Extended_Pictographic# 6.0 [24] (🕐..🕧) one o’clock..twelve-thirty +1F568..1F579 ; Extended_Pictographic# 7.0 [18] (🕨️..🕹️) RIGHT SPEAKER..joystick +1F57A ; Extended_Pictographic# 9.0 [1] (🕺) man dancing +1F57B..1F5A3 ; Extended_Pictographic# 7.0 [41] (🕻️..🖣️) LEFT HAND TELEPHONE RECEIVER..BLACK DOWN POINTING BACKHAND INDEX +1F5A4 ; Extended_Pictographic# 9.0 [1] (🖤) black heart +1F5A5..1F5FA ; Extended_Pictographic# 7.0 [86] (🖥️..🗺️) desktop computer..world map +1F5FB..1F5FF ; Extended_Pictographic# 6.0 [5] (🗻..🗿) mount fuji..moai +1F600 ; Extended_Pictographic# 6.1 [1] (😀) grinning face +1F601..1F610 ; Extended_Pictographic# 6.0 [16] (😁..😐) beaming face with smiling eyes..neutral face +1F611 ; Extended_Pictographic# 6.1 [1] (😑) expressionless face +1F612..1F614 ; Extended_Pictographic# 6.0 [3] (😒..😔) unamused face..pensive face +1F615 ; Extended_Pictographic# 6.1 [1] (😕) confused face +1F616 ; Extended_Pictographic# 6.0 [1] (😖) confounded face +1F617 ; Extended_Pictographic# 6.1 [1] (😗) kissing face +1F618 ; Extended_Pictographic# 6.0 [1] (😘) face blowing a kiss +1F619 ; Extended_Pictographic# 6.1 [1] (😙) kissing face with smiling eyes +1F61A ; Extended_Pictographic# 6.0 [1] (😚) kissing face with closed eyes +1F61B ; Extended_Pictographic# 6.1 [1] (😛) face with tongue +1F61C..1F61E ; Extended_Pictographic# 6.0 [3] (😜..😞) winking face with tongue..disappointed face +1F61F ; Extended_Pictographic# 6.1 [1] (😟) worried face +1F620..1F625 ; Extended_Pictographic# 6.0 [6] (😠..😥) angry face..sad but relieved face +1F626..1F627 ; Extended_Pictographic# 6.1 [2] (😦..😧) frowning face with open mouth..anguished face +1F628..1F62B ; Extended_Pictographic# 6.0 [4] (😨..😫) fearful face..tired face +1F62C ; Extended_Pictographic# 6.1 [1] (😬) grimacing face +1F62D ; Extended_Pictographic# 6.0 [1] (😭) loudly crying face +1F62E..1F62F ; Extended_Pictographic# 6.1 [2] (😮..😯) face with open mouth..hushed face +1F630..1F633 ; Extended_Pictographic# 6.0 [4] (😰..😳) anxious face with sweat..flushed face +1F634 ; Extended_Pictographic# 6.1 [1] (😴) sleeping face +1F635..1F640 ; Extended_Pictographic# 6.0 [12] (😵..🙀) dizzy face..weary cat face +1F641..1F642 ; Extended_Pictographic# 7.0 [2] (🙁..🙂) slightly frowning face..slightly smiling face +1F643..1F644 ; Extended_Pictographic# 8.0 [2] (🙃..🙄) upside-down face..face with rolling eyes +1F645..1F64F ; Extended_Pictographic# 6.0 [11] (🙅..🙏) person gesturing NO..folded hands +1F680..1F6C5 ; Extended_Pictographic# 6.0 [70] (🚀..🛅) rocket..left luggage +1F6C6..1F6CF ; Extended_Pictographic# 7.0 [10] (🛆️..🛏️) TRIANGLE WITH ROUNDED CORNERS..bed +1F6D0 ; Extended_Pictographic# 8.0 [1] (🛐) place of worship +1F6D1..1F6D2 ; Extended_Pictographic# 9.0 [2] (🛑..🛒) stop sign..shopping cart +1F6D3..1F6D4 ; Extended_Pictographic# 10.0 [2] (🛓️..🛔️) STUPA..PAGODA +1F6D5..1F6DF ; Extended_Pictographic# NA [11] (🛕️..🛟️) <reserved-1F6D5>..<reserved-1F6DF> +1F6E0..1F6EC ; Extended_Pictographic# 7.0 [13] (🛠️..🛬) hammer and wrench..airplane arrival +1F6ED..1F6EF ; Extended_Pictographic# NA [3] (️..️) <reserved-1F6ED>..<reserved-1F6EF> +1F6F0..1F6F3 ; Extended_Pictographic# 7.0 [4] (🛰️..🛳️) satellite..passenger ship +1F6F4..1F6F6 ; Extended_Pictographic# 9.0 [3] (🛴..🛶) kick scooter..canoe +1F6F7..1F6F8 ; Extended_Pictographic# 10.0 [2] (🛷..🛸) sled..flying saucer +1F6F9 ; Extended_Pictographic# 11.0 [1] (🛹) skateboard +1F6FA..1F6FF ; Extended_Pictographic# NA [6] (🛺️..️) <reserved-1F6FA>..<reserved-1F6FF> +1F774..1F77F ; Extended_Pictographic# NA [12] (🝴️..🝿️) <reserved-1F774>..<reserved-1F77F> +1F7D5..1F7D8 ; Extended_Pictographic# 11.0 [4] (🟕️..🟘️) CIRCLED TRIANGLE..NEGATIVE CIRCLED SQUARE +1F7D9..1F7FF ; Extended_Pictographic# NA [39] (🟙️..️) <reserved-1F7D9>..<reserved-1F7FF> +1F80C..1F80F ; Extended_Pictographic# NA [4] (️..️) <reserved-1F80C>..<reserved-1F80F> +1F848..1F84F ; Extended_Pictographic# NA [8] (️..️) <reserved-1F848>..<reserved-1F84F> +1F85A..1F85F ; Extended_Pictographic# NA [6] (️..️) <reserved-1F85A>..<reserved-1F85F> +1F888..1F88F ; Extended_Pictographic# NA [8] (️..️) <reserved-1F888>..<reserved-1F88F> +1F8AE..1F8FF ; Extended_Pictographic# NA [82] (️..️) <reserved-1F8AE>..<reserved-1F8FF> +1F90C..1F90F ; Extended_Pictographic# NA [4] (🤌️..🤏️) <reserved-1F90C>..<reserved-1F90F> +1F910..1F918 ; Extended_Pictographic# 8.0 [9] (🤐..🤘) zipper-mouth face..sign of the horns +1F919..1F91E ; Extended_Pictographic# 9.0 [6] (🤙..🤞) call me hand..crossed fingers +1F91F ; Extended_Pictographic# 10.0 [1] (🤟) love-you gesture +1F920..1F927 ; Extended_Pictographic# 9.0 [8] (🤠..🤧) cowboy hat face..sneezing face +1F928..1F92F ; Extended_Pictographic# 10.0 [8] (🤨..🤯) face with raised eyebrow..exploding head +1F930 ; Extended_Pictographic# 9.0 [1] (🤰) pregnant woman +1F931..1F932 ; Extended_Pictographic# 10.0 [2] (🤱..🤲) breast-feeding..palms up together +1F933..1F93A ; Extended_Pictographic# 9.0 [8] (🤳..🤺) selfie..person fencing +1F93C..1F93E ; Extended_Pictographic# 9.0 [3] (🤼..🤾) people wrestling..person playing handball +1F93F ; Extended_Pictographic# NA [1] (🤿️) <reserved-1F93F> +1F940..1F945 ; Extended_Pictographic# 9.0 [6] (🥀..🥅) wilted flower..goal net +1F947..1F94B ; Extended_Pictographic# 9.0 [5] (🥇..🥋) 1st place medal..martial arts uniform +1F94C ; Extended_Pictographic# 10.0 [1] (🥌) curling stone +1F94D..1F94F ; Extended_Pictographic# 11.0 [3] (🥍..🥏) lacrosse..flying disc +1F950..1F95E ; Extended_Pictographic# 9.0 [15] (🥐..🥞) croissant..pancakes +1F95F..1F96B ; Extended_Pictographic# 10.0 [13] (🥟..🥫) dumpling..canned food +1F96C..1F970 ; Extended_Pictographic# 11.0 [5] (🥬..🥰) leafy green..smiling face with 3 hearts +1F971..1F972 ; Extended_Pictographic# NA [2] (🥱️..🥲️) <reserved-1F971>..<reserved-1F972> +1F973..1F976 ; Extended_Pictographic# 11.0 [4] (🥳..🥶) partying face..cold face +1F977..1F979 ; Extended_Pictographic# NA [3] (🥷️..🥹️) <reserved-1F977>..<reserved-1F979> +1F97A ; Extended_Pictographic# 11.0 [1] (🥺) pleading face +1F97B ; Extended_Pictographic# NA [1] (🥻️) <reserved-1F97B> +1F97C..1F97F ; Extended_Pictographic# 11.0 [4] (🥼..🥿) lab coat..woman’s flat shoe +1F980..1F984 ; Extended_Pictographic# 8.0 [5] (🦀..🦄) crab..unicorn face +1F985..1F991 ; Extended_Pictographic# 9.0 [13] (🦅..🦑) eagle..squid +1F992..1F997 ; Extended_Pictographic# 10.0 [6] (🦒..🦗) giraffe..cricket +1F998..1F9A2 ; Extended_Pictographic# 11.0 [11] (🦘..🦢) kangaroo..swan +1F9A3..1F9AF ; Extended_Pictographic# NA [13] (🦣️..🦯️) <reserved-1F9A3>..<reserved-1F9AF> +1F9B0..1F9B9 ; Extended_Pictographic# 11.0 [10] (🦰..🦹) red-haired..supervillain +1F9BA..1F9BF ; Extended_Pictographic# NA [6] (🦺️..🦿️) <reserved-1F9BA>..<reserved-1F9BF> +1F9C0 ; Extended_Pictographic# 8.0 [1] (🧀) cheese wedge +1F9C1..1F9C2 ; Extended_Pictographic# 11.0 [2] (🧁..🧂) cupcake..salt +1F9C3..1F9CF ; Extended_Pictographic# NA [13] (🧃️..🧏️) <reserved-1F9C3>..<reserved-1F9CF> +1F9D0..1F9E6 ; Extended_Pictographic# 10.0 [23] (🧐..🧦) face with monocle..socks +1F9E7..1F9FF ; Extended_Pictographic# 11.0 [25] (🧧..🧿) red envelope..nazar amulet +1FA00..1FA5F ; Extended_Pictographic# NA [96] (🨀️..️) <reserved-1FA00>..<reserved-1FA5F> +1FA60..1FA6D ; Extended_Pictographic# 11.0 [14] (🩠️..🩭️) XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER +1FA6E..1FFFD ; Extended_Pictographic# NA[1424] (️..️) <reserved-1FA6E>..<reserved-1FFFD> + +# Total elements: 3793 + +#EOF diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index 535f01a1c5..de67b18afc 100755..100644 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -4,7 +4,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017. All Rights Reserved. +%% Copyright Ericsson AB 2017-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -48,13 +48,18 @@ main(_) -> ok = file:close(ExclF), %% GraphemeBreakProperty table + {ok, Emoji} = file:open("../uc_spec/emoji-data.txt", [read, raw, {read_ahead, 1000000}]), + Props00 = foldl(fun parse_properties/2, [], Emoji), + %% Filter Extended_Pictographic class which we are interested in. + Props0 = [EP || {extended_pictographic, _} = EP <- Props00], + ok = file:close(Emoji), {ok, GBPF} = file:open("../uc_spec/GraphemeBreakProperty.txt", [read, raw, {read_ahead, 1000000}]), - Props0 = foldl(fun parse_properties/2, [], GBPF), + Props1 = foldl(fun parse_properties/2, Props0, GBPF), ok = file:close(GBPF), {ok, PropF} = file:open("../uc_spec/PropList.txt", [read, raw, {read_ahead, 1000000}]), - Props1 = foldl(fun parse_properties/2, Props0, PropF), + Props2 = foldl(fun parse_properties/2, Props1, PropF), ok = file:close(PropF), - Props = sofs:to_external(sofs:relation_to_family(sofs:relation(Props1))), + Props = sofs:to_external(sofs:relation_to_family(sofs:relation(Props2))), %% Make module {ok, Out} = file:open(?MOD++".erl", [write]), @@ -170,7 +175,7 @@ gen_header(Fd) -> io:put_chars(Fd, "-export([spec_version/0, lookup/1, get_case/1]).\n"), io:put_chars(Fd, "-inline([class/1]).\n"), io:put_chars(Fd, "-compile(nowarn_unused_vars).\n"), - io:put_chars(Fd, "-dialyzer({no_improper_lists, [cp/1, gc/1, gc_prepend/2, gc_e_cont/2]}).\n"), + io:put_chars(Fd, "-dialyzer({no_improper_lists, [cp/1, gc/1, gc_prepend/2]}).\n"), io:put_chars(Fd, "-type gc() :: char()|[char()].\n\n\n"), ok. @@ -186,7 +191,7 @@ gen_static(Fd) -> " {U,L} -> #{upper=>U,lower=>L,title=>U,fold=>L};\n" " {U,L,T,F} -> #{upper=>U,lower=>L,title=>T,fold=>F}\n" " end.\n\n"), - io:put_chars(Fd, "spec_version() -> {10,0}.\n\n\n"), + io:put_chars(Fd, "spec_version() -> {11,0}.\n\n\n"), io:put_chars(Fd, "class(Codepoint) -> {CCC,_,_} = unicode_table(Codepoint),\n CCC.\n\n"), io:put_chars(Fd, "-spec uppercase(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), @@ -197,7 +202,8 @@ gen_static(Fd) -> io:put_chars(Fd, " {Upper,_} -> [Upper|Str];\n"), io:put_chars(Fd, " {Upper,_,_,_} -> [Upper|Str]\n"), io:put_chars(Fd, " end;\n"), - io:put_chars(Fd, " [] -> []\n"), + io:put_chars(Fd, " [] -> [];\n"), + io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), io:put_chars(Fd, "-spec lowercase(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), @@ -208,7 +214,8 @@ gen_static(Fd) -> io:put_chars(Fd, " {_,Lower} -> [Lower|Str];\n"), io:put_chars(Fd, " {_,Lower,_,_} -> [Lower|Str]\n"), io:put_chars(Fd, " end;\n"), - io:put_chars(Fd, " [] -> []\n"), + io:put_chars(Fd, " [] -> [];\n"), + io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), io:put_chars(Fd, "-spec titlecase(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), @@ -219,7 +226,8 @@ gen_static(Fd) -> io:put_chars(Fd, " {_,_,Title,_} -> [Title|Str];\n"), io:put_chars(Fd, " {Upper,_} -> [Upper|Str]\n"), io:put_chars(Fd, " end;\n"), - io:put_chars(Fd, " [] -> []\n"), + io:put_chars(Fd, " [] -> [];\n"), + io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), io:put_chars(Fd, "-spec casefold(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), @@ -230,7 +238,8 @@ gen_static(Fd) -> io:put_chars(Fd, " {_,_,_,Fold} -> [Fold|Str];\n"), io:put_chars(Fd, " {_,Lower} -> [Lower|Str]\n"), io:put_chars(Fd, " end;\n"), - io:put_chars(Fd, " [] -> []\n"), + io:put_chars(Fd, " [] -> [];\n"), + io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), ok. @@ -455,17 +464,73 @@ gen_cp(Fd) -> " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, "cp([C|_]=L) when is_integer(C) -> L;\n"), io:put_chars(Fd, "cp([List]) -> cp(List);\n"), - io:put_chars(Fd, "cp([List|R]) ->\n"), - io:put_chars(Fd, " case cp(List) of\n"), - io:put_chars(Fd, " [] -> cp(R);\n"), - io:put_chars(Fd, " [CP] -> [CP|R];\n"), - io:put_chars(Fd, " [C|R0] -> [C|[R0|R]];\n"), - io:put_chars(Fd, " {error,Error} -> {error,[Error|R]}\n"), - io:put_chars(Fd, " end;\n"), + io:put_chars(Fd, "cp([List|R]) -> cpl(List, R);\n"), io:put_chars(Fd, "cp([]) -> [];\n"), io:put_chars(Fd, "cp(<<C/utf8, R/binary>>) -> [C|R];\n"), io:put_chars(Fd, "cp(<<>>) -> [];\n"), - io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n\n"), + io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cpl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cpl([List], R) -> cpl(List, R);\n"), + io:put_chars(Fd, "cpl([List|T], R) -> cpl(List, [T|R]);\n"), + io:put_chars(Fd, "cpl([], R) -> cp(R);\n"), + io:put_chars(Fd, "cpl(<<C/utf8, T/binary>>, R) -> [C,T|R];\n"), + io:put_chars(Fd, "cpl(<<>>, R) -> cp(R);\n"), + io:put_chars(Fd, "cpl(<<B/binary>>, R) -> {error,[B|R]}.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont([C|T], R) when is_integer(C) -> [C|cpl_cont2(T, R)];\n"), + io:put_chars(Fd, "cpl_cont([L], R) -> cpl_cont(L, R);\n"), + io:put_chars(Fd, "cpl_cont([L|T], R) -> cpl_cont(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont([], R) -> cpl_1_cont(R);\n"), + io:put_chars(Fd, "cpl_cont(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont2([C|T], R) when is_integer(C) -> [C|cpl_cont3(T, R)];\n"), + io:put_chars(Fd, "cpl_cont2([L], R) -> cpl_cont2(L, R);\n"), + io:put_chars(Fd, "cpl_cont2([L|T], R) -> cpl_cont2(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont2([], R) -> cpl_1_cont2(R);\n"), + io:put_chars(Fd, "cpl_cont2(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont3([C], R) when is_integer(C) -> [C|R];\n"), + io:put_chars(Fd, "cpl_cont3([C|T], R) when is_integer(C) -> [C,T|R];\n"), + io:put_chars(Fd, "cpl_cont3([L], R) -> cpl_cont3(L, R);\n"), + io:put_chars(Fd, "cpl_cont3([L|T], R) -> cpl_cont3(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont3([], R) -> cpl_1_cont3(R);\n"), + io:put_chars(Fd, "cpl_cont3(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont([C|T]) when is_integer(C) -> [C|cpl_1_cont2(T)];\n"), + io:put_chars(Fd, "cpl_1_cont([L]) -> cpl_1_cont(L);\n"), + io:put_chars(Fd, "cpl_1_cont([L|T]) -> cpl_cont(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont2([C|T]) when is_integer(C) -> [C|cpl_1_cont3(T)];\n"), + io:put_chars(Fd, "cpl_1_cont2([L]) -> cpl_1_cont2(L);\n"), + io:put_chars(Fd, "cpl_1_cont2([L|T]) -> cpl_cont2(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont2(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont3([C|_]=T) when is_integer(C) -> T;\n"), + io:put_chars(Fd, "cpl_1_cont3([L]) -> cpl_1_cont3(L);\n"), + io:put_chars(Fd, "cpl_1_cont3([L|T]) -> cpl_cont3(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont3(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cp_no_bin([C|_]=L) when is_integer(C) -> L;\n"), + io:put_chars(Fd, "cp_no_bin([List]) -> cp_no_bin(List);\n"), + io:put_chars(Fd, "cp_no_bin([List|R]) -> cp_no_binl(List, R);\n"), + io:put_chars(Fd, "cp_no_bin([]) -> [];\n"), + io:put_chars(Fd, "cp_no_bin(_) -> binary_found.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cp_no_binl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cp_no_binl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cp_no_binl([List], R) -> cp_no_binl(List, R);\n"), + io:put_chars(Fd, "cp_no_binl([List|T], R) -> cp_no_binl(List, [T|R]);\n"), + io:put_chars(Fd, "cp_no_binl([], R) -> cp_no_bin(R);\n"), + io:put_chars(Fd, "cp_no_binl(_, _) -> binary_found.\n\n"), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -476,11 +541,26 @@ gen_gc(Fd, GBP) -> "-spec gc(String::unicode:chardata()) ->" " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, - "gc([CP1, CP2|_]=T)\n" - " when CP1 < 256, CP2 < 256, CP1 =/= $\r -> %% Ascii Fast path\n" - " T;\n" + "gc([]=R) -> R;\n" + "gc([CP]=R) when is_integer(CP) -> R;\n" + "gc([$\\r=CP|R0]) ->\n" + " case cp(R0) of % Don't break CRLF\n" + " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" + " T -> [CP|T]\n" + " end;\n" + "gc([CP1|T1]=T) when CP1 < 256 ->\n" + " case T1 of\n" + " [CP2|_] when CP2 < 256 -> T; %% Ascii Fast path\n" + " _ -> %% Keep the tail binary.\n" + " case cp_no_bin(T1) of\n" + " [CP2|_]=T3 when CP2 < 256 -> [CP1|T3]; %% Asciii Fast path\n" + " binary_found -> gc_1(T);\n" + " T4 -> gc_1([CP1|T4])\n" + " end\n" + " end;\n" + "gc(<<>>) -> [];\n" "gc(<<CP1/utf8, Rest/binary>>) ->\n" - " if CP1 < 256, CP1 =/= $\r ->\n" + " if CP1 < 256, CP1 =/= $\\r ->\n" " case Rest of\n" " <<CP2/utf8, _/binary>> when CP2 < 256 -> %% Ascii Fast path\n" " [CP1|Rest];\n" @@ -488,40 +568,52 @@ gen_gc(Fd, GBP) -> " end;\n" " true -> gc_1([CP1|Rest])\n" " end;\n" + "gc([CP|_]=T) when is_integer(CP) -> gc_1(T);\n" "gc(Str) ->\n" - " gc_1(cp(Str)).\n\n" - "gc_1([$\\r|R0] = R) ->\n" + " case cp(Str) of\n" + " {error,_}=Error -> Error;\n" + " CPs -> gc(CPs)\n" + " end.\n" + ), + + GenExtP = fun(Range) -> io:format(Fd, "gc_1~s gc_ext_pict(R1,[CP]);\n", [gen_clause(Range)]) end, + ExtendedPictographic0 = merge_ranges(maps:get(extended_pictographic,GBP)), + %% Pick codepoints below 256 (some data knowledge here) + {ExtendedPictographicLow,ExtendedPictographicHigh} = + lists:splitwith(fun({Start,undefined}) -> Start < 256 end,ExtendedPictographic0), + io:put_chars(Fd, + "\ngc_1([$\\r|R0] = R) ->\n" " case cp(R0) of % Don't break CRLF\n" " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" " _ -> R\n" - " end;\n" - %% "gc_1([CP1, CP2|_]=T) when CP1 < 256, CP2 < 256 ->\n" - %% " T; %% Fast path\n" - %% "gc_1([CP1|<<CP2/utf8, _/binary>>]=T) when CP1 < 256, CP2 < 256 ->\n" - %% " T; %% Fast path\n" - ), - - io:put_chars(Fd, "%% Handle control\n"), + " end;\n"), + io:put_chars(Fd, "\n%% Handle control\n"), GenControl = fun(Range) -> io:format(Fd, "gc_1~s R0;\n", [gen_clause(Range)]) end, CRs0 = merge_ranges(maps:get(cr, GBP) ++ maps:get(lf, GBP) ++ maps:get(control, GBP), false), [R1,R2,R3|Crs] = CRs0, [GenControl(CP) || CP <- merge_ranges([R1,R2,R3], split), CP =/= {$\r, undefined}], %%GenControl(R1),GenControl(R2),GenControl(R3), - io:format(Fd, "gc_1([CP|R]) when CP < 256 -> gc_extend(R,CP);\n", []), + io:put_chars(Fd, "\n%% Optimize Latin-1\n"), + [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicLow)], + + io:format(Fd, + "gc_1([CP|R]=R0) when CP < 256 ->\n" + " case R of\n" + " [CP2|_] when CP2 < 256 -> R0;\n" + " _ -> gc_extend(cp(R), R, CP)\n" + " end;\n", + []), + io:put_chars(Fd, "\n%% Continue control\n"), [GenControl(CP) || CP <- Crs], %% One clause per CP %% CRs0 = merge_ranges(maps:get(cr, GBP) ++ maps:get(lf, GBP) ++ maps:get(control, GBP)), %% [GenControl(CP) || CP <- CRs0, CP =/= {$\r, undefined}], - io:put_chars(Fd, "%% Handle ZWJ\n"), - GenZWJ = fun(Range) -> io:format(Fd, "gc_1~s gc_zwj(R1, [CP]);\n", [gen_clause(Range)]) end, - [GenZWJ(CP) || CP <- merge_ranges(maps:get(zwj,GBP))], - - io:put_chars(Fd, "%% Handle prepend\n"), + io:put_chars(Fd, "\n%% Handle prepend\n"), GenPrepend = fun(Range) -> io:format(Fd, "gc_1~s gc_prepend(R1, CP);\n", [gen_clause(Range)]) end, [GenPrepend(CP) || CP <- merge_ranges(maps:get(prepend,GBP))], - io:put_chars(Fd, "%% Handle Hangul L\n"), + io:put_chars(Fd, "\n%% Handle Hangul L\n"), GenHangulL = fun(Range) -> io:format(Fd, "gc_1~s gc_h_L(R1,[CP]);\n", [gen_clause(Range)]) end, [GenHangulL(CP) || CP <- merge_ranges(maps:get(l,GBP))], io:put_chars(Fd, "%% Handle Hangul V\n"), @@ -531,21 +623,22 @@ gen_gc(Fd, GBP) -> GenHangulT = fun(Range) -> io:format(Fd, "gc_1~s gc_h_T(R1,[CP]);\n", [gen_clause(Range)]) end, [GenHangulT(CP) || CP <- merge_ranges(maps:get(t,GBP))], io:put_chars(Fd, "%% Handle Hangul LV and LVT special, since they are large\n"), - io:put_chars(Fd, "gc_1([CP|_]=R0) when 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, []);\n"), + io:put_chars(Fd, "gc_1([CP|_]=R0) when 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, R0, []);\n"), - io:put_chars(Fd, "%% Handle Regional\n"), - GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,[CP]);\n", [gen_clause(Range)]) end, + io:put_chars(Fd, "\n%% Handle Regional\n"), + GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,CP);\n", [gen_clause(Range)]) end, [GenRegional(CP) || CP <- merge_ranges(maps:get(regional_indicator,GBP))], - io:put_chars(Fd, "%% Handle E_Base\n"), - GenEBase = fun(Range) -> io:format(Fd, "gc_1~s gc_e_cont(R1,[CP]);\n", [gen_clause(Range)]) end, - [GenEBase(CP) || CP <- merge_ranges(maps:get(e_base,GBP))], - io:put_chars(Fd, "%% Handle EBG\n"), - GenEBG = fun(Range) -> io:format(Fd, "gc_1~s gc_e_cont(R1,[CP]);\n", [gen_clause(Range)]) end, - [GenEBG(CP) || CP <- merge_ranges(maps:get(e_base_gaz,GBP))], - - io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(R, CP);\n"), - io:put_chars(Fd, "gc_1([]) -> [];\n"), - io:put_chars(Fd, "gc_1({error,_}=Error) -> Error.\n\n"), + %% io:put_chars(Fd, "%% Handle E_Base\n"), + %% GenEBase = fun(Range) -> io:format(Fd, "gc_1~s gc_e_cont(R1,[CP]);\n", [gen_clause(Range)]) end, + %% [GenEBase(CP) || CP <- merge_ranges(maps:get(e_base,GBP))], + %% io:put_chars(Fd, "%% Handle EBG\n"), + %% GenEBG = fun(Range) -> io:format(Fd, "gc_1~s gc_e_cont(R1,[CP]);\n", [gen_clause(Range)]) end, + %% [GenEBG(CP) || CP <- merge_ranges(maps:get(e_base_gaz,GBP))], + + io:put_chars(Fd, "%% Handle extended_pictographic\n"), + [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicHigh)], + io:put_chars(Fd, "\n%% default clauses\n"), + io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(cp(R), R, CP).\n\n"), io:put_chars(Fd, "%% Handle Prepend\n"), io:put_chars(Fd, @@ -572,36 +665,24 @@ gen_gc(Fd, GBP) -> "%% To simplify binary handling in libraries the tail should be kept binary\n" "%% and not a lookahead CP\n" ), - io:put_chars(Fd, "gc_extend(T, Acc) ->\n" - " gc_extend(cp(T), T, Acc).\n\n"), io:put_chars(Fd, - "gc_extend([CP|T], T0, Acc0) ->\n" + "gc_extend([CP|T], T0, CP0) ->\n" " case is_extend(CP) of\n" - " zwj ->\n" - " case Acc0 of\n" - " [_|_] -> gc_zwj(T, [CP|Acc0]);\n" - " Acc -> gc_zwj(T, [CP,Acc])\n" - " end;\n" - " true ->\n" - " case Acc0 of\n" - " [_|_] -> gc_extend(T, [CP|Acc0]);\n" - " Acc -> gc_extend(T, [CP,Acc])\n" - " end;\n" - " false ->\n" - " case Acc0 of\n" - " [Acc] -> [Acc|T0];\n" - " [_|_]=Acc -> [lists:reverse(Acc)|T0];\n" - " Acc -> [Acc|T0]\n" - " end\n" + " false -> [CP0|T0]; % losing work done on T\n" + " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP,CP0])\n" " end;\n" - "gc_extend([], _, Acc0) ->\n" - " case Acc0 of\n" - " [_]=Acc -> Acc;\n" - " [_|_]=Acc -> [lists:reverse(Acc)];\n" - " Acc -> [Acc]\n" + "gc_extend([], _, CP) -> [CP];\n" + "gc_extend({error,R}, _, CP) -> [CP|R].\n\n"), + io:put_chars(Fd, + "gc_extend2([CP|T], T0, Acc) ->\n" + " case is_extend(CP) of\n" + " false -> [lists:reverse(Acc)|T0]; % losing work done on T\n" + " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP|Acc])\n" " end;\n" - "gc_extend({error,R}, T, Acc0) ->\n" - " gc_extend([], T, Acc0) ++ [R].\n\n" + "gc_extend2([], _, Acc) ->\n" + " [lists:reverse(Acc)];\n" + "gc_extend2({error,R}, _, Acc) ->\n" + " [lists:reverse(Acc)] ++ [R].\n\n" ), [ZWJ] = maps:get(zwj, GBP), GenExtend = fun(R) when R =:= ZWJ -> io:format(Fd, "is_extend~s zwj;\n", [gen_single_clause(ZWJ)]); @@ -612,57 +693,54 @@ gen_gc(Fd, GBP) -> io:put_chars(Fd, "is_extend(_) -> false.\n\n"), io:put_chars(Fd, - "gc_e_cont(R0, Acc) ->\n" - " case cp(R0) of\n" - " [CP|R1] ->\n" - " case is_extend(CP) of\n" - " zwj -> gc_zwj(R1, [CP|Acc]);\n" - " true -> gc_e_cont(R1, [CP|Acc]);\n" - " false ->\n" - " case is_emodifier(CP) of\n" - " true -> [lists:reverse([CP|Acc])|R1];\n" - " false ->\n" - " case Acc of\n" - " [A] -> [A|R0];\n" - " _ -> [lists:reverse(Acc)|R0]\n" - " end\n" - " end\n" - " end;\n" - " [] ->\n" + "gc_ext_pict(T, Acc) ->\n" + " gc_ext_pict(cp(T), T, Acc).\n\n" + "gc_ext_pict([CP|R1], T0, Acc) ->\n" + " case is_extend(CP) of\n" + " zwj -> gc_ext_pict_zwj(cp(R1), R1, [CP|Acc]);\n" + " true -> gc_ext_pict(R1, [CP|Acc]);\n" + " false ->\n" " case Acc of\n" - " [A] -> [A];\n" - " _ -> [lists:reverse(Acc)]\n" - " end;\n" - " {error,R} ->\n" + " [A] -> [A|T0];\n" + " _ -> [lists:reverse(Acc)|T0]\n" + " end\n" + " end;\n" + "gc_ext_pict([], _T0, Acc) ->\n" + " case Acc of\n" + " [A] -> [A];\n" + " _ -> [lists:reverse(Acc)]\n" + " end;\n" + "gc_ext_pict({error,R}, T, Acc) ->\n" + " gc_ext_pict([], T, Acc) ++ [R].\n\n"), + io:put_chars(Fd, + "gc_ext_pict_zwj([CP|R1], T0, Acc) ->\n" + " case is_ext_pict(CP) of\n" + " true -> gc_ext_pict(R1, [CP|Acc]);\n" + " false ->\n" " case Acc of\n" - " [A] -> [A|R];\n" - " _ -> [lists:reverse(Acc)|R]\n" + " [A] -> [A|T0];\n" + " _ -> [lists:reverse(Acc)|T0]\n" " end\n" - " end.\n\n"), - - GenEMod = fun(Range) -> io:format(Fd, "is_emodifier~s true;\n", [gen_single_clause(Range)]) end, - EMods = merge_ranges(maps:get(e_modifier, GBP), split), - [GenEMod(CP) || CP <- EMods], - io:put_chars(Fd, "is_emodifier(_) -> false.\n\n"), - - io:put_chars(Fd, "gc_zwj(R0, Acc) ->\n case cp(R0) of\n"), - GenZWJGlue = fun(Range) -> io:format(Fd, "~8c~s gc_extend(cp(R1), R0, [CP|Acc]);\n", - [$\s,gen_case_clause(Range)]) end, - [GenZWJGlue(CP) || CP <- merge_ranges(maps:get(glue_after_zwj,GBP))], - GenZWJEBG = fun(Range) -> io:format(Fd, "~8c~s gc_e_cont(R1, [CP|Acc]);\n", - [$\s,gen_case_clause(Range)]) end, - [GenZWJEBG(CP) || CP <- merge_ranges(maps:get(e_base_gaz,GBP))], - io:put_chars(Fd," R1 -> gc_extend(R1, R0, Acc)\n" - " end.\n\n"), + " end;\n" + "gc_ext_pict_zwj([], _, Acc) ->\n" + " case Acc of\n" + " [A] -> [A];\n" + " _ -> [lists:reverse(Acc)]\n" + " end;\n" + "gc_ext_pict_zwj({error,R}, T, Acc) ->\n" + " gc_ext_pict_zwj([], T, Acc) ++ [R].\n\n"), + GenExtPict = fun(Range) -> io:format(Fd, "is_ext_pict~s true;\n", [gen_single_clause(Range)]) end, + [GenExtPict(CP) || CP <- ExtendedPictographic0], + io:put_chars(Fd, "is_ext_pict(_) -> false.\n\n"), %% -------------------- io:put_chars(Fd, "%% Handle Regional\n"), [{RLess,RLarge}] = merge_ranges(maps:get(regional_indicator,GBP)), - io:put_chars(Fd,"gc_regional(R0, Acc) ->\n" + io:put_chars(Fd,"gc_regional(R0, CP0) ->\n" " case cp(R0) of\n"), - io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend(R1,[CP|Acc]);~n",[RLess, RLarge]), - io:put_chars(Fd," R1 -> gc_extend(R1, R0, Acc)\n" + io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend2(cp(R1),R1,[CP,CP0]);~n",[RLess, RLarge]), + io:put_chars(Fd," R1 -> gc_extend(R1, R0, CP0)\n" " end.\n\n"), %% Special hangul @@ -674,7 +752,7 @@ gen_gc(Fd, GBP) -> GenHangulL_2 = fun(Range) -> io:format(Fd, "~8c~s gc_h_V(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulL_2(CP) || CP <- merge_ranges(maps:get(v,GBP))], - io:put_chars(Fd, " R1 -> gc_h_lv_lvt(R1, Acc)\n end.\n\n"), + io:put_chars(Fd, " R1 -> gc_h_lv_lvt(R1, R0, Acc)\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul V\n"), io:put_chars(Fd, "gc_h_V(R0, Acc) ->\n case cp(R0) of\n"), @@ -684,16 +762,23 @@ gen_gc(Fd, GBP) -> GenHangulV_2 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulV_2(CP) || CP <- merge_ranges(maps:get(t,GBP))], - io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"), - + io:put_chars(Fd, + " R1 ->\n" + " case Acc of\n" + " [CP] -> gc_extend(R1, R0, CP);\n" + " _ -> gc_extend2(R1, R0, Acc)\n" + " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul T\n"), io:put_chars(Fd, "gc_h_T(R0, Acc) ->\n case cp(R0) of\n"), GenHangulT_1 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulT_1(CP) || CP <- merge_ranges(maps:get(t,GBP))], - io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"), - - io:put_chars(Fd, "gc_h_lv_lvt({error,_}=Error, Acc) -> gc_extend(Error, [], Acc);\n"), + io:put_chars(Fd, + " R1 ->\n" + " case Acc of\n" + " [CP] -> gc_extend(R1, R0, CP);\n" + " _ -> gc_extend2(R1, R0, Acc)\n" + " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul LV\n"), GenHangulLV = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_V(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, @@ -702,8 +787,10 @@ gen_gc(Fd, GBP) -> GenHangulLVT = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_T(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, [GenHangulLVT(CP) || CP <- merge_ranges(maps:get(lvt,GBP))], - io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(R, CP);\n"), %% From gc_1/1 - io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend(R, Acc).\n\n"), + io:put_chars(Fd, "gc_h_lv_lvt([CP|R1], _, []) -> gc_extend(cp(R1), R1, CP);\n"), %% From gc_1/1 + io:put_chars(Fd, "%% Also handles error tuples\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R1, R0, [CP]) -> gc_extend(R1, R0, CP);\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R1, R0, Acc) -> gc_extend2(R1, R0, Acc).\n\n"), ok. gen_compose_pairs(Fd, ExclData, Data) -> @@ -804,9 +891,9 @@ gen_clause({R0, R1}) -> io_lib:format("([CP|R1]=R0) when ~w =< CP, CP =< ~w ->", [R0,R1]). gen_clause2({R0, undefined}) -> - io_lib:format("([~w=CP|R1], Acc) ->", [R0]); + io_lib:format("([~w=CP|R1], R0, Acc) ->", [R0]); gen_clause2({R0, R1}) -> - io_lib:format("([CP|R1], Acc) when ~w =< CP, CP =< ~w ->", [R0,R1]). + io_lib:format("([CP|R1], R0, Acc) when ~w =< CP, CP =< ~w ->", [R0,R1]). gen_case_clause({R0, undefined}) -> io_lib:format("[~w=CP|R1] ->", [R0]); diff --git a/lib/syntax_tools/doc/demo.erl b/lib/syntax_tools/doc/demo.erl new file mode 120000 index 0000000000..fe40fb65ec --- /dev/null +++ b/lib/syntax_tools/doc/demo.erl @@ -0,0 +1 @@ +../examples/demo.erl
\ No newline at end of file diff --git a/lib/syntax_tools/doc/overview.edoc b/lib/syntax_tools/doc/overview.edoc index 3111633a99..7be96f1a55 100644 --- a/lib/syntax_tools/doc/overview.edoc +++ b/lib/syntax_tools/doc/overview.edoc @@ -26,7 +26,7 @@ library module {@link prettypr}: this is a powerful and flexible generic pretty printing library, which is also distributed separately. For a short demonstration of parsing and pretty-printing, simply -compile the included module <a href="../examples/demo.erl">`demo.erl'</a>, +compile the included module <a href="demo.erl">`demo.erl'</a>, and execute `demo:run()' from the Erlang shell. It will compile the remaining modules and give you further instructions. diff --git a/lib/syntax_tools/doc/src/Makefile b/lib/syntax_tools/doc/src/Makefile index d953287bad..b799c76177 100644 --- a/lib/syntax_tools/doc/src/Makefile +++ b/lib/syntax_tools/doc/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2006-2018. All Rights Reserved. +# Copyright Ericsson AB 2006-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -82,6 +82,9 @@ HTML_REF_MAN_FILE = $(HTMLDIR)/index.html TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf +EXAMPLES_DIR = ../../examples +EXAMPLES = $(EXAMPLES_DIR)/demo.erl + SPECS_FILES = $(XML_REF3_FILES:%.xml=$(SPECDIR)/specs_%.xml) TOP_SPECS_FILE = specs.xml @@ -146,5 +149,7 @@ release_docs_spec: docs $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" $(INSTALL_DIR) "$(RELEASE_PATH)/man/man3" $(INSTALL_DATA) $(MAN3DIR)/* "$(RELEASE_PATH)/man/man3" + $(INSTALL_DIR) "$(RELSYSDIR)/examples" + $(INSTALL_DATA) $(EXAMPLES) "$(RELSYSDIR)/doc/html" release_spec: diff --git a/lib/syntax_tools/src/igor.erl b/lib/syntax_tools/src/igor.erl index 16e3511734..b712b77e9f 100644 --- a/lib/syntax_tools/src/igor.erl +++ b/lib/syntax_tools/src/igor.erl @@ -660,7 +660,7 @@ merge_files1(Files, Opts) -> %% transitions), code replacement is expected to be detected. Then, if %% we in the merged code do not check at these points if the %% <em>target</em> module (the result of the merge) has been replaced, -%% we can not be sure in general that we will be able to do code +%% we cannot be sure in general that we will be able to do code %% replacement of the merged state machine - it could run forever %% without detecting the code change. Therefore, all such calls must %% remain remote-calls (detecting code changes), but may call the target diff --git a/lib/syntax_tools/test/syntax_tools_SUITE.erl b/lib/syntax_tools/test/syntax_tools_SUITE.erl index 6b42f7a0a1..e1dd1bd73b 100644 --- a/lib/syntax_tools/test/syntax_tools_SUITE.erl +++ b/lib/syntax_tools/test/syntax_tools_SUITE.erl @@ -74,7 +74,7 @@ smoke_test_file(File) -> [print_error_markers(F, File) || F <- Forms], ok; {error,Reason} -> - io:format("~s: ~p\n", [File,Reason]), + io:format("~ts: ~p\n", [File,Reason]), error end. @@ -82,7 +82,7 @@ print_error_markers(F, File) -> case erl_syntax:type(F) of error_marker -> {L,M,Info} = erl_syntax:error_marker_info(F), - io:format("~ts:~p: ~s", [File,L,M:format_error(Info)]); + io:format("~ts:~p: ~ts", [File,L,M:format_error(Info)]); _ -> ok end. @@ -362,7 +362,7 @@ test_comment_scan([File|Files],DataDir) -> end, Fs1 = erl_recomment:recomment_forms(Fs0, Comments), Fs2 = erl_syntax_lib:map(Fun, Fs1), - io:format("File: ~s~n", [Filename]), + io:format("File: ~ts~n", [Filename]), io:put_chars(erl_prettypr:format(Fs2, [{paper, 120}, {ribbon, 110}])), test_comment_scan(Files,DataDir). @@ -377,8 +377,8 @@ test_prettypr([File|Files],DataDir,PrivDir) -> PP = erl_prettypr:format(Fs, [{paper, 120}, {ribbon, 110}]), io:put_chars(PP), OutFile = filename:join(PrivDir, File), - ok = file:write_file(OutFile,iolist_to_binary(PP)), - io:format("Parsing OutFile: ~s~n", [OutFile]), + ok = file:write_file(OutFile,unicode:characters_to_binary(PP)), + io:format("Parsing OutFile: ~ts~n", [OutFile]), {ok, Fs2} = epp:parse_file(OutFile, [], []), case [Error || {error, _} = Error <- Fs2] of [] -> @@ -445,7 +445,7 @@ pretty_print_parse_forms([{Fs0,Type}|FsForms],PrivDir,Filename) -> {Fs2,{CC,CT}} = erl_syntax_lib:mapfold(Comment,{0,0}, Fs1), io:format("Commented on ~w cases and ~w tries~n", [CC,CT]), PP = erl_prettypr:format(Fs2), - ok = file:write_file(OutFile,iolist_to_binary(PP)), + ok = file:write_file(OutFile,unicode:characters_to_binary(PP)), pretty_print_parse_forms(FsForms,PrivDir,Filename). diff --git a/lib/tools/doc/src/cover.xml b/lib/tools/doc/src/cover.xml index 64c24cea2a..e9f782977d 100644 --- a/lib/tools/doc/src/cover.xml +++ b/lib/tools/doc/src/cover.xml @@ -128,14 +128,26 @@ </desc> </func> <func> - <name since="">start(Nodes) -> {ok,StartedNodes} | {error,not_main_node}</name> + <name since="OTP 22.0">local_only() -> ok | {error,too_late}</name> + <fsummary>Only support running Cover on the local node.</fsummary> + <desc> + <p>Only support running Cover on the local node. This function + must be called before any modules have been compiled or any + nodes added. When running in this mode, modules will be Cover + compiled in a more efficient way, but the resulting code will + only work on the same node they were compiled on.</p> + </desc> + </func> + <func> + <name since="">start(Nodes) -> {ok,StartedNodes} | {error,not_main_node} | {error,local_only}</name> <fsummary>Start Cover on remote nodes.</fsummary> <type> <v>Nodes = StartedNodes = [atom()]</v> </type> <desc> <p>Starts a Cover server on the each of given nodes, and loads - all cover compiled modules.</p> + all cover compiled modules. This call will fail if + <c>cover:local_only/0</c> has been called.</p> </desc> </func> <func> @@ -234,7 +246,7 @@ <c>{already_cover_compiled,no_beam_found,Module}</c> is returned.</p> <p><c>{error,BeamFile}</c> is returned if the compiled code - can not be loaded on the node.</p> + cannot be loaded on the node.</p> <p>If a list of <c>ModFiles</c> is given as input, a list of <c>Result</c> will be returned. The order of the returned list is undefined.</p> @@ -470,7 +482,7 @@ <p>Exports the current coverage data for <c>Module</c> to the file <c>ExportFile</c>. It is recommended to name the <c>ExportFile</c> with the extension <c>.coverdata</c>, since - other filenames can not be read by the web based interface to + other filenames cannot be read by the web based interface to cover.</p> <p>If <c>Module</c> is not given, data for all Cover compiled or earlier imported modules is exported.</p> @@ -496,7 +508,7 @@ <p>Coverage data from several export files can be imported into one system. The coverage data is then added up when analysing.</p> - <p>Coverage data for a module can not be imported from the + <p>Coverage data for a module cannot be imported from the same file twice unless the module is first reset or compiled. The check is based on the filename, so you can easily fool the system by renaming your export file.</p> diff --git a/lib/tools/doc/src/instrument.xml b/lib/tools/doc/src/instrument.xml index 75be22de9b..7e9cbaebb0 100644 --- a/lib/tools/doc/src/instrument.xml +++ b/lib/tools/doc/src/instrument.xml @@ -111,15 +111,18 @@ default, but this can be configured an a per-allocator basis with the <seealso marker="erts:erts_alloc#M_atags"><c>+M<S>atags</c> </seealso> emulator option.</p> - <p>If tagged allocations are not enabled on any of the specified - allocator types, the call will fail with - <c>{error, not_enabled}</c>.</p> + <p>If the specified allocator types are not enabled, the call will fail + with <c>{error, not_enabled}</c>.</p> <p>The following options can be used:</p> <taglist> <tag><c>allocator_types</c></tag> <item> - <p>The allocator types that will be searched. Defaults to all - <c>alloc_util</c> allocators.</p> + <p>The allocator types that will be searched. Note that blocks can + move freely between allocator types, so restricting the search to + certain allocators may return unexpected types (e.g. process + heaps when searching binary_alloc), or hide blocks that were + migrated out.</p> + <p>Defaults to all <c>alloc_util</c> allocators.</p> </item> <tag><c>scheduler_ids</c></tag> <item> diff --git a/lib/tools/doc/src/make.xml b/lib/tools/doc/src/make.xml index af2404707f..322d77323f 100644 --- a/lib/tools/doc/src/make.xml +++ b/lib/tools/doc/src/make.xml @@ -145,5 +145,10 @@ Modules. {'*',[debug_info]}. </code> <p></p> </section> + + <section> + <title>See Also</title> + <p><seealso marker="compiler:compile"><c>compile(3)</c></seealso></p> + </section> </erlref> diff --git a/lib/tools/emacs/erlang-test.el b/lib/tools/emacs/erlang-test.el index 2ee584d11a..fbdd298da3 100644 --- a/lib/tools/emacs/erlang-test.el +++ b/lib/tools/emacs/erlang-test.el @@ -50,8 +50,15 @@ ;; The -L option adds a directory to the load-path. It should be the ;; directory containing erlang.el and erlang-test.el. ;; -;; 3. Call the script test-erlang-mode in this directory. This script -;; use the second method. +;; 3. Run the emacs_SUITE. The testcases tests_interpreted/1 and +;; tests_compiled/1 in this suite are using the second method. One +;; way to run this suite is with the ct_run tool, for example like the +;; following when standing at the OTP repo top directory: +;; +;; ct_run -suite lib/tools/test/emacs_SUITE +;; +;; Note that this creates a lot of html log files in the current +;; directory. ;;; Code: diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 3cbe9daa60..0b3a2319e2 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -4,7 +4,7 @@ ;; Author: Anders Lindgren ;; Keywords: erlang, languages, processes ;; Date: 2011-12-11 -;; Version: 2.8.1 +;; Version: 2.8.2 ;; Package-Requires: ((emacs "24.1")) ;; %CopyrightBegin% @@ -87,7 +87,7 @@ "The Erlang programming language." :group 'languages) -(defconst erlang-version "2.8.1" +(defconst erlang-version "2.8.2" "The version number of Erlang mode.") (defcustom erlang-root-dir nil @@ -502,6 +502,13 @@ regardless of where in the line point is when the TAB command is used." :type 'boolean :safe 'booleanp) +(defcustom erlang-max-files-to-visit-for-refining-xrefs 32 + "Upper limit how many files to visit for checking arity. +When `nil' there is no limit." + :group 'erlang + :type '(restricted-sexp :match-alternatives (integerp 'nil)) + :safe (lambda (val) (or (eq val nil) (integerp val)))) + (defvar erlang-man-inhibit (eq system-type 'windows-nt) "Inhibit the creation of the Erlang Manual Pages menu. @@ -905,8 +912,10 @@ resulting regexp is surrounded by \\_< and \\_>." "dist_get_stat" "dist_ctrl_get_data" "dist_ctrl_get_data_notification" + "dist_ctrl_get_opt" "dist_ctrl_input_handler" "dist_ctrl_put_data" + "dist_ctrl_set_opt" "dmonitor_node" "dt_append_vm_tag_data" "dt_get_tag" @@ -3687,10 +3696,13 @@ When an identifier is found return a list with 4 elements: module or nil. 2. Module - Module name string or nil. In case of a -qualified-function a search fails if no entries with correct -module are found. For other kinds the module is just a -preference. If no matching entries are found the search will be -retried without regard to module. +qualified-function the module is explicitly specified (like +module:fun()) and the search fails if no entries with correct +module are found. For other kinds the module is guessed: either +fetched from import statements or it is assumed to be the local +module. In these cases the module is just a preference. If no +matching entries are found the search will be retried without +regard to module. 3. Name - String name of function, module, record or macro. @@ -3702,18 +3714,22 @@ of arguments could be found, otherwise nil." (if (eq (char-syntax (following-char)) ? ) (skip-chars-backward " \t")) (skip-chars-backward "[:word:]_:'") - (cond ((looking-at erlang-module-function-regexp) + (cond ((and (eq (preceding-char) ??) + (looking-at (concat "\\(MODULE\\):" erlang-atom-regexp))) + (erlang-get-qualified-function-id-at-point (erlang-get-module))) + ((looking-at erlang-module-function-regexp) (erlang-get-qualified-function-id-at-point)) ((looking-at (concat erlang-atom-regexp ":")) (erlang-get-module-id-at-point)) ((looking-at erlang-name-regexp) (erlang-get-some-other-id-at-point))))))) -(defun erlang-get-qualified-function-id-at-point () +(defun erlang-get-qualified-function-id-at-point (&optional module) (let ((kind 'qualified-function) - (module (erlang-remove-quotes - (buffer-substring-no-properties - (match-beginning 1) (match-end 1)))) + (module (or module + (erlang-remove-quotes + (buffer-substring-no-properties + (match-beginning 1) (match-end 1))))) (name (erlang-remove-quotes (buffer-substring-no-properties (match-beginning (1+ erlang-atom-regexp-matches)) @@ -3823,7 +3839,8 @@ of arguments could be found, otherwise nil." (let ((case-fold-search nil)) ; force string matching to be case sensitive (if (and (stringp str) (not (string-match (eval-when-compile - (concat "\\`" erlang-atom-regexp "\\'")) str))) + (concat "\\`" erlang-atom-regexp "\\'")) + str))) (progn (setq str (replace-regexp-in-string "'" "\\'" str t t )) (concat "'" str "'")) @@ -4877,15 +4894,36 @@ about Erlang modules." ;; The backend below is a wrapper around the built-in etags backend. ;; It adds awareness of the module:tag syntax in a similar way that is ;; done above for the old etags commands. +;; +;; In addition arity is also considered when jumping to definitions. +;; There is however currently no information about arity in the TAGS +;; file. Also two functions with the same name but different arity +;; _sometimes_ get one TAGS entry each and sometimes are joined in one +;; single entry. If they are directly consecutive they will be +;; joined. If there are other functions etc in between then they will +;; get one entry each. +;; +;; These limitations are present in both the etags program shipped +;; with GNU Emacs and the tags.erl program in this repository. +;; +;; Therefore erlang.el must complement the information in TAGS by +;; visiting files and checking arity. When searching for popular +;; function names (like init, handle_call etc) in a big TAGS file +;; (like one indexing this repository) this may be quite +;; time-consuming. There exists therefore an upper limit for the +;; number of files to visit (called +;; `erlang-max-files-to-visit-for-refining-xrefs'). +;; +;; As mentioned this xref implementation is based on the etags xref +;; implementation. But in the cases where arity is considered the +;; etags information structures (class xref-etags-location) will be +;; translated to our own structures which include arity (class +;; erlang-xref-location). This translation is started in the function +;; `erlang-refine-xrefs'. -(defvar erlang-current-arity nil - "The arity of the function currently being searched. - -There is no information about arity in the TAGS file. -Consecutive functions with same name but different arity will -only get one entry in the TAGS file. Matching TAGS entries are -therefore selected without regarding arity. The arity is -considered first when it is time to jump to the definition.") +;; I mention this as a head up that some of the functions below deal +;; with xref items with xref-etags-location and some deal with xref +;; items with erlang-xref-location. (defun erlang-etags--xref-backend () 'erlang-etags) @@ -4893,127 +4931,80 @@ considered first when it is time to jump to the definition.") (when (locate-library (symbol-name feature)) (require feature))) -(and (erlang-soft-require 'xref) - (erlang-soft-require 'cl-generic) - (erlang-soft-require 'eieio) - (erlang-soft-require 'etags) - ;; The purpose of using eval here is to avoid compilation - ;; warnings in emacsen without cl-defmethod etc. - (eval - '(progn - (cl-defmethod xref-backend-identifier-at-point - ((_backend (eql erlang-etags))) - (if (eq this-command 'xref-find-references) - (if (use-region-p) - (buffer-substring-no-properties (region-beginning) - (region-end)) - (thing-at-point 'symbol)) - (erlang-id-to-string (erlang-get-identifier-at-point)))) - - (cl-defmethod xref-backend-definitions - ((_backend (eql erlang-etags)) identifier) - (erlang-xref-find-definitions identifier)) - - (cl-defmethod xref-backend-apropos - ((_backend (eql erlang-etags)) identifier) - (erlang-xref-find-definitions identifier t)) - - (cl-defmethod xref-backend-identifier-completion-table - ((_backend (eql erlang-etags))) - (let ((erlang-replace-etags-tags-completion-table t)) - (tags-completion-table))) - - (defclass erlang-xref-location (xref-etags-location) ()) - - (defun erlang-convert-xrefs (xrefs) - (mapcar (lambda (xref) - (oset xref location (erlang-make-location - (oref xref location))) - xref) - xrefs)) - - (defun erlang-make-location (etags-location) - (with-slots (tag-info file) etags-location - (make-instance 'erlang-xref-location :tag-info tag-info - :file file))) - - (cl-defmethod xref-location-marker ((locus erlang-xref-location)) - (with-slots (tag-info file) locus - (with-current-buffer (find-file-noselect file) - (save-excursion - (or (erlang-goto-tag-location-by-arity tag-info) - (etags-goto-tag-location tag-info)) - ;; Reset erlang-current-arity. We want to jump to - ;; correct arity in the first attempt. That is now - ;; done. Possible remaining jumps will be from - ;; entries in the *xref* buffer and then we want to - ;; ignore the arity. (Alternatively we could remove - ;; all but one xref entry per file when we know the - ;; arity). - (setq erlang-current-arity nil) - (point-marker))))) - - (defun erlang-xref-context (xref) - (with-slots (tag-info) (xref-item-location xref) - (car tag-info)))))) - - -(defun erlang-goto-tag-location-by-arity (tag-info) - (when erlang-current-arity - (let* ((tag-text (car tag-info)) - (tag-pos (cdr (cdr tag-info))) - (tag-line (car (cdr tag-info))) - (regexp (erlang-tag-info-regexp tag-text)) - (startpos (or tag-pos - (when tag-line - (goto-char (point-min)) - (forward-line (1- tag-line)) - (point)) - (point-min)))) - (setq startpos (max (- startpos 2000) - (point-min))) - (goto-char startpos) - (let ((pos (or (erlang-search-by-arity regexp) - (unless (eq startpos (point-min)) - (goto-char (point-min)) - (erlang-search-by-arity regexp))))) - (when pos - (goto-char pos) - t))))) - -(defun erlang-tag-info-regexp (tag-text) - (concat "^" - (regexp-quote tag-text) - ;; Erlang function entries in TAGS includes the opening - ;; parenthesis for the argument list. Erlang macro entries - ;; do not. Add it here in order to end up in correct - ;; position for erlang-get-arity. - (if (string-prefix-p "-define" tag-text) - "\\s-*(" - ""))) - -(defun erlang-search-by-arity (regexp) - (let (pos) - (while (and (null pos) - (re-search-forward regexp nil t)) - (when (eq erlang-current-arity (save-excursion (erlang-get-arity))) - (setq pos (point-at-bol)))) - pos)) - - +(when (and (erlang-soft-require 'xref) + (erlang-soft-require 'cl-generic) + (erlang-soft-require 'eieio) + (erlang-soft-require 'etags)) + ;; The purpose of using eval here is to avoid compilation + ;; warnings in emacsen without cl-defmethod etc. + (eval + '(progn + (cl-defmethod xref-backend-identifier-at-point ((_backend + (eql erlang-etags))) + (if (eq this-command 'xref-find-references) + (if (use-region-p) + (buffer-substring-no-properties (region-beginning) + (region-end)) + (thing-at-point 'symbol)) + (erlang-id-to-string (erlang-get-identifier-at-point)))) + + (cl-defmethod xref-backend-definitions ((_backend (eql erlang-etags)) + identifier) + (erlang-xref-find-definitions identifier)) + + (cl-defmethod xref-backend-apropos ((_backend (eql erlang-etags)) + identifier) + (erlang-xref-find-definitions identifier t)) + + (cl-defmethod xref-backend-identifier-completion-table + ((_backend (eql erlang-etags))) + (let ((erlang-replace-etags-tags-completion-table t)) + (tags-completion-table))) + + (defclass erlang-xref-location (xref-file-location) + ((arity :type fixnum :initarg :arity + :reader erlang-xref-location-arity)) + :documentation "An erlang location is a file location plus arity.") + + ;; This method definition only calls the superclass which is + ;; the default behaviour if it was not defined. It is only + ;; needed for "upgrade" purposes. In version 2.8.1 of + ;; erlang.el this method was defined differently and in case + ;; user switch to a new erlang.el without restarting Emacs + ;; this method needs to be redefined. + (cl-defmethod xref-location-marker ((locus erlang-xref-location)) + (cl-call-next-method locus))))) + +;; If this function returns a single xref the user will jump to that +;; directly. If two or more xrefs are returned a *xref* window is +;; displayed and the user can choose where to jump. Hence we want to +;; return a single xref when we are pretty sure that is where the user +;; wants to go. Otherwise return all possible xrefs but sort them so +;; that xrefs in the local file is first and if arity is known sort +;; the xrefs with matching arity before others. + +;; Note that the arity sorting work may partly be undone later when +;; the hits are presented in the *xref* buffer since they then will be +;; grouped together by file. Ie when one file have one hit with +;; correct arity and others with wrong arity these hits will be +;; grouped together and may end up before hits with correct arity. (defun erlang-xref-find-definitions (identifier &optional is-regexp) (erlang-with-id (kind module name arity) identifier - (setq erlang-current-arity arity) (cond ((eq kind 'module) (erlang-xref-find-definitions-module name)) + ((eq kind 'qualified-function) + (erlang-xref-find-definitions-qualified-function module + name + arity + is-regexp)) (module - (erlang-xref-find-definitions-module-tag module + (erlang-xref-find-definitions-module-tag kind + module name - (eq kind - 'qualified-function) + arity is-regexp)) (t - (erlang-xref-find-definitions-tag kind name is-regexp))))) + (erlang-xref-find-definitions-tag kind name arity is-regexp))))) (defun erlang-xref-find-definitions-module (module) (and (fboundp 'xref-make) @@ -5038,65 +5029,252 @@ considered first when it is time to jump to the definition.") (setq files (cdr files)))))) (nreverse xrefs)))) -(defun erlang-visit-tags-table-buffer (cont cbuf) - (if (< emacs-major-version 26) - (visit-tags-table-buffer cont) - ;; Remove this with-no-warnings when Emacs 26 is the required - ;; version minimum. - (with-no-warnings - (visit-tags-table-buffer cont cbuf)))) - -(defun erlang-xref-find-definitions-module-tag (module +(defun erlang-xref-find-definitions-qualified-function (module + tag + arity + is-regexp) + "Find definitions of TAG in MODULE preferably with arity ARITY. +If one single perfect match was found return only that (ignoring +other definitions matching TAG). If IS-REGEXP is non-nil then +TAG is a regexp." + (let* ((xrefs (when (fboundp 'etags--xref-find-definitions) + (etags--xref-find-definitions tag is-regexp))) + (xrefs-split (erlang-split-xrefs-on-module xrefs module)) + (module-xrefs (car xrefs-split)) + (module-xrefs (erlang-refine-xrefs module-xrefs + 'qualified-function + tag + is-regexp))) + (or (erlang-single-arity-match module-xrefs arity) + (erlang-sort-by-arity module-xrefs arity)))) + + +;; We will end up here when erlang-get-some-other-id-at-point either +;; found module among the import statements or module is just the +;; current local file. +(defun erlang-xref-find-definitions-module-tag (kind + module tag - is-qualified + arity is-regexp) - "Find definitions of TAG and filter away definitions outside of -MODULE. If IS-QUALIFIED is nil and no definitions was found inside -the MODULE then return any definitions found outside. If -IS-REGEXP is non-nil then TAG is a regexp." - (and (fboundp 'etags--xref-find-definitions) - (fboundp 'erlang-convert-xrefs) - (let ((xrefs (erlang-convert-xrefs - (etags--xref-find-definitions tag is-regexp))) - xrefs-in-module) - (dolist (xref xrefs) - (when (string-equal module (erlang-xref-module xref)) - (push xref xrefs-in-module))) - (cond (is-qualified xrefs-in-module) - (xrefs-in-module xrefs-in-module) - (t xrefs))))) - -(defun erlang-xref-find-definitions-tag (kind tag is-regexp) - "Find all definitions of TAG and reorder them so that -definitions in the currently visited file comes first." - (and (fboundp 'etags--xref-find-definitions) - (fboundp 'erlang-convert-xrefs) - (let* ((current-file (and (buffer-file-name) - (file-truename (buffer-file-name)))) - (regexp (erlang-etags-regexp kind tag is-regexp)) - (xrefs (erlang-convert-xrefs - (etags--xref-find-definitions regexp t))) - local-xrefs non-local-xrefs) - (while xrefs - (let ((xref (car xrefs))) - (if (string-equal (erlang-xref-truename-file xref) - current-file) - (push xref local-xrefs) - (push xref non-local-xrefs)) - (setq xrefs (cdr xrefs)))) - (append (reverse local-xrefs) - (reverse non-local-xrefs))))) + "Find definitions of TAG preferably in MODULE and with arity ARITY. +Return definitions outside MODULE if none are found inside. If +IS-REGEXP is non-nil then TAG is a regexp. + +If one single perfect match was found return only that (ignoring +other definitions matching TAG)." + (let* ((xrefs (when (fboundp 'etags--xref-find-definitions) + (etags--xref-find-definitions tag is-regexp))) + (xrefs-split (erlang-split-xrefs-on-module xrefs module)) + (module-xrefs (car xrefs-split)) + (module-xrefs (erlang-refine-xrefs module-xrefs + kind + tag + is-regexp))) + (or (erlang-single-arity-match module-xrefs arity) + (erlang-xref-find-definitions-tag kind tag arity is-regexp xrefs)))) + +(defun erlang-xref-find-definitions-tag (kind + tag + arity + is-regexp + &optional xrefs) + "Find definitions of TAG preferably in local file and with arity ARITY. +If one single perfect match was found return only that (ignoring +other definitions matching TAG). If no such local match was +found then look for a matching BIF in the same way. If IS-REGEXP +is non-nil then TAG is a regexp." + (let* ((regexp (erlang-etags-regexp kind tag is-regexp)) + (xrefs (or xrefs + (when (fboundp 'etags--xref-find-definitions) + (etags--xref-find-definitions regexp t)))) + (xrefs-split (erlang-split-xrefs xrefs)) + (local-xrefs (car xrefs-split)) + (local-xrefs (erlang-refine-xrefs local-xrefs + kind + tag + is-regexp)) + (bif-xrefs (cadr xrefs-split)) + (other-xrefs (caddr xrefs-split))) + (or (erlang-single-arity-match local-xrefs arity) + ;; No local match, look for a matching BIF. + (progn + (setq bif-xrefs (erlang-refine-xrefs bif-xrefs + kind + tag + is-regexp)) + (erlang-single-arity-match bif-xrefs arity)) + (progn + (setq other-xrefs (erlang-refine-xrefs other-xrefs + kind + tag + is-regexp)) + (and (null local-xrefs) + (null bif-xrefs) + ;; No local of BIF matches at all. Is there a single + ;; arity match among the rest? + (erlang-single-arity-match other-xrefs arity))) + (append (erlang-sort-by-arity local-xrefs arity) + (erlang-sort-by-arity bif-xrefs arity) + (erlang-sort-by-arity other-xrefs arity))))) + + +(defun erlang-refine-xrefs (xrefs kind tag is-regexp) + (if (or (memq kind '(record module)) + ;; No support for apropos here. + is-regexp + (erlang-too-many-files-in-xrefs xrefs)) + xrefs + (when (and xrefs + (fboundp 'xref-item-location) + (fboundp 'xref-location-group) + (fboundp 'slot-value)) + (let (files) + (cl-loop for xref in xrefs + for loc = (xref-item-location xref) + for file = (xref-location-group loc) + do (pushnew file files :test 'string-equal)) + (or (cl-loop for file in files + append (erlang-xrefs-in-file file kind tag is-regexp)) + ;; Failed for some reason. Pretend like it is raining and + ;; return the unrefined xrefs. + xrefs))))) + +(defun erlang-too-many-files-in-xrefs (xrefs) + (and erlang-max-files-to-visit-for-refining-xrefs + (let ((files-to-visit (delete-dups + (mapcar #'erlang-xref-truename-file + xrefs)))) + (if (< (length files-to-visit) + erlang-max-files-to-visit-for-refining-xrefs) + nil + (message (concat "Too many hits to consider arity (see " + "`erlang-max-files-to-visit-for-refining-xrefs')")) + t)))) + +(defun erlang-xrefs-in-file (file kind tag is-regexp) + (when (fboundp 'make-instance) + (with-current-buffer (find-file-noselect file) + (save-excursion + (goto-char (point-min)) + (let ((regexp (concat ; "^" + (erlang-etags-regexp kind tag is-regexp) + "\\s *(")) + last-arity) + (cl-loop while (re-search-forward regexp nil t) + for name = (match-string-no-properties 1) + for arity = (save-excursion + (erlang-get-arity)) + for loc = (make-instance 'erlang-xref-location + :file file + :line (line-number-at-pos) + :column 0 + :arity arity) + for sum = (erlang-xref-summary kind name arity) + when (and arity + (not (eq arity last-arity))) + collect (make-instance 'xref-item + :summary sum + :location loc) + do (setq last-arity arity))))))) + +(defun erlang-xref-summary (kind tag arity) + (format "%s%s%s" + (if (memq kind '(record macro module)) + (format "%s " kind) + "") + tag + (if arity (format "/%s" arity) ""))) + +(defun erlang-single-arity-match (xrefs wanted-arity) + "Attempt to find one perfect match. + +If we have all information needed to consider arity then return a +single perfect match or nothing. If there are more than one +match nothing is returned. + +If we don't have all information needed to consider arity just +return XREFS as is." + (if (erlang-should-consider-arity-p xrefs wanted-arity) + (let ((nr-matches 0) + match) + (while (and xrefs + (< nr-matches 2)) + (let* ((xref (car xrefs)) + (arity (erlang-xref-arity xref))) + (when (eq arity wanted-arity) + (setq match xref + nr-matches (1+ nr-matches))) + (setq xrefs (cdr xrefs)))) + (when (eq nr-matches 1) + (list match))) + (when (eq (length xrefs) 1) + xrefs))) + +(defun erlang-sort-by-arity (xrefs wanted-arity) + (if (erlang-should-consider-arity-p xrefs wanted-arity) + (let (matches non-matches) + (while xrefs + (let* ((xref (car xrefs)) + (arity (erlang-xref-arity xref))) + (push xref (if (eq arity wanted-arity) + matches + non-matches)) + (setq xrefs (cdr xrefs)))) + (append (reverse matches) (reverse non-matches) xrefs)) + xrefs)) + +(defun erlang-should-consider-arity-p (xrefs wanted-arity) + (and wanted-arity + xrefs + (fboundp 'erlang-xref-location-p) + (fboundp 'xref-item-location) + (erlang-xref-location-p (xref-item-location (car xrefs))))) (defun erlang-etags-regexp (kind tag is-regexp) - (let ((tag-regexp (if is-regexp - tag - (regexp-quote tag)))) - (cond ((eq kind 'record) - (concat "-record\\s-*(\\s-*" tag-regexp)) - ((eq kind 'macro) - (concat "-define\\s-*(\\s-*" tag-regexp)) - (t tag-regexp)))) - + (let ((tag-regexp (concat "\\(" + (if is-regexp + tag + (regexp-quote tag)) + "\\)"))) + (concat (if is-regexp "" "^") + (cond ((eq kind 'record) + (concat "-record\\s-*(\\s-*" tag-regexp)) + ((eq kind 'macro) + (concat "-define\\s-*(\\s-*" tag-regexp)) + (t + tag-regexp)) + (if is-regexp "" "\\_>")))) + +(defun erlang-xref-arity (xref) + (and (fboundp 'erlang-xref-location-arity) + (fboundp 'xref-item-location) + (erlang-xref-location-arity (xref-item-location xref)))) + +(defun erlang-split-xrefs-on-module (xrefs module) + (let (local-xrefs non-local-xrefs) + (dolist (xref xrefs) + (if (string-equal (erlang-xref-module xref) + module) + (push xref local-xrefs) + (push xref non-local-xrefs))) + (cons (reverse local-xrefs) + (reverse non-local-xrefs)))) + +(defun erlang-split-xrefs (xrefs) + (let ((current-file (and (buffer-file-name) + (file-truename (buffer-file-name)))) + local-xrefs bif-xrefs other-xrefs) + (dolist (xref xrefs) + (cond ((string-equal (erlang-xref-truename-file xref) current-file) + (push xref local-xrefs)) + ((string-equal (erlang-xref-module xref) "erlang") + (push xref bif-xrefs)) + (t + (push xref other-xrefs)))) + (list (reverse local-xrefs) + (reverse bif-xrefs) + (reverse other-xrefs)))) (defun erlang-xref-module (xref) (erlang-get-module-from-file-name (erlang-xref-file xref))) @@ -5111,7 +5289,13 @@ definitions in the currently visited file comes first." (fboundp 'xref-item-location) (xref-location-group (xref-item-location xref)))) - +(defun erlang-visit-tags-table-buffer (cont cbuf) + (if (< emacs-major-version 26) + (visit-tags-table-buffer cont) + ;; Remove this with-no-warnings when Emacs 26 is the required + ;; version minimum. + (with-no-warnings + (visit-tags-table-buffer cont cbuf)))) ;;; ;;; Prepare for other methods to run an Erlang slave process. diff --git a/lib/tools/emacs/erldoc.el b/lib/tools/emacs/erldoc.el index 770ab299ee..bc16d7a14d 100644 --- a/lib/tools/emacs/erldoc.el +++ b/lib/tools/emacs/erldoc.el @@ -89,6 +89,13 @@ up the indexing." :type 'file :group 'erldoc) +(defcustom erldoc-no-signature-function #'ignore + "Notification function called if no function signature was found." + :type '(choice (function-item :tag "Ignore" ignore) + (function-item :tag "Warn" warn) + (function-item :tag "Error" error)) + :group 'erldoc) + (defun erldoc-strip-string (s) (let* ((re "[ \t\n\r\f\v\u00a0]+") (from (if (string-match (concat "\\`" re) s) (match-end 0) 0)) @@ -212,11 +219,21 @@ up the indexing." ;; Get the full function signature. (when (and (eq (car-safe d) 'a) (gethash (erldoc-dom-get-attribute d 'name) table)) - (push (append (gethash (erldoc-dom-get-attribute d 'name) table) - (list (funcall span-content - (or (erldoc-dom-get-element d 'span) - (cadr (memq d erldoc-dom-walk-siblings)))))) - entries)) + (let* ((name (erldoc-dom-get-attribute d 'name)) + (mfa-url (gethash name table)) + (mfa (car mfa-url)) + (sig (or (funcall span-content d) + (funcall span-content + (or (erldoc-dom-get-element d 'span) + (cadr + (memq d erldoc-dom-walk-siblings)))) + (progn + (funcall erldoc-no-signature-function + "erldoc-parse-man: no sig for %s" + mfa) + nil)))) + (push (append mfa-url (list sig)) + entries))) ;; Get data types (when (and (eq (car-safe d) 'a) (string-prefix-p "type-" @@ -280,7 +297,7 @@ up the indexing." (unless (file-exists-p of) (erldoc-parse-all erldoc-man-index of)) (unless (string= erldoc-output-file of) - (make-symbolic-link of erldoc-output-file)))) + (make-symbolic-link (expand-file-name of) erldoc-output-file)))) (setq erldoc-lookup-table (with-temp-buffer (insert-file-contents erldoc-output-file) @@ -356,9 +373,12 @@ up the indexing." (sigs)) (maphash (lambda (k v) (when (string-match re k) - (push (cons (string-to-number (match-string 1 k)) - (cdr (erldoc-tokenize-signature (cadr v)))) - sigs))) + (if (cadr v) + (push (cons (string-to-number (match-string 1 k)) + (cdr (erldoc-tokenize-signature (cadr v)))) + sigs) + (funcall erldoc-no-signature-function + "erldoc-format-signature: No sig for %s" k)))) (erldoc-lookup-table)) (when sigs ;; Mostly single return type but there are exceptions such as diff --git a/lib/tools/src/cover.erl b/lib/tools/src/cover.erl index 8d4561ca9e..8fe866cb69 100644 --- a/lib/tools/src/cover.erl +++ b/lib/tools/src/cover.erl @@ -23,6 +23,7 @@ %% This module implements the Erlang coverage tool. %% %% ARCHITECTURE +%% %% The coverage tool consists of one process on each node involved in %% coverage analysis. The process is registered as 'cover_server' %% (?SERVER). The cover_server on the 'main' node is in charge, and @@ -30,45 +31,62 @@ %% 'DOWN' message for another cover_server, it marks the node as %% 'lost'. If a nodeup is received for a lost node the main node %% ensures that the cover compiled modules are loaded again. If the -%% remote node was alive during the disconnected periode, cover data -%% for this periode will also be included in the analysis. +%% remote node was alive during the disconnected period, cover data +%% for this period will also be included in the analysis. %% %% The cover_server process on the main node is implemented by the %% functions init_main/1 and main_process_loop/1. The cover_server on %% the remote nodes are implemented by the functions init_remote/2 and %% remote_process_loop/1. %% +%% COUNTERS +%% +%% The 'counters' modules is used for counting how many time each line +%% executed. Each cover-compiled module will have its own array of +%% counters. +%% +%% The counter reference for module Module is stored in a persistent +%% term with the key {cover,Module}. +%% +%% When the cover:local_only/0 function has been called, the reference +%% for the counter array will be compiled into each cover-compiled +%% module directly (instead of retrieving it from a persistent term). +%% That will be faster, but the resulting code can be only be used on +%% the main node. +%% %% TABLES -%% Each nodes has two tables: cover_internal_data_table (?COVER_TABLE) and. -%% cover_internal_clause_table (?COVER_CLAUSE_TABLE). -%% ?COVER_TABLE contains the bump data i.e. the data about which lines -%% have been executed how many times. +%% +%% Each node has two tables: ?COVER_MAPPING_TABLE and ?COVER_CLAUSE_TABLE. +%% ?COVER_MAPPING_TABLE maps from a #bump{} record to an index in the +%% counter array for the module. It is used both during instrumentation +%% of cover-compiled modules and when collecting the counter values. +%% %% ?COVER_CLAUSE_TABLE contains information about which clauses in which modules %% cover is currently collecting statistics. -%% -%% The main node owns tables named -%% 'cover_collected_remote_data_table' (?COLLECTION_TABLE) and -%% 'cover_collected_remote_clause_table' (?COLLECTION_CLAUSE_TABLE). -%% These tables contain data which is collected from remote nodes (either when a -%% remote node is stopped with cover:stop/1 or when analysing). When -%% analysing, data is even moved from the COVER tables on the main -%% node to the COLLECTION tables. %% -%% The main node also has a table named 'cover_binary_code_table' -%% (?BINARY_TABLE). This table contains the binary code for each cover -%% compiled module. This is necessary so that the code can be loaded -%% on remote nodes that are started after the compilation. +%% The main node owns the tables ?COLLECTION_TABLE and +%% ?COLLECTION_CLAUSE_TABLE. The counter data is consolidated into those +%% tables from the counters on both the main node and from remote nodes. +%% This consolidation is done when a remote node is stopped with +%% cover:stop/1 or just before starting an analysis. +%% +%% The main node also has a table named ?BINARY_TABLE. This table +%% contains the abstract code code for each cover-compiled +%% module. This is necessary so that the code can be loaded on remote +%% nodes that are started after the compilation. %% %% PARALLELISM +%% %% To take advantage of SMP when doing the cover analysis both the data %% collection and analysis has been parallelized. One process is spawned for %% each node when collecting data, and on the remote node when collecting data %% one process is spawned per module. %% -%% When analyzing data it is possible to issue multiple analyse(_to_file)/X -%% calls at once. They are however all calls (for backwards compatibility -%% reasons) so the user of cover will have to spawn several processes to to the -%% calls ( or use async_analyse_to_file ). +%% When analyzing data it is possible to issue multiple +%% analyse(_to_file)/X calls at once. They are, however, all calls +%% (for backwards compatibility reasons), so the user of cover will +%% have to spawn several processes to to the calls (or use +%% async_analyse_to_file/X). %% %% External exports @@ -89,7 +107,8 @@ modules/0, imported/0, imported_modules/0, which_nodes/0, is_compiled/1, reset/1, reset/0, flush/1, - stop/0, stop/1]). + stop/0, stop/1, + local_only/0]). -export([remote_start/1,get_main_node/0]). %% Used internally to ensure we upgrade the code to the latest version. @@ -98,9 +117,16 @@ -record(main_state, {compiled=[], % [{Module,File}] imported=[], % [{Module,File,ImportFile}] stopper, % undefined | pid() + local_only=false, % true | false nodes=[], % [Node] lost_nodes=[]}). % [Node] +-record(remote_data, {module, + file, + code, + mapping, + clauses}). + -record(remote_state, {compiled=[], % [{Module,File}] main_node}). % atom() @@ -126,11 +152,12 @@ is_guard=false % boolean }). --define(COVER_TABLE, 'cover_internal_data_table'). +-define(COVER_MAPPING_TABLE, 'cover_internal_mapping_table'). -define(COVER_CLAUSE_TABLE, 'cover_internal_clause_table'). -define(BINARY_TABLE, 'cover_binary_code_table'). -define(COLLECTION_TABLE, 'cover_collected_remote_data_table'). -define(COLLECTION_CLAUSE_TABLE, 'cover_collected_remote_clause_table'). + -define(TAG, cover_compiled). -define(SERVER, cover_server). @@ -186,6 +213,11 @@ start(Node) when is_atom(Node) -> start(Nodes) -> call({start_nodes,remove_myself(Nodes,[])}). +%% local_only() -> ok | {error,too_late} + +local_only() -> + call(local_only). + %% compile(ModFiles) -> %% compile(ModFiles, Options) -> %% compile_module(ModFiles) -> Result @@ -255,15 +287,8 @@ compile_directory(Dir, Options) when is_list(Dir), is_list(Options) -> compile_modules(Files,Options) -> Options2 = filter_options(Options), - %% compile_modules(Files,Options2,[]). call({compile, Files, Options2}). -%% compile_modules([File|Files], Options, Result) -> -%% R = call({compile, File, Options}), -%% compile_modules(Files,Options,[R|Result]); -%% compile_modules([],_Opts,Result) -> -%% lists:reverse(Result). - filter_options(Options) -> lists:filter(fun(Option) -> case Option of @@ -561,16 +586,6 @@ flush(Nodes) -> get_main_node() -> call(get_main_node). -%% bump(Module, Function, Arity, Clause, Line) -%% Module = Function = atom() -%% Arity = Clause = Line = integer() -%% This function is inserted into Cover compiled modules, once for each -%% executable line. -%bump(Module, Function, Arity, Clause, Line) -> -% Key = #bump{module=Module, function=Function, arity=Arity, clause=Clause, -% line=Line}, -% ets:update_counter(?COVER_TABLE, Key, 1). - call(Request) -> Ref = erlang:monitor(process,?SERVER), receive {'DOWN', Ref, _Type, _Object, noproc} -> @@ -631,10 +646,8 @@ remote_reply(MainNode,Reply) -> init_main(Starter) -> register(?SERVER,self()), - %% Having write concurrancy here gives a 40% performance boost - %% when collect/1 is called. - ?COVER_TABLE = ets:new(?COVER_TABLE, [set, public, named_table, - {write_concurrency, true}]), + ?COVER_MAPPING_TABLE = ets:new(?COVER_MAPPING_TABLE, + [ordered_set, public, named_table]), ?COVER_CLAUSE_TABLE = ets:new(?COVER_CLAUSE_TABLE, [set, public, named_table]), ?BINARY_TABLE = ets:new(?BINARY_TABLE, [set, public, named_table]), @@ -648,10 +661,26 @@ init_main(Starter) -> main_process_loop(State) -> receive + {From, local_only} -> + case State of + #main_state{compiled=[],nodes=[]} -> + reply(From, ok), + main_process_loop(State#main_state{local_only=true}); + #main_state{} -> + reply(From, {error,too_late}), + main_process_loop(State) + end; + {From, {start_nodes,Nodes}} -> - {StartedNodes,State1} = do_start_nodes(Nodes, State), - reply(From, {ok,StartedNodes}), - main_process_loop(State1); + case State#main_state.local_only of + false -> + {StartedNodes,State1} = do_start_nodes(Nodes, State), + reply(From, {ok,StartedNodes}), + main_process_loop(State1); + true -> + reply(From, {error,local_only}), + main_process_loop(State) + end; {From, {compile, Files, Options}} -> {R,S} = do_compile(Files, Options, State), @@ -742,11 +771,12 @@ main_process_loop(State) -> end, State#main_state.nodes), reload_originals(State#main_state.compiled), - ets:delete(?COVER_TABLE), + ets:delete(?COVER_MAPPING_TABLE), ets:delete(?COVER_CLAUSE_TABLE), ets:delete(?BINARY_TABLE), ets:delete(?COLLECTION_TABLE), ets:delete(?COLLECTION_CLAUSE_TABLE), + delete_all_counters(), unregister(?SERVER), reply(From, ok); @@ -878,10 +908,8 @@ main_process_loop(State) -> init_remote(Starter,MainNode) -> register(?SERVER,self()), - %% write_concurrency here makes otp_8270 break :( - ?COVER_TABLE = ets:new(?COVER_TABLE, [set, public, named_table - %,{write_concurrency, true} - ]), + ?COVER_MAPPING_TABLE = ets:new(?COVER_MAPPING_TABLE, + [ordered_set, public, named_table]), ?COVER_CLAUSE_TABLE = ets:new(?COVER_CLAUSE_TABLE, [set, public, named_table]), Starter ! {self(),started}, @@ -904,7 +932,7 @@ remote_process_loop(State) -> remote_process_loop(State#remote_state{compiled=Compiled}); {remote,reset,Module} -> - do_reset(Module), + reset_counters(Module), remote_reply(State#remote_state.main_node, ok), remote_process_loop(State); @@ -925,8 +953,9 @@ remote_process_loop(State) -> {remote,stop} -> reload_originals(State#remote_state.compiled), - ets:delete(?COVER_TABLE), + ets:delete(?COVER_MAPPING_TABLE), ets:delete(?COVER_CLAUSE_TABLE), + delete_all_counters(), unregister(?SERVER), ok; % not replying since 'DOWN' message will be received anyway @@ -961,28 +990,12 @@ remote_process_loop(State) -> end. do_collect(Modules, CollectorPid, From) -> - _ = pmap( - fun(Module) -> - Pattern = {#bump{module=Module, _='_'}, '$1'}, - MatchSpec = [{Pattern,[{'=/=','$1',0}],['$_']}], - Match = ets:select(?COVER_TABLE,MatchSpec,?CHUNK_SIZE), - send_chunks(Match, CollectorPid, []) - end,Modules), + _ = pmap(fun(Module) -> + send_counters(Module, CollectorPid) + end, Modules), CollectorPid ! done, remote_reply(From, ok). -send_chunks('$end_of_table', _CollectorPid, Mons) -> - get_downs(Mons); -send_chunks({Chunk,Continuation}, CollectorPid, Mons) -> - Mon = spawn_monitor( - fun() -> - lists:foreach(fun({Bump,_N}) -> - ets:insert(?COVER_TABLE, {Bump,0}) - end, - Chunk) end), - send_chunk(CollectorPid,Chunk), - send_chunks(ets:select(Continuation), CollectorPid, [Mon|Mons]). - send_chunk(CollectorPid,Chunk) -> CollectorPid ! {chunk,Chunk,self()}, receive continue -> ok end. @@ -1021,10 +1034,15 @@ do_reload_original(Module) -> ignore end. -load_compiled([{Module,File,Binary,InitialTable}|Compiled],Acc) -> - %% Make sure the #bump{} records are available *before* the - %% module is loaded. - insert_initial_data(InitialTable), +load_compiled([Data|Compiled],Acc) -> + %% Make sure the #bump{} records and counters are available *before* + %% compiling and loading the code. + #remote_data{module=Module,file=File,code=Beam, + mapping=InitialMapping,clauses=InitialClauses} = Data, + ets:insert(?COVER_MAPPING_TABLE, InitialMapping), + ets:insert(?COVER_CLAUSE_TABLE, InitialClauses), + maybe_create_counters(Module, true), + Sticky = case code:is_sticky(Module) of true -> code:unstick_mod(Module), @@ -1032,7 +1050,7 @@ load_compiled([{Module,File,Binary,InitialTable}|Compiled],Acc) -> false -> false end, - NewAcc = case code:load_binary(Module, ?TAG, Binary) of + NewAcc = case code:load_binary(Module, ?TAG, Beam) of {module,Module} -> add_compiled(Module, File, Acc); _ -> @@ -1047,16 +1065,6 @@ load_compiled([{Module,File,Binary,InitialTable}|Compiled],Acc) -> load_compiled([],Acc) -> Acc. -insert_initial_data([Item|Items]) when is_atom(element(1,Item)) -> - ets:insert(?COVER_CLAUSE_TABLE, Item), - insert_initial_data(Items); -insert_initial_data([Item|Items]) -> - ets:insert(?COVER_TABLE, Item), - insert_initial_data(Items); -insert_initial_data([]) -> - ok. - - unload([Module|Modules]) -> do_clear(Module), do_reload_original(Module), @@ -1177,7 +1185,7 @@ get_downs_r([]) -> []; get_downs_r(Mons) -> receive - {'DOWN', Ref, _Type, Pid, R={_,_,_,_}} -> + {'DOWN', Ref, _Type, Pid, #remote_data{}=R} -> [R|get_downs_r(lists:delete({Pid,Ref},Mons))]; {'DOWN', Ref, _Type, Pid, Reason} = Down -> case lists:member({Pid,Ref},Mons) of @@ -1196,19 +1204,13 @@ get_downs_r(Mons) -> %% Binary is the beam code for the module and InitialTable is the initial %% data to insert in ?COVER_TABLE. get_data_for_remote_loading({Module,File}) -> - [{Module,Binary}] = ets:lookup(?BINARY_TABLE,Module), + [{Module,Code}] = ets:lookup(?BINARY_TABLE, Module), %%! The InitialTable list will be long if the module is big - what to do?? - InitialBumps = ets:select(?COVER_TABLE,ms(Module)), + Mapping = counters_mapping_table(Module), InitialClauses = ets:lookup(?COVER_CLAUSE_TABLE,Module), - {Module,File,Binary,InitialBumps ++ InitialClauses}. - -%% Create a match spec which returns the clause info {Module,InitInfo} and -%% all #bump keys for the given module with 0 number of calls. -ms(Module) -> - ets:fun2ms(fun({Key,_}) when Key#bump.module=:=Module -> - {Key,0} - end). + #remote_data{module=Module,file=File,code=Code, + mapping=Mapping,clauses=InitialClauses}. %% Unload modules on remote nodes remote_unload(Nodes,UnloadedModules) -> @@ -1464,7 +1466,7 @@ get_compiled_still_loaded(Nodes,Compiled0) -> do_compile_beams(ModsAndFiles, State) -> Result0 = pmap(fun({ok,Module,File}) -> - do_compile_beam(Module,File,State); + do_compile_beam(Module, File, State); (Error) -> Error end, @@ -1476,8 +1478,10 @@ do_compile_beams(ModsAndFiles, State) -> do_compile_beam(Module,BeamFile0,State) -> case get_beam_file(Module,BeamFile0,State#main_state.compiled) of {ok,BeamFile} -> + LocalOnly = State#main_state.local_only, UserOptions = get_compile_options(Module,BeamFile), - case do_compile_beam1(Module,BeamFile,UserOptions) of + case do_compile_beam1(Module,BeamFile, + UserOptions,LocalOnly) of {ok, Module} -> {ok,Module,BeamFile}; error -> @@ -1503,41 +1507,39 @@ fix_state_and_result([],State,Acc) -> do_compile(Files, Options, State) -> + LocalOnly = State#main_state.local_only, Result0 = pmap(fun(File) -> - do_compile(File, Options) + do_compile1(File, Options, LocalOnly) end, Files), Compiled = [{M,F} || {ok,M,F} <- Result0], remote_load_compiled(State#main_state.nodes,Compiled), fix_state_and_result(Result0,State,[]). -do_compile(File, Options) -> - case do_compile1(File, Options) of +do_compile1(File, Options, LocalOnly) -> + case do_compile2(File, Options, LocalOnly) of {ok, Module} -> {ok,Module,File}; error -> {error,File} end. -%% do_compile1(File, Options) -> {ok,Module} | error -do_compile1(File, UserOptions) -> +%% do_compile2(File, Options) -> {ok,Module} | error +do_compile2(File, UserOptions, LocalOnly) -> Options = [debug_info,binary,report_errors,report_warnings] ++ UserOptions, case compile:file(File, Options) of {ok, Module, Binary} -> - do_compile_beam1(Module,Binary,UserOptions); + do_compile_beam1(Module,Binary,UserOptions,LocalOnly); error -> error end. %% Beam is a binary or a .beam file name -do_compile_beam1(Module,Beam,UserOptions) -> +do_compile_beam1(Module,Beam,UserOptions,LocalOnly) -> %% Clear database do_clear(Module), - %% Extract the abstract format and insert calls to bump/6 at - %% every executable line and, as a side effect, initiate - %% the database - + %% Extract the abstract format. case get_abstract_code(Module, Beam) of no_abstract_code=E -> {error,E}; @@ -1547,7 +1549,8 @@ do_compile_beam1(Module,Beam,UserOptions) -> Forms0 = epp:interpret_file_attribute(Code), case find_main_filename(Forms0) of {ok,MainFile} -> - do_compile_beam2(Module,Beam,UserOptions,Forms0,MainFile); + do_compile_beam2(Module,Beam,UserOptions, + Forms0,MainFile,LocalOnly); Error -> Error end; @@ -1566,26 +1569,35 @@ get_abstract_code(Module, Beam) -> Error -> Error end. -do_compile_beam2(Module,Beam,UserOptions,Forms0,MainFile) -> - {Forms,Vars} = transform(Forms0, Module, MainFile), +do_compile_beam2(Module,Beam,UserOptions,Forms0,MainFile,LocalOnly) -> + init_counter_mapping(Module), + + %% Instrument the abstract code by inserting + %% calls to update the counters. + {Forms,Vars} = transform(Forms0, Module, MainFile, LocalOnly), + + %% Create counters. + maybe_create_counters(Module, not LocalOnly), %% We need to recover the source from the compilation %% info otherwise the newly compiled module will have %% source pointing to the current directory SourceInfo = get_source_info(Module, Beam), - %% Compile and load the result + %% Compile and load the result. %% It's necessary to check the result of loading since it may - %% fail, for example if Module resides in a sticky directory - {ok, Module, Binary} = compile:forms(Forms, SourceInfo ++ UserOptions), + %% fail, for example if Module resides in a sticky directory. + Options = SourceInfo ++ UserOptions, + {ok, Module, Binary} = compile:forms(Forms, Options), + case code:load_binary(Module, ?TAG, Binary) of {module, Module} -> - %% Store info about all function clauses in database + %% Store info about all function clauses in database. InitInfo = lists:reverse(Vars#vars.init_info), ets:insert(?COVER_CLAUSE_TABLE, {Module, InitInfo}), - %% Store binary code so it can be loaded on remote nodes + %% Store binary code so it can be loaded on remote nodes. ets:insert(?BINARY_TABLE, {Module, Binary}), {ok, Module}; @@ -1617,11 +1629,12 @@ get_compile_info(Module, Beam) -> [] end. -transform(Code, Module, MainFile) -> +transform(Code, Module, MainFile, LocalOnly) -> Vars0 = #vars{module=Module}, - {ok,MungedForms,Vars} = transform_2(Code,[],Vars0,MainFile,on), + {ok,MungedForms0,Vars} = transform_2(Code, [], Vars0, MainFile, on), + MungedForms = patch_code(Module, MungedForms0, LocalOnly), {MungedForms,Vars}. - + %% Helpfunction which returns the first found file-attribute, which can %% be interpreted as the name of the main erlang source file. find_main_filename([{attribute,_,file,{MainFile,_}}|_]) -> @@ -1788,19 +1801,7 @@ munge_body([Expr|Body], Vars, MungedBody, LastExprBumpLines) -> MungedExprs1 = [MungedExpr|MungedBody1], munge_body(Body, Vars3, MungedExprs1, NewBumps); false -> - ets:insert(?COVER_TABLE, {#bump{module = Vars#vars.module, - function = Vars#vars.function, - arity = Vars#vars.arity, - clause = Vars#vars.clause, - line = Line}, - 0}), Bump = bump_call(Vars, Line), -% Bump = {call, 0, {remote, 0, {atom,0,cover}, {atom,0,bump}}, -% [{atom, 0, Vars#vars.module}, -% {atom, 0, Vars#vars.function}, -% {integer, 0, Vars#vars.arity}, -% {integer, 0, Vars#vars.clause}, -% {integer, 0, Line}]}, Lines2 = [Line|Lines], {MungedExpr, Vars2} = munge_expr(Expr, Vars#vars{lines=Lines2}), NewBumps = new_bumps(Vars2, Vars), @@ -1855,8 +1856,10 @@ maybe_fix_last_expr(MungedExprs, Vars, LastExprBumpLines) -> last_expr_needs_fixing(Vars, LastExprBumpLines) -> case common_elems(Vars#vars.no_bump_lines, LastExprBumpLines) of - [Line] -> {yes, Line}; - _ -> no + [Line] -> + {yes, Line}; + _ -> + no end. fix_last_expr([MungedExpr|MungedExprs], Line, Vars) -> @@ -1921,9 +1924,7 @@ fix_cls([Cl | Cls], Line, Bump) -> bumps_line(E, L) -> try bumps_line1(E, L) catch true -> true end. -bumps_line1({call,_,{remote,_,{atom,_,ets},{atom,_,update_counter}}, - [{atom,_,?COVER_TABLE},{tuple,_,[_,_,_,_,_,{integer,_,Line}]},_]}, - Line) -> +bumps_line1({'BUMP',Line,_}, Line) -> throw(true); bumps_line1([E | Es], Line) -> bumps_line1(E, Line), @@ -1933,19 +1934,12 @@ bumps_line1(T, Line) when is_tuple(T) -> bumps_line1(_, _) -> false. -%%% End of fix of last expression. - +%% Insert a place holder for the call to counters:add/3 in the +%% abstract code. bump_call(Vars, Line) -> - A = erl_anno:new(0), - {call,A,{remote,A,{atom,A,ets},{atom,A,update_counter}}, - [{atom,A,?COVER_TABLE}, - {tuple,A,[{atom,A,?BUMP_REC_NAME}, - {atom,A,Vars#vars.module}, - {atom,A,Vars#vars.function}, - {integer,A,Vars#vars.arity}, - {integer,A,Vars#vars.clause}, - {integer,A,Line}]}, - {integer,A,1}]}. + {'BUMP',Line,counter_index(Vars, Line)}. + +%%% End of fix of last expression. munge_expr({match,Line,ExprL,ExprR}, Vars) -> {MungedExprL, Vars2} = munge_expr(ExprL, Vars), @@ -2105,6 +2099,159 @@ subtract(L1, L2) -> common_elems(L1, L2) -> [E || E <- L1, lists:member(E, L2)]. +%%%--Counters------------------------------------------------------------ + +init_counter_mapping(Mod) -> + true = ets:insert_new(?COVER_MAPPING_TABLE, {Mod,0}), + ok. + +counter_index(Vars, Line) -> + #vars{module=Mod,function=F,arity=A,clause=C} = Vars, + Key = #bump{module=Mod,function=F,arity=A, + clause=C,line=Line}, + case ets:lookup(?COVER_MAPPING_TABLE, Key) of + [] -> + Index = ets:update_counter(?COVER_MAPPING_TABLE, + Mod, {2,1}), + true = ets:insert(?COVER_MAPPING_TABLE, {Key,Index}), + Index; + [{Key,Index}] -> + Index + end. + +%% Create the counter array and store as a persistent term. +maybe_create_counters(Mod, true) -> + Cref = create_counters(Mod), + Key = {?MODULE,Mod}, + persistent_term:put(Key, Cref), + ok; +maybe_create_counters(_Mod, false) -> + ok. + +create_counters(Mod) -> + Size0 = ets:lookup_element(?COVER_MAPPING_TABLE, Mod, 2), + Size = max(1, Size0), %Size must not be 0. + Cref = counters:new(Size, [write_concurrency]), + ets:insert(?COVER_MAPPING_TABLE, {{counters,Mod},Cref}), + Cref. + +patch_code(Mod, Forms, false) -> + A = erl_anno:new(0), + AbstrKey = {tuple,A,[{atom,A,?MODULE},{atom,A,Mod}]}, + patch_code1(Forms, {distributed,AbstrKey}); +patch_code(Mod, Forms, true) -> + Cref = create_counters(Mod), + AbstrCref = cid_to_abstract(Cref), + patch_code1(Forms, {local_only,AbstrCref}). + +%% Go through the abstract code and replace 'BUMP' forms +%% with the actual code to increment the counters. +patch_code1({'BUMP',_Line,Index}, {distributed,AbstrKey}) -> + %% Replace with counters:add(persistent_term:get(Key), Index, 1). + %% This code will work on any node. + A = element(2, AbstrKey), + GetCref = {call,A,{remote,A,{atom,A,persistent_term},{atom,A,get}}, + [AbstrKey]}, + {call,A,{remote,A,{atom,A,counters},{atom,A,add}}, + [GetCref,{integer,A,Index},{integer,A,1}]}; +patch_code1({'BUMP',_Line,Index}, {local_only,AbstrCref}) -> + %% Replace with counters:add(Cref, Index, 1). This code + %% will only work on the local node. + A = element(2, AbstrCref), + {call,A,{remote,A,{atom,A,counters},{atom,A,add}}, + [AbstrCref,{integer,A,Index},{integer,A,1}]}; +patch_code1({clauses,Cs}, Key) -> + {clauses,[patch_code1(El, Key) || El <- Cs]}; +patch_code1([_|_]=List, Key) -> + [patch_code1(El, Key) || El <- List]; +patch_code1(Tuple, Key) when tuple_size(Tuple) >= 3 -> + Acc = [element(2, Tuple),element(1, Tuple)], + patch_code_tuple(3, tuple_size(Tuple), Tuple, Key, Acc); +patch_code1(Other, _Key) -> + Other. + +patch_code_tuple(I, Size, Tuple, Key, Acc) when I =< Size -> + El = patch_code1(element(I, Tuple), Key), + patch_code_tuple(I + 1, Size, Tuple, Key, [El|Acc]); +patch_code_tuple(_I, _Size, _Tuple, _Key, Acc) -> + list_to_tuple(lists:reverse(Acc)). + +%% Don't try this at home! Assumes knowledge of the internal +%% representation of a counter ref. +cid_to_abstract(Cref0) -> + A = erl_anno:new(0), + %% Disable dialyzer warning for breaking opacity. + Cref = binary_to_term(term_to_binary(Cref0)), + {write_concurrency,Ref} = Cref, + {tuple,A,[{atom,A,write_concurrency},{integer,A,Ref}]}. + +%% Called on the remote node. Collect and send counters to +%% the main node. Also zero the counters. +send_counters(Mod, CollectorPid) -> + Process = fun(Chunk) -> send_chunk(CollectorPid, Chunk) end, + move_counters(Mod, Process). + +%% Called on the main node. Collect the counters and consolidate +%% them into the collection table. Also zero the counters. +move_counters(Mod) -> + move_counters(Mod, fun insert_in_collection_table/1). + +move_counters(Mod, Process) -> + Pattern = {#bump{module=Mod,_='_'},'_'}, + Matches = ets:match_object(?COVER_MAPPING_TABLE, Pattern, ?CHUNK_SIZE), + Cref = get_counters_ref(Mod), + move_counters1(Matches, Cref, Process). + +move_counters1({Mappings,Continuation}, Cref, Process) -> + Move = fun({Key,Index}) -> + Count = counters:get(Cref, Index), + ok = counters:sub(Cref, Index, Count), + {Key,Count} + end, + Process(lists:map(Move, Mappings)), + move_counters1(ets:match_object(Continuation), Cref, Process); +move_counters1('$end_of_table', _Cref, _Process) -> + ok. + +counters_mapping_table(Mod) -> + Mapping = counters_mapping(Mod), + Cref = get_counters_ref(Mod), + #{size:=Size} = counters:info(Cref), + [{Mod,Size}|Mapping]. + +get_counters_ref(Mod) -> + ets:lookup_element(?COVER_MAPPING_TABLE, {counters,Mod}, 2). + +counters_mapping(Mod) -> + Pattern = {#bump{module=Mod,_='_'},'_'}, + ets:match_object(?COVER_MAPPING_TABLE, Pattern). + +clear_counters(Mod) -> + _ = persistent_term:erase({?MODULE,Mod}), + ets:delete(?COVER_MAPPING_TABLE, Mod), + Pattern = {#bump{module=Mod,_='_'},'_'}, + _ = ets:match_delete(?COVER_MAPPING_TABLE, Pattern), + ok. + +%% Reset counters (set counters to 0). +reset_counters(Mod) -> + Pattern = {#bump{module=Mod,_='_'},'$1'}, + MatchSpec = [{Pattern,[],['$1']}], + Matches = ets:select(?COVER_MAPPING_TABLE, + MatchSpec, ?CHUNK_SIZE), + Cref = get_counters_ref(Mod), + reset_counters1(Matches, Cref). + +reset_counters1({Indices,Continuation}, Cref) -> + _ = [counters:put(Cref, N, 0) || N <- Indices], + reset_counters1(ets:select(Continuation), Cref); +reset_counters1('$end_of_table', _Cref) -> + ok. + +delete_all_counters() -> + _ = [persistent_term:erase(Key) || {?MODULE,_}=Key <- persistent_term:get()], + ok. + %%%--Analysis------------------------------------------------------------ %% Collect data for all modules @@ -2140,20 +2287,7 @@ collect(Module,Clauses,Nodes) -> %% ?COLLECTION_TABLE. Resetting data in ?COVER_TABLE move_modules({Module,Clauses}) -> ets:insert(?COLLECTION_CLAUSE_TABLE,{Module,Clauses}), - Pattern = {#bump{module=Module, _='_'}, '_'}, - MatchSpec = [{Pattern,[],['$_']}], - Match = ets:select(?COVER_TABLE,MatchSpec,?CHUNK_SIZE), - do_move_module(Match). - -do_move_module({Bumps,Continuation}) -> - lists:foreach(fun({Key,Val}) -> - ets:insert(?COVER_TABLE, {Key,0}), - insert_in_collection_table(Key,Val) - end, - Bumps), - do_move_module(ets:select(Continuation)); -do_move_module('$end_of_table') -> - ok. + move_counters(Module). %% Given a .beam file, find the .erl file. Look first in same directory as %% the .beam file, then in ../src, then in compile info. @@ -2709,7 +2843,7 @@ get_term(Fd) -> %% Reset main node and all remote nodes do_reset_main_node(Module,Nodes) -> - do_reset(Module), + reset_counters(Module), do_reset_collection_table(Module), remote_reset(Module,Nodes). @@ -2717,27 +2851,9 @@ do_reset_collection_table(Module) -> ets:delete(?COLLECTION_CLAUSE_TABLE,Module), ets:match_delete(?COLLECTION_TABLE, {#bump{module=Module},'_'}). -%% do_reset(Module) -> ok -%% The reset is done on ?CHUNK_SIZE number of bumps to avoid building -%% long lists in the case of very large modules -do_reset(Module) -> - Pattern = {#bump{module=Module, _='_'}, '$1'}, - MatchSpec = [{Pattern,[{'=/=','$1',0}],['$_']}], - Match = ets:select(?COVER_TABLE,MatchSpec,?CHUNK_SIZE), - do_reset2(Match). - -do_reset2({Bumps,Continuation}) -> - lists:foreach(fun({Bump,_N}) -> - ets:insert(?COVER_TABLE, {Bump,0}) - end, - Bumps), - do_reset2(ets:select(Continuation)); -do_reset2('$end_of_table') -> - ok. - do_clear(Module) -> ets:match_delete(?COVER_CLAUSE_TABLE, {Module,'_'}), - ets:match_delete(?COVER_TABLE, {#bump{module=Module},'_'}), + clear_counters(Module), case lists:member(?COLLECTION_TABLE, ets:all()) of true -> %% We're on the main node diff --git a/lib/tools/src/tools.app.src b/lib/tools/src/tools.app.src index f8c6aa22cb..f0e0fc4bec 100644 --- a/lib/tools/src/tools.app.src +++ b/lib/tools/src/tools.app.src @@ -21,11 +21,13 @@ [{description, "DEVTOOLS CXC 138 16"}, {vsn, "%VSN%"}, {modules, [cover, + cprof, eprof, fprof, instrument, lcnt, make, + tags, xref, xref_base, xref_compiler, diff --git a/lib/tools/test/Makefile b/lib/tools/test/Makefile index 2b7b17afb3..7a0a941ccc 100644 --- a/lib/tools/test/Makefile +++ b/lib/tools/test/Makefile @@ -32,6 +32,7 @@ MODULES = \ make_SUITE \ tools_SUITE \ xref_SUITE \ + prof_bench_SUITE \ ignore_cores ERL_FILES= $(MODULES:%=%.erl) @@ -41,7 +42,7 @@ INSTALL_PROGS= $(TARGET_FILES) EMAKEFILE=Emakefile -SPEC_FILES= tools.spec +SPEC_FILES= tools.spec tools_bench.spec COVER_FILE = tools.cover # ---------------------------------------------------- diff --git a/lib/tools/test/cover_SUITE.erl b/lib/tools/test/cover_SUITE.erl index 161b0105b9..ee58fd7a10 100644 --- a/lib/tools/test/cover_SUITE.erl +++ b/lib/tools/test/cover_SUITE.erl @@ -24,7 +24,8 @@ -include_lib("common_test/include/ct.hrl"). suite() -> - [{ct_hooks,[ts_install_cth]}]. + [{ct_hooks,[ts_install_cth]}, + {timetrap,{minutes,5}}]. all() -> NoStartStop = [eif,otp_5305,otp_5418,otp_7095,otp_8273, @@ -35,7 +36,8 @@ all() -> distribution, reconnect, die_and_reconnect, dont_reconnect_after_stop, stop_node_after_disconnect, export_import, otp_5031, otp_6115, - otp_8270, otp_10979_hanging_node, otp_14817], + otp_8270, otp_10979_hanging_node, otp_14817, + local_only], case whereis(cover_server) of undefined -> [coverage,StartStop ++ NoStartStop]; @@ -1742,6 +1744,40 @@ otp_13289(Config) -> ok = file:delete(File), ok. +local_only(Config) -> + ok = file:set_cwd(proplists:get_value(data_dir, Config)), + + %% Trying restricting to local nodes too late. + cover:start(), + {ok,a} = cover:compile(a), + [a] = cover:modules(), + {error,too_late} = cover:local_only(), + cover:stop(), + + %% Now test local only mode. + cover:start(), + ok = cover:local_only(), + [] = cover:modules(), + {ok,a} = cover:compile(a), + [a] = cover:modules(), + done = a:start(5), + {ok, {a,{17,2}}} = cover:analyse(a, coverage, module), + {ok, [{{a,exit_kalle,0},{1,0}}, + {{a,loop,3},{5,1}}, + {{a,pong,1},{1,0}}, + {{a,start,1},{6,0}}, + {{a,stop,1},{0,1}}, + {{a,trycatch,1},{4,0}}]} = + cover:analyse(a, coverage, function), + + %% Make sure that it is not possible to run cover on + %% slave nodes. + {ok,Name} = test_server:start_node(?FUNCTION_NAME, slave, []), + {error,local_only} = cover:start([Name]), + test_server:stop_node(Name), + + ok. + %%--Auxiliary------------------------------------------------------------ analyse_expr(Expr, Config) -> diff --git a/lib/tools/test/emacs_SUITE.erl b/lib/tools/test/emacs_SUITE.erl index a6d43d1816..73270e6ed6 100644 --- a/lib/tools/test/emacs_SUITE.erl +++ b/lib/tools/test/emacs_SUITE.erl @@ -70,19 +70,20 @@ bif_highlight(Config) -> check_bif_highlight(Bin, Tag, Compare) -> - [_H,IntMatch,_T] = + [_H,Match,_T] = re:split(Bin,<<"defvar ",Tag/binary, "[^(]*\\(([^)]*)">>,[]), - EmacsIntBifs = [list_to_atom(S) || - S <- string:tokens(binary_to_list(IntMatch)," '\"\n")], + EmacsBifs = [list_to_atom(S) || + S <- string:tokens(binary_to_list(Match)," '\"\n")], - ct:log("Emacs ~p",[EmacsIntBifs]), - ct:log("Int ~p",[Compare]), + ct:log("Comparing ~s", [Tag]), + ct:log("Emacs ~p",[EmacsBifs]), + ct:log("Erlang ~p",[Compare]), - ct:log("Diff1 ~p",[Compare -- EmacsIntBifs]), - ct:log("Diff2 ~p",[EmacsIntBifs -- Compare]), - [] = Compare -- EmacsIntBifs, - [] = EmacsIntBifs -- Compare. + ct:log("Only in Erlang ~p",[Compare -- EmacsBifs]), + ct:log("Only in Emacs ~p",[EmacsBifs -- Compare]), + [] = Compare -- EmacsBifs, + [] = EmacsBifs -- Compare. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -118,7 +119,7 @@ compile_and_load(_Config) -> false -> " " end, emacs([Pedantic, - " -f batch-byte-compile ",filename:join(Dir, File)]), + " -f batch-byte-compile ", dquote(filename:join(Dir, File))]), true end, lists:foreach(Compile, Files), @@ -143,6 +144,10 @@ tests_compiled(_Config) -> ok end. + +dquote(Str) -> + "\"" ++ Str ++ "\"". + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% indent(Config) -> @@ -188,7 +193,9 @@ diff(Orig, File) -> end. emacs_version_ok(AcceptVer) -> - case os:cmd("emacs --version | head -1") of + VersionLine = os:cmd("emacs --version | head -1"), + io:format("~s~n", [VersionLine]), + case VersionLine of "GNU Emacs " ++ Ver -> case string:to_float(Ver) of {Vsn, _} when Vsn >= AcceptVer -> @@ -204,14 +211,14 @@ emacs_version_ok(AcceptVer) -> emacs(EmacsCmds) when is_list(EmacsCmds) -> Cmd = ["emacs ", "--batch --quick ", - "--directory ", emacs_dir(), " ", + "--directory ", dquote(emacs_dir()), " ", "--eval \"(require 'erlang-start)\" " | EmacsCmds], Res0 = os:cmd(Cmd ++ " ; echo $?"), Rows = string:lexemes(Res0, ["\r\n", $\n]), Res = lists:last(Rows), Output = string:join(lists:droplast(Rows), "\n"), - io:format("Cmd ~s:~n => ~s ~ts~n", [Cmd, Res, Output]), + io:format("Cmd ~ts:~n => ~s ~ts~n", [Cmd, Res, Output]), "0" = Res, Output. diff --git a/lib/tools/test/instrument_SUITE.erl b/lib/tools/test/instrument_SUITE.erl index 8c521b2e1a..f474669836 100644 --- a/lib/tools/test/instrument_SUITE.erl +++ b/lib/tools/test/instrument_SUITE.erl @@ -77,6 +77,8 @@ allocations_ramv(Config) when is_list(Config) -> verify_allocations_disabled(_AllocType, Result) -> verify_allocations_disabled(Result). +verify_allocations_disabled({ok, {_HistStart, _UnscannedBytes, Allocs}}) -> + true = Allocs =:= #{}; verify_allocations_disabled({error, not_enabled}) -> ok. @@ -91,6 +93,13 @@ verify_allocations_enabled(_AllocType, Result) -> verify_allocations_enabled({ok, {_HistStart, _UnscannedBytes, Allocs}}) -> true = Allocs =/= #{}. +verify_allocations_output(#{}, {ok, {_, _, Allocs}}) when Allocs =:= #{} -> + %% This happens when the allocator is enabled but tagging is disabled. If + %% there's an error that causes Allocs to always be empty when enabled it + %% will be caught by verify_allocations_enabled. + ok; +verify_allocations_output(#{}, {error, not_enabled}) -> + ok; verify_allocations_output(#{ histogram_start := HistStart, histogram_width := HistWidth }, {ok, {HistStart, _UnscannedBytes, ByOrigin}}) -> @@ -124,8 +133,6 @@ verify_allocations_output(#{ histogram_start := HistStart, [BlockCount, GenTotalBlockCount]) end, - ok; -verify_allocations_output(#{}, {error, not_enabled}) -> ok. %% %% %% %% %% %% @@ -214,7 +221,8 @@ verify_carriers_output(#{ histogram_start := HistStart, ct:fail("Carrier count is ~p, expected at least ~p (SBC).", [CarrierCount, GenSBCCount]); CarrierCount >= GenSBCCount -> - ok + ct:pal("Found ~p carriers, required at least ~p (SBC)." , + [CarrierCount, GenSBCCount]) end, ok; @@ -252,13 +260,18 @@ test_format(Options0, Gather, Verify) -> test_abort(Gather) -> %% There's no way for us to tell whether this actually aborted or ran to %% completion, but it might catch a few segfaults. + %% This testcase is mostly useful when run in an debug emulator as it needs + %% the modified reduction count to trigger the odd trap scenarios Runner = self(), Ref = make_ref(), spawn_opt(fun() -> - [Gather({Type, SchedId, 1, 1, Ref}) || - Type <- erlang:system_info(alloc_util_allocators), - SchedId <- lists:seq(0, erlang:system_info(schedulers))], - Runner ! Ref + [begin + Ref2 = make_ref(), + [Gather({Type, SchedId, 1, 1, Ref2}) || + Type <- erlang:system_info(alloc_util_allocators), + SchedId <- lists:seq(0, erlang:system_info(schedulers))] + end || _ <- lists:seq(1,100)], + Runner ! Ref end, [{priority, max}]), receive Ref -> ok @@ -292,9 +305,19 @@ start_slave(Args) -> MicroSecs = erlang:monotonic_time(), Name = "instr" ++ integer_to_list(MicroSecs), Pa = filename:dirname(code:which(?MODULE)), - {ok, Node} = test_server:start_node(list_to_atom(Name), - slave, - [{args, "-pa " ++ Pa ++ " " ++ Args}]), + + %% We pass arguments through ZFLAGS as the nightly tests rotate + %% +Meamax/+Meamin which breaks the _enabled and _disabled tests unless + %% overridden. + ZFlags = os:getenv("ERL_ZFLAGS", ""), + {ok, Node} = try + os:putenv("ERL_ZFLAGS", ZFlags ++ [" " | Args]), + test_server:start_node(list_to_atom(Name), + slave, + [{args, "-pa " ++ Pa}]) + after + os:putenv("ERL_ZFLAGS", ZFlags) + end, Node. generate_test_blocks() -> @@ -309,8 +332,9 @@ generate_test_blocks() -> MBCs = [<<I, 0:64/unit:8>> || I <- lists:seq(1, ?GENERATED_MBC_BLOCK_COUNT)], Runner ! Ref, - receive after infinity -> ok end, - unreachable ! {SBCs, MBCs} + receive + gurka -> gaffel ! {SBCs, MBCs} + end end), receive Ref -> ok diff --git a/lib/tools/test/prof_bench_SUITE.erl b/lib/tools/test/prof_bench_SUITE.erl new file mode 100644 index 0000000000..50d0ba9cd9 --- /dev/null +++ b/lib/tools/test/prof_bench_SUITE.erl @@ -0,0 +1,126 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2001-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(prof_bench_SUITE). + +-include_lib("common_test/include/ct_event.hrl"). + +%% Test server framework exports +-export([all/0, suite/0, init_per_suite/1, end_per_suite/1]). + +-export([overhead/1]). + +%%%--------------------------------------------------------------------- +%%% Test suites +%%%--------------------------------------------------------------------- + + +suite() -> + [{timetrap,{minutes,10}}]. + +all() -> + [overhead]. + +init_per_suite(Config) -> + case {test_server:is_native(fprof_SUITE) or + (lists:any(fun(M) -> test_server:is_native(M) end, modules())) or + (whereis(cover_server) =/= undefined), + erlang:system_info(wordsize)} + of + {true, _} -> {skip, "Native or cover code"}; + {_, 4} -> {skip, "Can't run on 32-bit as files will be large"}; + {false, 8} -> Config + end. + +end_per_suite(Config) -> + LogFile = filename:join(proplists:get_value(priv_dir, Config), "fprof.trace"), + file:delete(LogFile), + ok. + +%%%--------------------------------------------------------------------- + +%% ct:run_test([{suite, prof_bench_SUITE}]). +overhead(Config) -> + LogFile = filename:join(proplists:get_value(priv_dir, Config), "fprof.trace"), + SofsCopy = filename:join(proplists:get_value(data_dir, Config), "sofs_copy.erl"), + TC = fun() -> compile:file(SofsCopy, [binary]) end, + _Warmup = timer:tc(TC), + + {NormTime,{ok, sofs_copy, _}} = timer:tc(TC), + {FProfTime,{ok,sofs_copy,_}} = fprof:apply(timer, tc, [TC], [{file, LogFile}]), + ct:pal("FProf: ~p Norm: ~p Ratio: ~p",[FProfTime, NormTime, NormTime / FProfTime * 100]), + {ok,{EProfTime,{ok,sofs_copy,_}}} = eprof:profile([], timer, tc, [TC]), + ct:pal("EProf: ~p Norm: ~p Ratio: ~p",[EProfTime, NormTime, NormTime / EProfTime * 100]), + {CProfTime,{ok,sofs_copy,_}} = cprof_apply(timer, tc, [TC]), + ct:pal("CProf: ~p Norm: ~p Ratio: ~p",[CProfTime, NormTime, NormTime / CProfTime * 100]), + {CoverTime,{ok,sofs_copy,_}} = cover_apply(timer, tc, [TC]), + ct:pal("Cover: ~p Norm: ~p Ratio: ~p",[CoverTime, NormTime, NormTime / CoverTime * 100]), + + ct_event:notify(#event{name = benchmark_data, + data = [{name, fprof_overhead}, + {value, NormTime / FProfTime * 100}]}), + ct_event:notify(#event{name = benchmark_data, + data = [{name, eprof_overhead}, + {value, NormTime / EProfTime * 100}]}), + ct_event:notify(#event{name = benchmark_data, + data = [{name, cprof_overhead}, + {value, NormTime / CProfTime * 100}]}), + ct_event:notify(#event{name = benchmark_data, + data = [{name, cover_overhead}, + {value, NormTime / CoverTime * 100}]}). + +%% overhead(Config) -> +%% LogFile = filename:join(proplists:get_value(priv_dir, Config), "fprof.trace"), +%% SofsCopy = filename:join(proplists:get_value(data_dir, Config), "sofs_copy.erl"), +%% TC = fun() -> compile:file(SofsCopy, [binary]) end, +%% _Warmup = timer:tc(TC), + +%% [{ok,{EProfTime,{ok,sofs_copy,_}}} = eprof:profile([], timer, tc, [TC]) +%% || _ <- lists:seq(1,10)], +%% %% [fprof:apply(timer, tc, [TC], [{file, LogFile}]) || _ <- lists:seq(1,10)], +%% {FProfTime,{ok,sofs_copy,_}} = fprof:apply(timer, tc, [TC], [{file, LogFile}]), +%% {NormTime,{ok, sofs_copy, _}} = timer:tc(TC), + + %% ct:pal("FProf: ~p Norm: ~p Ratio: ~p",[FProfTime, NormTime, FProfTime / NormTime]). + +cprof_apply(M, F, A) -> + cprof:start(), + Res = apply(M, F, A), + cprof:stop(), + Res. + +cover_apply(M, F, A) -> + cover:start(), + catch cover:local_only(), + Modules = modules(), + [code:unstick_mod(Mod) || Mod <- Modules], + cover:compile_beam(Modules), + [code:stick_mod(Mod) || Mod <- Modules], + Res = apply(M, F, A), + cover:stop(), + Res. + +modules() -> + application:load(compiler), + {ok, CompilerModules} = application:get_key(compiler, modules), + %% Only cover compile a subset of the stdlib modules + StdlibModules = [erl_parse, erl_expand_records, erl_lint, gb_trees, gb_sets, sofs, + beam_lib, dict, epp, erl_anno, erl_bits, + orddict, ordsets, sets, string, unicode, unicode_util], + CompilerModules ++ StdlibModules. diff --git a/lib/tools/test/prof_bench_SUITE_data/sofs_copy.erl b/lib/tools/test/prof_bench_SUITE_data/sofs_copy.erl new file mode 100644 index 0000000000..2a9b19177e --- /dev/null +++ b/lib/tools/test/prof_bench_SUITE_data/sofs_copy.erl @@ -0,0 +1,2809 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2001-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(sofs_copy). + +-export([from_term/1, from_term/2, from_external/2, empty_set/0, + is_type/1, set/1, set/2, from_sets/1, relation/1, relation/2, + a_function/1, a_function/2, family/1, family/2, + to_external/1, type/1, to_sets/1, no_elements/1, + specification/2, union/2, intersection/2, difference/2, + symdiff/2, symmetric_partition/2, product/1, product/2, + constant_function/2, is_equal/2, is_subset/2, is_sofs_set/1, + is_set/1, is_empty_set/1, is_disjoint/2]). + +-export([union/1, intersection/1, canonical_relation/1]). + +-export([relation_to_family/1, domain/1, range/1, field/1, + relative_product/1, relative_product/2, relative_product1/2, + converse/1, image/2, inverse_image/2, strict_relation/1, + weak_relation/1, extension/3, is_a_function/1]). + +-export([composite/2, inverse/1]). + +-export([restriction/2, restriction/3, drestriction/2, drestriction/3, + substitution/2, projection/2, partition/1, partition/2, + partition/3, multiple_relative_product/2, join/4]). + +-export([family_to_relation/1, family_specification/2, + union_of_family/1, intersection_of_family/1, + family_union/1, family_intersection/1, + family_domain/1, family_range/1, family_field/1, + family_union/2, family_intersection/2, family_difference/2, + partition_family/2, family_projection/2]). + +-export([family_to_digraph/1, family_to_digraph/2, + digraph_to_family/1, digraph_to_family/2]). + +%% Shorter names of some functions. +-export([fam2rel/1, rel2fam/1]). + +-import(lists, + [any/2, append/1, flatten/1, foreach/2, + keysort/2, last/1, map/2, mapfoldl/3, member/2, merge/2, + reverse/1, reverse/2, sort/1, umerge/1, umerge/2, usort/1]). + +-compile({inline, [{family_to_relation,1}, {relation_to_family,1}]}). + +-compile({inline, [{rel,2},{a_func,2},{fam,2},{term2set,2}]}). + +-compile({inline, [{external_fun,1},{element_type,1}]}). + +-compile({inline, + [{unify_types,2}, {match_types,2}, + {test_rel,3}, {symdiff,3}, + {subst,3}]}). + +-compile({inline, [{fam_binop,3}]}). + +%% Nope, no is_member, del_member or add_member. +%% +%% See also "Naive Set Theory" by Paul R. Halmos. +%% +%% By convention, erlang:error/1 is called from exported functions. + +-define(TAG, 'Set'). +-define(ORDTAG, 'OrdSet'). + +-record(?TAG, {data = [] :: list(), type = type :: term()}). +-record(?ORDTAG, {orddata = {} :: tuple() | atom(), + ordtype = type :: term()}). + +-define(LIST(S), (S)#?TAG.data). +-define(TYPE(S), (S)#?TAG.type). +-define(SET(L, T), #?TAG{data = L, type = T}). +-define(IS_SET(S), is_record(S, ?TAG)). +-define(IS_UNTYPED_SET(S), ?TYPE(S) =:= ?ANYTYPE). + +%% Ordered sets and atoms: +-define(ORDDATA(S), (S)#?ORDTAG.orddata). +-define(ORDTYPE(S), (S)#?ORDTAG.ordtype). +-define(ORDSET(L, T), #?ORDTAG{orddata = L, ordtype = T}). +-define(IS_ORDSET(S), is_record(S, ?ORDTAG)). +-define(ATOM_TYPE, atom). +-define(IS_ATOM_TYPE(T), is_atom(T)). % true for ?ANYTYPE... + +%% When IS_SET is true: +-define(ANYTYPE, '_'). +-define(BINREL(X, Y), {X, Y}). +-define(IS_RELATION(R), is_tuple(R)). +-define(REL_ARITY(R), tuple_size(R)). +-define(REL_TYPE(I, R), element(I, R)). +-define(SET_OF(X), [X]). +-define(IS_SET_OF(X), is_list(X)). +-define(FAMILY(X, Y), ?BINREL(X, ?SET_OF(Y))). + +-export_type([anyset/0, binary_relation/0, external_set/0, a_function/0, + family/0, relation/0, set_of_sets/0, set_fun/0, spec_fun/0, + type/0]). +-export_type([ordset/0, a_set/0]). + +-type(anyset() :: ordset() | a_set()). +-type(binary_relation() :: relation()). +-type(external_set() :: term()). +-type(a_function() :: relation()). +-type(family() :: a_function()). +-opaque(ordset() :: #?ORDTAG{}). +-type(relation() :: a_set()). +-opaque(a_set() :: #?TAG{}). +-type(set_of_sets() :: a_set()). +-type(set_fun() :: pos_integer() + | {external, fun((external_set()) -> external_set())} + | fun((anyset()) -> anyset())). +-type(spec_fun() :: {external, fun((external_set()) -> boolean())} + | fun((anyset()) -> boolean())). +-type(type() :: term()). + +-type(tuple_of(_T) :: tuple()). + +%% +%% Exported functions +%% + +%%% +%%% Create sets +%%% + +-spec(from_term(Term) -> AnySet when + AnySet :: anyset(), + Term :: term()). +from_term(T) -> + Type = case T of + _ when is_list(T) -> [?ANYTYPE]; + _ -> ?ANYTYPE + end, + try setify(T, Type) + catch _:_ -> erlang:error(badarg) + end. + +-spec(from_term(Term, Type) -> AnySet when + AnySet :: anyset(), + Term :: term(), + Type :: type()). +from_term(L, T) -> + case is_type(T) of + true -> + try setify(L, T) + catch _:_ -> erlang:error(badarg) + end; + false -> + erlang:error(badarg) + end. + +-spec(from_external(ExternalSet, Type) -> AnySet when + ExternalSet :: external_set(), + AnySet :: anyset(), + Type :: type()). +from_external(L, ?SET_OF(Type)) -> + ?SET(L, Type); +from_external(T, Type) -> + ?ORDSET(T, Type). + +-spec(empty_set() -> Set when + Set :: a_set()). +empty_set() -> + ?SET([], ?ANYTYPE). + +-spec(is_type(Term) -> Bool when + Bool :: boolean(), + Term :: term()). +is_type(Atom) when ?IS_ATOM_TYPE(Atom), Atom =/= ?ANYTYPE -> + true; +is_type(?SET_OF(T)) -> + is_element_type(T); +is_type(T) when tuple_size(T) > 0 -> + is_types(tuple_size(T), T); +is_type(_T) -> + false. + +-spec(set(Terms) -> Set when + Set :: a_set(), + Terms :: [term()]). +set(L) -> + try usort(L) of + SL -> ?SET(SL, ?ATOM_TYPE) + catch _:_ -> erlang:error(badarg) + end. + +-spec(set(Terms, Type) -> Set when + Set :: a_set(), + Terms :: [term()], + Type :: type()). +set(L, ?SET_OF(Type)) when ?IS_ATOM_TYPE(Type), Type =/= ?ANYTYPE -> + try usort(L) of + SL -> ?SET(SL, Type) + catch _:_ -> erlang:error(badarg) + end; +set(L, ?SET_OF(_) = T) -> + try setify(L, T) + catch _:_ -> erlang:error(badarg) + end; +set(_, _) -> + erlang:error(badarg). + +-spec(from_sets(ListOfSets) -> Set when + Set :: a_set(), + ListOfSets :: [anyset()]; + (TupleOfSets) -> Ordset when + Ordset :: ordset(), + TupleOfSets :: tuple_of(anyset())). +from_sets(Ss) when is_list(Ss) -> + case set_of_sets(Ss, [], ?ANYTYPE) of + {error, Error} -> + erlang:error(Error); + Set -> + Set + end; +from_sets(Tuple) when is_tuple(Tuple) -> + case ordset_of_sets(tuple_to_list(Tuple), [], []) of + error -> + erlang:error(badarg); + Set -> + Set + end; +from_sets(_) -> + erlang:error(badarg). + +-spec(relation(Tuples) -> Relation when + Relation :: relation(), + Tuples :: [tuple()]). +relation([]) -> + ?SET([], ?BINREL(?ATOM_TYPE, ?ATOM_TYPE)); +relation(Ts = [T | _]) when is_tuple(T) -> + try rel(Ts, tuple_size(T)) + catch _:_ -> erlang:error(badarg) + end; +relation(_) -> + erlang:error(badarg). + +-spec(relation(Tuples, Type) -> Relation when + N :: integer(), + Type :: N | type(), + Relation :: relation(), + Tuples :: [tuple()]). +relation(Ts, TS) -> + try rel(Ts, TS) + catch _:_ -> erlang:error(badarg) + end. + +-spec(a_function(Tuples) -> Function when + Function :: a_function(), + Tuples :: [tuple()]). +a_function(Ts) -> + try func(Ts, ?BINREL(?ATOM_TYPE, ?ATOM_TYPE)) of + Bad when is_atom(Bad) -> + erlang:error(Bad); + Set -> + Set + catch _:_ -> erlang:error(badarg) + end. + +-spec(a_function(Tuples, Type) -> Function when + Function :: a_function(), + Tuples :: [tuple()], + Type :: type()). +a_function(Ts, T) -> + try a_func(Ts, T) of + Bad when is_atom(Bad) -> + erlang:error(Bad); + Set -> + Set + catch _:_ -> erlang:error(badarg) + end. + +-spec(family(Tuples) -> Family when + Family :: family(), + Tuples :: [tuple()]). +family(Ts) -> + try fam2(Ts, ?FAMILY(?ATOM_TYPE, ?ATOM_TYPE)) of + Bad when is_atom(Bad) -> + erlang:error(Bad); + Set -> + Set + catch _:_ -> erlang:error(badarg) + end. + +-spec(family(Tuples, Type) -> Family when + Family :: family(), + Tuples :: [tuple()], + Type :: type()). +family(Ts, T) -> + try fam(Ts, T) of + Bad when is_atom(Bad) -> + erlang:error(Bad); + Set -> + Set + catch _:_ -> erlang:error(badarg) + end. + +%%% +%%% Functions on sets. +%%% + +-spec(to_external(AnySet) -> ExternalSet when + ExternalSet :: external_set(), + AnySet :: anyset()). +to_external(S) when ?IS_SET(S) -> + ?LIST(S); +to_external(S) when ?IS_ORDSET(S) -> + ?ORDDATA(S). + +-spec(type(AnySet) -> Type when + AnySet :: anyset(), + Type :: type()). +type(S) when ?IS_SET(S) -> + ?SET_OF(?TYPE(S)); +type(S) when ?IS_ORDSET(S) -> + ?ORDTYPE(S). + +-spec(to_sets(ASet) -> Sets when + ASet :: a_set() | ordset(), + Sets :: tuple_of(AnySet) | [AnySet], + AnySet :: anyset()). +to_sets(S) when ?IS_SET(S) -> + case ?TYPE(S) of + ?SET_OF(Type) -> list_of_sets(?LIST(S), Type, []); + Type -> list_of_ordsets(?LIST(S), Type, []) + end; +to_sets(S) when ?IS_ORDSET(S), is_tuple(?ORDTYPE(S)) -> + tuple_of_sets(tuple_to_list(?ORDDATA(S)), tuple_to_list(?ORDTYPE(S)), []); +to_sets(S) when ?IS_ORDSET(S) -> + erlang:error(badarg). + +-spec(no_elements(ASet) -> NoElements when + ASet :: a_set() | ordset(), + NoElements :: non_neg_integer()). +no_elements(S) when ?IS_SET(S) -> + length(?LIST(S)); +no_elements(S) when ?IS_ORDSET(S), is_tuple(?ORDTYPE(S)) -> + tuple_size(?ORDDATA(S)); +no_elements(S) when ?IS_ORDSET(S) -> + erlang:error(badarg). + +-spec(specification(Fun, Set1) -> Set2 when + Fun :: spec_fun(), + Set1 :: a_set(), + Set2 :: a_set()). +specification(Fun, S) when ?IS_SET(S) -> + Type = ?TYPE(S), + R = case external_fun(Fun) of + false -> + spec(?LIST(S), Fun, element_type(Type), []); + XFun -> + specification(?LIST(S), XFun, []) + end, + case R of + SL when is_list(SL) -> + ?SET(SL, Type); + Bad -> + erlang:error(Bad) + end. + +-spec(union(Set1, Set2) -> Set3 when + Set1 :: a_set(), + Set2 :: a_set(), + Set3 :: a_set()). +union(S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + case unify_types(?TYPE(S1), ?TYPE(S2)) of + [] -> erlang:error(type_mismatch); + Type -> ?SET(umerge(?LIST(S1), ?LIST(S2)), Type) + end. + +-spec(intersection(Set1, Set2) -> Set3 when + Set1 :: a_set(), + Set2 :: a_set(), + Set3 :: a_set()). +intersection(S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + case unify_types(?TYPE(S1), ?TYPE(S2)) of + [] -> erlang:error(type_mismatch); + Type -> ?SET(intersection(?LIST(S1), ?LIST(S2), []), Type) + end. + +-spec(difference(Set1, Set2) -> Set3 when + Set1 :: a_set(), + Set2 :: a_set(), + Set3 :: a_set()). +difference(S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + case unify_types(?TYPE(S1), ?TYPE(S2)) of + [] -> erlang:error(type_mismatch); + Type -> ?SET(difference(?LIST(S1), ?LIST(S2), []), Type) + end. + +-spec(symdiff(Set1, Set2) -> Set3 when + Set1 :: a_set(), + Set2 :: a_set(), + Set3 :: a_set()). +symdiff(S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + case unify_types(?TYPE(S1), ?TYPE(S2)) of + [] -> erlang:error(type_mismatch); + Type -> ?SET(symdiff(?LIST(S1), ?LIST(S2), []), Type) + end. + +-spec(symmetric_partition(Set1, Set2) -> {Set3, Set4, Set5} when + Set1 :: a_set(), + Set2 :: a_set(), + Set3 :: a_set(), + Set4 :: a_set(), + Set5 :: a_set()). +symmetric_partition(S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + case unify_types(?TYPE(S1), ?TYPE(S2)) of + [] -> erlang:error(type_mismatch); + Type -> sympart(?LIST(S1), ?LIST(S2), [], [], [], Type) + end. + +-spec(product(Set1, Set2) -> BinRel when + BinRel :: binary_relation(), + Set1 :: a_set(), + Set2 :: a_set()). +product(S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + if + ?TYPE(S1) =:= ?ANYTYPE -> S1; + ?TYPE(S2) =:= ?ANYTYPE -> S2; + true -> + F = fun(E) -> {0, E} end, + T = ?BINREL(?TYPE(S1), ?TYPE(S2)), + ?SET(relprod(map(F, ?LIST(S1)), map(F, ?LIST(S2))), T) + end. + +-spec(product(TupleOfSets) -> Relation when + Relation :: relation(), + TupleOfSets :: tuple_of(a_set())). +product({S1, S2}) -> + product(S1, S2); +product(T) when is_tuple(T) -> + Ss = tuple_to_list(T), + try sets_to_list(Ss) of + [] -> + erlang:error(badarg); + L -> + Type = types(Ss, []), + case member([], L) of + true -> + empty_set(); + false -> + ?SET(reverse(prod(L, [], [])), Type) + end + catch _:_ -> erlang:error(badarg) + end. + +-spec(constant_function(Set, AnySet) -> Function when + AnySet :: anyset(), + Function :: a_function(), + Set :: a_set()). +constant_function(S, E) when ?IS_SET(S) -> + case {?TYPE(S), is_sofs_set(E)} of + {?ANYTYPE, true} -> S; + {Type, true} -> + NType = ?BINREL(Type, type(E)), + ?SET(constant_function(?LIST(S), to_external(E), []), NType); + _ -> erlang:error(badarg) + end; +constant_function(S, _) when ?IS_ORDSET(S) -> + erlang:error(badarg). + +-spec(is_equal(AnySet1, AnySet2) -> Bool when + AnySet1 :: anyset(), + AnySet2 :: anyset(), + Bool :: boolean()). +is_equal(S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + case match_types(?TYPE(S1), ?TYPE(S2)) of + true -> ?LIST(S1) == ?LIST(S2); + false -> erlang:error(type_mismatch) + end; +is_equal(S1, S2) when ?IS_ORDSET(S1), ?IS_ORDSET(S2) -> + case match_types(?ORDTYPE(S1), ?ORDTYPE(S2)) of + true -> ?ORDDATA(S1) == ?ORDDATA(S2); + false -> erlang:error(type_mismatch) + end; +is_equal(S1, S2) when ?IS_SET(S1), ?IS_ORDSET(S2) -> + erlang:error(type_mismatch); +is_equal(S1, S2) when ?IS_ORDSET(S1), ?IS_SET(S2) -> + erlang:error(type_mismatch). + +-spec(is_subset(Set1, Set2) -> Bool when + Bool :: boolean(), + Set1 :: a_set(), + Set2 :: a_set()). +is_subset(S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + case match_types(?TYPE(S1), ?TYPE(S2)) of + true -> subset(?LIST(S1), ?LIST(S2)); + false -> erlang:error(type_mismatch) + end. + +-spec(is_sofs_set(Term) -> Bool when + Bool :: boolean(), + Term :: term()). +is_sofs_set(S) when ?IS_SET(S) -> + true; +is_sofs_set(S) when ?IS_ORDSET(S) -> + true; +is_sofs_set(_S) -> + false. + +-spec(is_set(AnySet) -> Bool when + AnySet :: anyset(), + Bool :: boolean()). +is_set(S) when ?IS_SET(S) -> + true; +is_set(S) when ?IS_ORDSET(S) -> + false. + +-spec(is_empty_set(AnySet) -> Bool when + AnySet :: anyset(), + Bool :: boolean()). +is_empty_set(S) when ?IS_SET(S) -> + ?LIST(S) =:= []; +is_empty_set(S) when ?IS_ORDSET(S) -> + false. + +-spec(is_disjoint(Set1, Set2) -> Bool when + Bool :: boolean(), + Set1 :: a_set(), + Set2 :: a_set()). +is_disjoint(S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + case match_types(?TYPE(S1), ?TYPE(S2)) of + true -> + case ?LIST(S1) of + [] -> true; + [A | As] -> disjoint(?LIST(S2), A, As) + end; + false -> erlang:error(type_mismatch) + end. + +%%% +%%% Functions on set-of-sets. +%%% + +-spec(union(SetOfSets) -> Set when + Set :: a_set(), + SetOfSets :: set_of_sets()). +union(Sets) when ?IS_SET(Sets) -> + case ?TYPE(Sets) of + ?SET_OF(Type) -> ?SET(lunion(?LIST(Sets)), Type); + ?ANYTYPE -> Sets; + _ -> erlang:error(badarg) + end. + +-spec(intersection(SetOfSets) -> Set when + Set :: a_set(), + SetOfSets :: set_of_sets()). +intersection(Sets) when ?IS_SET(Sets) -> + case ?LIST(Sets) of + [] -> erlang:error(badarg); + [L | Ls] -> + case ?TYPE(Sets) of + ?SET_OF(Type) -> + ?SET(lintersection(Ls, L), Type); + _ -> erlang:error(badarg) + end + end. + +-spec(canonical_relation(SetOfSets) -> BinRel when + BinRel :: binary_relation(), + SetOfSets :: set_of_sets()). +canonical_relation(Sets) when ?IS_SET(Sets) -> + ST = ?TYPE(Sets), + case ST of + ?SET_OF(?ANYTYPE) -> empty_set(); + ?SET_OF(Type) -> + ?SET(can_rel(?LIST(Sets), []), ?BINREL(Type, ST)); + ?ANYTYPE -> Sets; + _ -> erlang:error(badarg) + end. + +%%% +%%% Functions on binary relations only. +%%% + +-spec(rel2fam(BinRel) -> Family when + Family :: family(), + BinRel :: binary_relation()). +rel2fam(R) -> + relation_to_family(R). + +-spec(relation_to_family(BinRel) -> Family when + Family :: family(), + BinRel :: binary_relation()). +%% Inlined. +relation_to_family(R) when ?IS_SET(R) -> + case ?TYPE(R) of + ?BINREL(DT, RT) -> + ?SET(rel2family(?LIST(R)), ?FAMILY(DT, RT)); + ?ANYTYPE -> R; + _Else -> erlang:error(badarg) + end. + +-spec(domain(BinRel) -> Set when + BinRel :: binary_relation(), + Set :: a_set()). +domain(R) when ?IS_SET(R) -> + case ?TYPE(R) of + ?BINREL(DT, _) -> ?SET(dom(?LIST(R)), DT); + ?ANYTYPE -> R; + _Else -> erlang:error(badarg) + end. + +-spec(range(BinRel) -> Set when + BinRel :: binary_relation(), + Set :: a_set()). +range(R) when ?IS_SET(R) -> + case ?TYPE(R) of + ?BINREL(_, RT) -> ?SET(ran(?LIST(R), []), RT); + ?ANYTYPE -> R; + _ -> erlang:error(badarg) + end. + +-spec(field(BinRel) -> Set when + BinRel :: binary_relation(), + Set :: a_set()). +%% In "Introduction to LOGIC", Suppes defines the field of a binary +%% relation to be the union of the domain and the range (or +%% counterdomain). +field(R) -> + union(domain(R), range(R)). + +-spec(relative_product(ListOfBinRels) -> BinRel2 when + ListOfBinRels :: [BinRel, ...], + BinRel :: binary_relation(), + BinRel2 :: binary_relation()). +%% The following clause is kept for backward compatibility. +%% The list is due to Dialyzer's specs. +relative_product(RT) when is_tuple(RT) -> + relative_product(tuple_to_list(RT)); +relative_product(RL) when is_list(RL) -> + case relprod_n(RL, foo, false, false) of + {error, Reason} -> + erlang:error(Reason); + Reply -> + Reply + end. + +-spec(relative_product(ListOfBinRels, BinRel1) -> BinRel2 when + ListOfBinRels :: [BinRel, ...], + BinRel :: binary_relation(), + BinRel1 :: binary_relation(), + BinRel2 :: binary_relation(); + (BinRel1, BinRel2) -> BinRel3 when + BinRel1 :: binary_relation(), + BinRel2 :: binary_relation(), + BinRel3 :: binary_relation()). +relative_product(R1, R2) when ?IS_SET(R1), ?IS_SET(R2) -> + relative_product1(converse(R1), R2); +%% The following clause is kept for backward compatibility. +%% The list is due to Dialyzer's specs. +relative_product(RT, R) when is_tuple(RT), ?IS_SET(R) -> + relative_product(tuple_to_list(RT), R); +relative_product(RL, R) when is_list(RL), ?IS_SET(R) -> + EmptyR = case ?TYPE(R) of + ?BINREL(_, _) -> ?LIST(R) =:= []; + ?ANYTYPE -> true; + _ -> erlang:error(badarg) + end, + case relprod_n(RL, R, EmptyR, true) of + {error, Reason} -> + erlang:error(Reason); + Reply -> + Reply + end. + +-spec(relative_product1(BinRel1, BinRel2) -> BinRel3 when + BinRel1 :: binary_relation(), + BinRel2 :: binary_relation(), + BinRel3 :: binary_relation()). +relative_product1(R1, R2) when ?IS_SET(R1), ?IS_SET(R2) -> + {DTR1, RTR1} = case ?TYPE(R1) of + ?BINREL(_, _) = R1T -> R1T; + ?ANYTYPE -> {?ANYTYPE, ?ANYTYPE}; + _ -> erlang:error(badarg) + end, + {DTR2, RTR2} = case ?TYPE(R2) of + ?BINREL(_, _) = R2T -> R2T; + ?ANYTYPE -> {?ANYTYPE, ?ANYTYPE}; + _ -> erlang:error(badarg) + end, + case match_types(DTR1, DTR2) of + true when DTR1 =:= ?ANYTYPE -> R1; + true when DTR2 =:= ?ANYTYPE -> R2; + true -> ?SET(relprod(?LIST(R1), ?LIST(R2)), ?BINREL(RTR1, RTR2)); + false -> erlang:error(type_mismatch) + end. + +-spec(converse(BinRel1) -> BinRel2 when + BinRel1 :: binary_relation(), + BinRel2 :: binary_relation()). +converse(R) when ?IS_SET(R) -> + case ?TYPE(R) of + ?BINREL(DT, RT) -> ?SET(converse(?LIST(R), []), ?BINREL(RT, DT)); + ?ANYTYPE -> R; + _ -> erlang:error(badarg) + end. + +-spec(image(BinRel, Set1) -> Set2 when + BinRel :: binary_relation(), + Set1 :: a_set(), + Set2 :: a_set()). +image(R, S) when ?IS_SET(R), ?IS_SET(S) -> + case ?TYPE(R) of + ?BINREL(DT, RT) -> + case match_types(DT, ?TYPE(S)) of + true -> + ?SET(usort(restrict(?LIST(S), ?LIST(R))), RT); + false -> + erlang:error(type_mismatch) + end; + ?ANYTYPE -> R; + _ -> erlang:error(badarg) + end. + +-spec(inverse_image(BinRel, Set1) -> Set2 when + BinRel :: binary_relation(), + Set1 :: a_set(), + Set2 :: a_set()). +inverse_image(R, S) when ?IS_SET(R), ?IS_SET(S) -> + case ?TYPE(R) of + ?BINREL(DT, RT) -> + case match_types(RT, ?TYPE(S)) of + true -> + NL = restrict(?LIST(S), converse(?LIST(R), [])), + ?SET(usort(NL), DT); + false -> + erlang:error(type_mismatch) + end; + ?ANYTYPE -> R; + _ -> erlang:error(badarg) + end. + +-spec(strict_relation(BinRel1) -> BinRel2 when + BinRel1 :: binary_relation(), + BinRel2 :: binary_relation()). +strict_relation(R) when ?IS_SET(R) -> + case ?TYPE(R) of + Type = ?BINREL(_, _) -> + ?SET(strict(?LIST(R), []), Type); + ?ANYTYPE -> R; + _ -> erlang:error(badarg) + end. + +-spec(weak_relation(BinRel1) -> BinRel2 when + BinRel1 :: binary_relation(), + BinRel2 :: binary_relation()). +weak_relation(R) when ?IS_SET(R) -> + case ?TYPE(R) of + ?BINREL(DT, RT) -> + case unify_types(DT, RT) of + [] -> + erlang:error(badarg); + Type -> + ?SET(weak(?LIST(R)), ?BINREL(Type, Type)) + end; + ?ANYTYPE -> R; + _ -> erlang:error(badarg) + end. + +-spec(extension(BinRel1, Set, AnySet) -> BinRel2 when + AnySet :: anyset(), + BinRel1 :: binary_relation(), + BinRel2 :: binary_relation(), + Set :: a_set()). +extension(R, S, E) when ?IS_SET(R), ?IS_SET(S) -> + case {?TYPE(R), ?TYPE(S), is_sofs_set(E)} of + {T=?BINREL(DT, RT), ST, true} -> + case match_types(DT, ST) and match_types(RT, type(E)) of + false -> + erlang:error(type_mismatch); + true -> + RL = ?LIST(R), + case extc([], ?LIST(S), to_external(E), RL) of + [] -> + R; + L -> + ?SET(merge(RL, reverse(L)), T) + end + end; + {?ANYTYPE, ?ANYTYPE, true} -> + R; + {?ANYTYPE, ST, true} -> + case type(E) of + ?SET_OF(?ANYTYPE) -> + R; + ET -> + ?SET([], ?BINREL(ST, ET)) + end; + {_, _, true} -> + erlang:error(badarg) + end. + +-spec(is_a_function(BinRel) -> Bool when + Bool :: boolean(), + BinRel :: binary_relation()). +is_a_function(R) when ?IS_SET(R) -> + case ?TYPE(R) of + ?BINREL(_, _) -> + case ?LIST(R) of + [] -> true; + [{V,_} | Es] -> is_a_func(Es, V) + end; + ?ANYTYPE -> true; + _ -> erlang:error(badarg) + end. + +-spec(restriction(BinRel1, Set) -> BinRel2 when + BinRel1 :: binary_relation(), + BinRel2 :: binary_relation(), + Set :: a_set()). +restriction(Relation, Set) -> + restriction(1, Relation, Set). + +-spec(drestriction(BinRel1, Set) -> BinRel2 when + BinRel1 :: binary_relation(), + BinRel2 :: binary_relation(), + Set :: a_set()). +drestriction(Relation, Set) -> + drestriction(1, Relation, Set). + +%%% +%%% Functions on functions only. +%%% + +-spec(composite(Function1, Function2) -> Function3 when + Function1 :: a_function(), + Function2 :: a_function(), + Function3 :: a_function()). +composite(Fn1, Fn2) when ?IS_SET(Fn1), ?IS_SET(Fn2) -> + ?BINREL(DTF1, RTF1) = case ?TYPE(Fn1)of + ?BINREL(_, _) = F1T -> F1T; + ?ANYTYPE -> {?ANYTYPE, ?ANYTYPE}; + _ -> erlang:error(badarg) + end, + ?BINREL(DTF2, RTF2) = case ?TYPE(Fn2) of + ?BINREL(_, _) = F2T -> F2T; + ?ANYTYPE -> {?ANYTYPE, ?ANYTYPE}; + _ -> erlang:error(badarg) + end, + case match_types(RTF1, DTF2) of + true when DTF1 =:= ?ANYTYPE -> Fn1; + true when DTF2 =:= ?ANYTYPE -> Fn2; + true -> + case comp(?LIST(Fn1), ?LIST(Fn2)) of + SL when is_list(SL) -> + ?SET(sort(SL), ?BINREL(DTF1, RTF2)); + Bad -> + erlang:error(Bad) + end; + false -> erlang:error(type_mismatch) + end. + +-spec(inverse(Function1) -> Function2 when + Function1 :: a_function(), + Function2 :: a_function()). +inverse(Fn) when ?IS_SET(Fn) -> + case ?TYPE(Fn) of + ?BINREL(DT, RT) -> + case inverse1(?LIST(Fn)) of + SL when is_list(SL) -> + ?SET(SL, ?BINREL(RT, DT)); + Bad -> + erlang:error(Bad) + end; + ?ANYTYPE -> Fn; + _ -> erlang:error(badarg) + end. + +%%% +%%% Functions on relations (binary or other). +%%% + +-spec(restriction(SetFun, Set1, Set2) -> Set3 when + SetFun :: set_fun(), + Set1 :: a_set(), + Set2 :: a_set(), + Set3 :: a_set()). +%% Equivalent to range(restriction(inverse(substitution(Fun, S1)), S2)). +restriction(I, R, S) when is_integer(I), ?IS_SET(R), ?IS_SET(S) -> + RT = ?TYPE(R), + ST = ?TYPE(S), + case check_for_sort(RT, I) of + empty -> + R; + error -> + erlang:error(badarg); + Sort -> + RL = ?LIST(R), + case {match_types(?REL_TYPE(I, RT), ST), ?LIST(S)} of + {true, _SL} when RL =:= [] -> + R; + {true, []} -> + ?SET([], RT); + {true, [E | Es]} when Sort =:= false -> % I =:= 1 + ?SET(reverse(restrict_n(I, RL, E, Es, [])), RT); + {true, [E | Es]} -> + ?SET(sort(restrict_n(I, keysort(I, RL), E, Es, [])), RT); + {false, _SL} -> + erlang:error(type_mismatch) + end + end; +restriction(SetFun, S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + Type1 = ?TYPE(S1), + Type2 = ?TYPE(S2), + SL1 = ?LIST(S1), + case external_fun(SetFun) of + false when Type2 =:= ?ANYTYPE -> + S2; + false -> + case subst(SL1, SetFun, element_type(Type1)) of + {NSL, NewType} -> % NewType can be ?ANYTYPE + case match_types(NewType, Type2) of + true -> + NL = sort(restrict(?LIST(S2), converse(NSL, []))), + ?SET(NL, Type1); + false -> + erlang:error(type_mismatch) + end; + Bad -> + erlang:error(Bad) + end; + _ when Type1 =:= ?ANYTYPE -> + S1; + _XFun when ?IS_SET_OF(Type1) -> + erlang:error(badarg); + XFun -> + FunT = XFun(Type1), + try check_fun(Type1, XFun, FunT) of + Sort -> + case match_types(FunT, Type2) of + true -> + R1 = inverse_substitution(SL1, XFun, Sort), + ?SET(sort(Sort, restrict(?LIST(S2), R1)), Type1); + false -> + erlang:error(type_mismatch) + end + catch _:_ -> erlang:error(badarg) + end + end. + +-spec(drestriction(SetFun, Set1, Set2) -> Set3 when + SetFun :: set_fun(), + Set1 :: a_set(), + Set2 :: a_set(), + Set3 :: a_set()). +drestriction(I, R, S) when is_integer(I), ?IS_SET(R), ?IS_SET(S) -> + RT = ?TYPE(R), + ST = ?TYPE(S), + case check_for_sort(RT, I) of + empty -> + R; + error -> + erlang:error(badarg); + Sort -> + RL = ?LIST(R), + case {match_types(?REL_TYPE(I, RT), ST), ?LIST(S)} of + {true, []} -> + R; + {true, _SL} when RL =:= [] -> + R; + {true, [E | Es]} when Sort =:= false -> % I =:= 1 + ?SET(diff_restrict_n(I, RL, E, Es, []), RT); + {true, [E | Es]} -> + ?SET(diff_restrict_n(I, keysort(I, RL), E, Es, []), RT); + {false, _SL} -> + erlang:error(type_mismatch) + end + end; +drestriction(SetFun, S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + Type1 = ?TYPE(S1), + Type2 = ?TYPE(S2), + SL1 = ?LIST(S1), + case external_fun(SetFun) of + false when Type2 =:= ?ANYTYPE -> + S1; + false -> + case subst(SL1, SetFun, element_type(Type1)) of + {NSL, NewType} -> % NewType can be ?ANYTYPE + case match_types(NewType, Type2) of + true -> + SL2 = ?LIST(S2), + NL = sort(diff_restrict(SL2, converse(NSL, []))), + ?SET(NL, Type1); + false -> + erlang:error(type_mismatch) + end; + Bad -> + erlang:error(Bad) + end; + _ when Type1 =:= ?ANYTYPE -> + S1; + _XFun when ?IS_SET_OF(Type1) -> + erlang:error(badarg); + XFun -> + FunT = XFun(Type1), + try check_fun(Type1, XFun, FunT) of + Sort -> + case match_types(FunT, Type2) of + true -> + R1 = inverse_substitution(SL1, XFun, Sort), + SL2 = ?LIST(S2), + ?SET(sort(Sort, diff_restrict(SL2, R1)), Type1); + false -> + erlang:error(type_mismatch) + end + catch _:_ -> erlang:error(badarg) + end + end. + +-spec(projection(SetFun, Set1) -> Set2 when + SetFun :: set_fun(), + Set1 :: a_set(), + Set2 :: a_set()). +projection(I, Set) when is_integer(I), ?IS_SET(Set) -> + Type = ?TYPE(Set), + case check_for_sort(Type, I) of + empty -> + Set; + error -> + erlang:error(badarg); + _ when I =:= 1 -> + ?SET(projection1(?LIST(Set)), ?REL_TYPE(I, Type)); + _ -> + ?SET(projection_n(?LIST(Set), I, []), ?REL_TYPE(I, Type)) + end; +projection(Fun, Set) -> + range(substitution(Fun, Set)). + +-spec(substitution(SetFun, Set1) -> Set2 when + SetFun :: set_fun(), + Set1 :: a_set(), + Set2 :: a_set()). +substitution(I, Set) when is_integer(I), ?IS_SET(Set) -> + Type = ?TYPE(Set), + case check_for_sort(Type, I) of + empty -> + Set; + error -> + erlang:error(badarg); + _Sort -> + NType = ?REL_TYPE(I, Type), + NSL = substitute_element(?LIST(Set), I, []), + ?SET(NSL, ?BINREL(Type, NType)) + end; +substitution(SetFun, Set) when ?IS_SET(Set) -> + Type = ?TYPE(Set), + L = ?LIST(Set), + case external_fun(SetFun) of + false when L =/= [] -> + case subst(L, SetFun, element_type(Type)) of + {SL, NewType} -> + ?SET(reverse(SL), ?BINREL(Type, NewType)); + Bad -> + erlang:error(Bad) + end; + false -> + empty_set(); + _ when Type =:= ?ANYTYPE -> + empty_set(); + _XFun when ?IS_SET_OF(Type) -> + erlang:error(badarg); + XFun -> + FunT = XFun(Type), + try check_fun(Type, XFun, FunT) of + _Sort -> + SL = substitute(L, XFun, []), + ?SET(SL, ?BINREL(Type, FunT)) + catch _:_ -> erlang:error(badarg) + end + end. + +-spec(partition(SetOfSets) -> Partition when + SetOfSets :: set_of_sets(), + Partition :: a_set()). +partition(Sets) -> + F1 = relation_to_family(canonical_relation(Sets)), + F2 = relation_to_family(converse(F1)), + range(F2). + +-spec(partition(SetFun, Set) -> Partition when + SetFun :: set_fun(), + Partition :: a_set(), + Set :: a_set()). +partition(I, Set) when is_integer(I), ?IS_SET(Set) -> + Type = ?TYPE(Set), + case check_for_sort(Type, I) of + empty -> + Set; + error -> + erlang:error(badarg); + false -> % I =:= 1 + ?SET(partition_n(I, ?LIST(Set)), ?SET_OF(Type)); + true -> + ?SET(partition_n(I, keysort(I, ?LIST(Set))), ?SET_OF(Type)) + end; +partition(Fun, Set) -> + range(partition_family(Fun, Set)). + +-spec(partition(SetFun, Set1, Set2) -> {Set3, Set4} when + SetFun :: set_fun(), + Set1 :: a_set(), + Set2 :: a_set(), + Set3 :: a_set(), + Set4 :: a_set()). +partition(I, R, S) when is_integer(I), ?IS_SET(R), ?IS_SET(S) -> + RT = ?TYPE(R), + ST = ?TYPE(S), + case check_for_sort(RT, I) of + empty -> + {R, R}; + error -> + erlang:error(badarg); + Sort -> + RL = ?LIST(R), + case {match_types(?REL_TYPE(I, RT), ST), ?LIST(S)} of + {true, _SL} when RL =:= [] -> + {R, R}; + {true, []} -> + {?SET([], RT), R}; + {true, [E | Es]} when Sort =:= false -> % I =:= 1 + [L1 | L2] = partition3_n(I, RL, E, Es, [], []), + {?SET(L1, RT), ?SET(L2, RT)}; + {true, [E | Es]} -> + [L1 | L2] = partition3_n(I, keysort(I,RL), E, Es, [], []), + {?SET(L1, RT), ?SET(L2, RT)}; + {false, _SL} -> + erlang:error(type_mismatch) + end + end; +partition(SetFun, S1, S2) when ?IS_SET(S1), ?IS_SET(S2) -> + Type1 = ?TYPE(S1), + Type2 = ?TYPE(S2), + SL1 = ?LIST(S1), + case external_fun(SetFun) of + false when Type2 =:= ?ANYTYPE -> + {S2, S1}; + false -> + case subst(SL1, SetFun, element_type(Type1)) of + {NSL, NewType} -> % NewType can be ?ANYTYPE + case match_types(NewType, Type2) of + true -> + R1 = converse(NSL, []), + [L1 | L2] = partition3(?LIST(S2), R1), + {?SET(sort(L1), Type1), ?SET(sort(L2), Type1)}; + false -> + erlang:error(type_mismatch) + end; + Bad -> + erlang:error(Bad) + end; + _ when Type1 =:= ?ANYTYPE -> + {S1, S1}; + _XFun when ?IS_SET_OF(Type1) -> + erlang:error(badarg); + XFun -> + FunT = XFun(Type1), + try check_fun(Type1, XFun, FunT) of + Sort -> + case match_types(FunT, Type2) of + true -> + R1 = inverse_substitution(SL1, XFun, Sort), + [L1 | L2] = partition3(?LIST(S2), R1), + {?SET(sort(L1), Type1), ?SET(sort(L2), Type1)}; + false -> + erlang:error(type_mismatch) + end + catch _:_ -> erlang:error(badarg) + end + end. + +-spec(multiple_relative_product(TupleOfBinRels, BinRel1) -> BinRel2 when + TupleOfBinRels :: tuple_of(BinRel), + BinRel :: binary_relation(), + BinRel1 :: binary_relation(), + BinRel2 :: binary_relation()). +multiple_relative_product(T, R) when is_tuple(T), ?IS_SET(R) -> + case test_rel(R, tuple_size(T), eq) of + true when ?TYPE(R) =:= ?ANYTYPE -> + empty_set(); + true -> + MProd = mul_relprod(tuple_to_list(T), 1, R), + relative_product(MProd); + false -> + erlang:error(badarg) + end. + +-spec(join(Relation1, I, Relation2, J) -> Relation3 when + Relation1 :: relation(), + Relation2 :: relation(), + Relation3 :: relation(), + I :: pos_integer(), + J :: pos_integer()). +join(R1, I1, R2, I2) + when ?IS_SET(R1), ?IS_SET(R2), is_integer(I1), is_integer(I2) -> + case test_rel(R1, I1, lte) and test_rel(R2, I2, lte) of + false -> erlang:error(badarg); + true when ?TYPE(R1) =:= ?ANYTYPE -> R1; + true when ?TYPE(R2) =:= ?ANYTYPE -> R2; + true -> + L1 = ?LIST(raise_element(R1, I1)), + L2 = ?LIST(raise_element(R2, I2)), + T = relprod1(L1, L2), + F = case (I1 =:= 1) and (I2 =:= 1) of + true -> + fun({X,Y}) -> join_element(X, Y) end; + false -> + fun({X,Y}) -> + list_to_tuple(join_element(X, Y, I2)) + end + end, + ?SET(replace(T, F, []), F({?TYPE(R1), ?TYPE(R2)})) + end. + +%% Inlined. +test_rel(R, I, C) -> + case ?TYPE(R) of + Rel when ?IS_RELATION(Rel), C =:= eq, I =:= ?REL_ARITY(Rel) -> true; + Rel when ?IS_RELATION(Rel), C =:= lte, I>=1, I =< ?REL_ARITY(Rel) -> + true; + ?ANYTYPE -> true; + _ -> false + end. + +%%% +%%% Family functions +%%% + +-spec(fam2rel(Family) -> BinRel when + Family :: family(), + BinRel :: binary_relation()). +fam2rel(F) -> + family_to_relation(F). + +-spec(family_to_relation(Family) -> BinRel when + Family :: family(), + BinRel :: binary_relation()). +%% Inlined. +family_to_relation(F) when ?IS_SET(F) -> + case ?TYPE(F) of + ?FAMILY(DT, RT) -> + ?SET(family2rel(?LIST(F), []), ?BINREL(DT, RT)); + ?ANYTYPE -> F; + _ -> erlang:error(badarg) + end. + +-spec(family_specification(Fun, Family1) -> Family2 when + Fun :: spec_fun(), + Family1 :: family(), + Family2 :: family()). +family_specification(Fun, F) when ?IS_SET(F) -> + case ?TYPE(F) of + ?FAMILY(_DT, Type) = FType -> + R = case external_fun(Fun) of + false -> + fam_spec(?LIST(F), Fun, Type, []); + XFun -> + fam_specification(?LIST(F), XFun, []) + end, + case R of + SL when is_list(SL) -> + ?SET(SL, FType); + Bad -> + erlang:error(Bad) + end; + ?ANYTYPE -> F; + _ -> erlang:error(badarg) + end. + +-spec(union_of_family(Family) -> Set when + Family :: family(), + Set :: a_set()). +union_of_family(F) when ?IS_SET(F) -> + case ?TYPE(F) of + ?FAMILY(_DT, Type) -> + ?SET(un_of_fam(?LIST(F), []), Type); + ?ANYTYPE -> F; + _ -> erlang:error(badarg) + end. + +-spec(intersection_of_family(Family) -> Set when + Family :: family(), + Set :: a_set()). +intersection_of_family(F) when ?IS_SET(F) -> + case ?TYPE(F) of + ?FAMILY(_DT, Type) -> + case int_of_fam(?LIST(F)) of + FU when is_list(FU) -> + ?SET(FU, Type); + Bad -> + erlang:error(Bad) + end; + _ -> erlang:error(badarg) + end. + +-spec(family_union(Family1) -> Family2 when + Family1 :: family(), + Family2 :: family()). +family_union(F) when ?IS_SET(F) -> + case ?TYPE(F) of + ?FAMILY(DT, ?SET_OF(Type)) -> + ?SET(fam_un(?LIST(F), []), ?FAMILY(DT, Type)); + ?ANYTYPE -> F; + _ -> erlang:error(badarg) + end. + +-spec(family_intersection(Family1) -> Family2 when + Family1 :: family(), + Family2 :: family()). +family_intersection(F) when ?IS_SET(F) -> + case ?TYPE(F) of + ?FAMILY(DT, ?SET_OF(Type)) -> + case fam_int(?LIST(F), []) of + FU when is_list(FU) -> + ?SET(FU, ?FAMILY(DT, Type)); + Bad -> + erlang:error(Bad) + end; + ?ANYTYPE -> F; + _ -> erlang:error(badarg) + end. + +-spec(family_domain(Family1) -> Family2 when + Family1 :: family(), + Family2 :: family()). +family_domain(F) when ?IS_SET(F) -> + case ?TYPE(F) of + ?FAMILY(FDT, ?BINREL(DT, _)) -> + ?SET(fam_dom(?LIST(F), []), ?FAMILY(FDT, DT)); + ?ANYTYPE -> F; + ?FAMILY(_, ?ANYTYPE) -> F; + _ -> erlang:error(badarg) + end. + +-spec(family_range(Family1) -> Family2 when + Family1 :: family(), + Family2 :: family()). +family_range(F) when ?IS_SET(F) -> + case ?TYPE(F) of + ?FAMILY(DT, ?BINREL(_, RT)) -> + ?SET(fam_ran(?LIST(F), []), ?FAMILY(DT, RT)); + ?ANYTYPE -> F; + ?FAMILY(_, ?ANYTYPE) -> F; + _ -> erlang:error(badarg) + end. + +-spec(family_field(Family1) -> Family2 when + Family1 :: family(), + Family2 :: family()). +family_field(F) -> + family_union(family_domain(F), family_range(F)). + +-spec(family_union(Family1, Family2) -> Family3 when + Family1 :: family(), + Family2 :: family(), + Family3 :: family()). +family_union(F1, F2) -> + fam_binop(F1, F2, fun fam_union/3). + +-spec(family_intersection(Family1, Family2) -> Family3 when + Family1 :: family(), + Family2 :: family(), + Family3 :: family()). +family_intersection(F1, F2) -> + fam_binop(F1, F2, fun fam_intersect/3). + +-spec(family_difference(Family1, Family2) -> Family3 when + Family1 :: family(), + Family2 :: family(), + Family3 :: family()). +family_difference(F1, F2) -> + fam_binop(F1, F2, fun fam_difference/3). + +%% Inlined. +fam_binop(F1, F2, FF) when ?IS_SET(F1), ?IS_SET(F2) -> + case unify_types(?TYPE(F1), ?TYPE(F2)) of + [] -> + erlang:error(type_mismatch); + ?ANYTYPE -> + F1; + Type = ?FAMILY(_, _) -> + ?SET(FF(?LIST(F1), ?LIST(F2), []), Type); + _ -> erlang:error(badarg) + end. + +-spec(partition_family(SetFun, Set) -> Family when + Family :: family(), + SetFun :: set_fun(), + Set :: a_set()). +partition_family(I, Set) when is_integer(I), ?IS_SET(Set) -> + Type = ?TYPE(Set), + case check_for_sort(Type, I) of + empty -> + Set; + error -> + erlang:error(badarg); + false -> % when I =:= 1 + ?SET(fam_partition_n(I, ?LIST(Set)), + ?BINREL(?REL_TYPE(I, Type), ?SET_OF(Type))); + true -> + ?SET(fam_partition_n(I, keysort(I, ?LIST(Set))), + ?BINREL(?REL_TYPE(I, Type), ?SET_OF(Type))) + end; +partition_family(SetFun, Set) when ?IS_SET(Set) -> + Type = ?TYPE(Set), + SL = ?LIST(Set), + case external_fun(SetFun) of + false when SL =/= [] -> + case subst(SL, SetFun, element_type(Type)) of + {NSL, NewType} -> + P = fam_partition(converse(NSL, []), true), + ?SET(reverse(P), ?BINREL(NewType, ?SET_OF(Type))); + Bad -> + erlang:error(Bad) + end; + false -> + empty_set(); + _ when Type =:= ?ANYTYPE -> + empty_set(); + _XFun when ?IS_SET_OF(Type) -> + erlang:error(badarg); + XFun -> + DType = XFun(Type), + try check_fun(Type, XFun, DType) of + Sort -> + Ts = inverse_substitution(?LIST(Set), XFun, Sort), + P = fam_partition(Ts, Sort), + ?SET(reverse(P), ?BINREL(DType, ?SET_OF(Type))) + catch _:_ -> erlang:error(badarg) + end + end. + +-spec(family_projection(SetFun, Family1) -> Family2 when + SetFun :: set_fun(), + Family1 :: family(), + Family2 :: family()). +family_projection(SetFun, F) when ?IS_SET(F) -> + case ?TYPE(F) of + ?FAMILY(_, _) when [] =:= ?LIST(F) -> + empty_set(); + ?FAMILY(DT, Type) -> + case external_fun(SetFun) of + false -> + case fam_proj(?LIST(F), SetFun, Type, ?ANYTYPE, []) of + {SL, NewType} -> + ?SET(SL, ?BINREL(DT, NewType)); + Bad -> + erlang:error(Bad) + end; + _ -> + erlang:error(badarg) + end; + ?ANYTYPE -> F; + _ -> erlang:error(badarg) + end. + +%%% +%%% Digraph functions +%%% + +-spec(family_to_digraph(Family) -> Graph when + Graph :: digraph:graph(), + Family :: family()). +family_to_digraph(F) when ?IS_SET(F) -> + case ?TYPE(F) of + ?FAMILY(_, _) -> fam2digraph(F, digraph:new()); + ?ANYTYPE -> digraph:new(); + _Else -> erlang:error(badarg) + end. + +-spec(family_to_digraph(Family, GraphType) -> Graph when + Graph :: digraph:graph(), + Family :: family(), + GraphType :: [digraph:d_type()]). +family_to_digraph(F, Type) when ?IS_SET(F) -> + case ?TYPE(F) of + ?FAMILY(_, _) -> ok; + ?ANYTYPE -> ok; + _Else -> erlang:error(badarg) + end, + try digraph:new(Type) of + G -> case catch fam2digraph(F, G) of + {error, Reason} -> + true = digraph:delete(G), + erlang:error(Reason); + _ -> + G + end + catch + error:badarg -> erlang:error(badarg) + end. + +-spec(digraph_to_family(Graph) -> Family when + Graph :: digraph:graph(), + Family :: family()). +digraph_to_family(G) -> + try digraph_family(G) of + L -> ?SET(L, ?FAMILY(?ATOM_TYPE, ?ATOM_TYPE)) + catch _:_ -> erlang:error(badarg) + end. + +-spec(digraph_to_family(Graph, Type) -> Family when + Graph :: digraph:graph(), + Family :: family(), + Type :: type()). +digraph_to_family(G, T) -> + case {is_type(T), T} of + {true, ?SET_OF(?FAMILY(_,_) = Type)} -> + try digraph_family(G) of + L -> ?SET(L, Type) + catch _:_ -> erlang:error(badarg) + end; + _ -> + erlang:error(badarg) + end. + +%% +%% Local functions +%% + +%% Type = OrderedSetType +%% | SetType +%% | atom() except '_' +%% OrderedSetType = {Type, ..., Type} +%% SetType = [ElementType] % list of exactly one element +%% ElementType = '_' % any type (implies empty set) +%% | Type + +is_types(0, _T) -> + true; +is_types(I, T) -> + case is_type(?REL_TYPE(I, T)) of + true -> is_types(I-1, T); + false -> false + end. + +is_element_type(?ANYTYPE) -> + true; +is_element_type(T) -> + is_type(T). + +set_of_sets([S | Ss], L, T0) when ?IS_SET(S) -> + case unify_types([?TYPE(S)], T0) of + [] -> {error, type_mismatch}; + Type -> set_of_sets(Ss, [?LIST(S) | L], Type) + end; +set_of_sets([S | Ss], L, T0) when ?IS_ORDSET(S) -> + case unify_types(?ORDTYPE(S), T0) of + [] -> {error, type_mismatch}; + Type -> set_of_sets(Ss, [?ORDDATA(S) | L], Type) + end; +set_of_sets([], L, T) -> + ?SET(usort(L), T); +set_of_sets(_, _L, _T) -> + {error, badarg}. + +ordset_of_sets([S | Ss], L, T) when ?IS_SET(S) -> + ordset_of_sets(Ss, [?LIST(S) | L], [[?TYPE(S)] | T]); +ordset_of_sets([S | Ss], L, T) when ?IS_ORDSET(S) -> + ordset_of_sets(Ss, [?ORDDATA(S) | L], [?ORDTYPE(S) | T]); +ordset_of_sets([], L, T) -> + ?ORDSET(list_to_tuple(reverse(L)), list_to_tuple(reverse(T))); +ordset_of_sets(_, _L, _T) -> + error. + +%% Inlined. +rel(Ts, [Type]) -> + case is_type(Type) and atoms_only(Type, 1) of + true -> + rel(Ts, tuple_size(Type), Type); + false -> + rel_type(Ts, [], Type) + end; +rel(Ts, Sz) -> + rel(Ts, Sz, erlang:make_tuple(Sz, ?ATOM_TYPE)). + +atoms_only(Type, I) when ?IS_ATOM_TYPE(?REL_TYPE(I, Type)) -> + atoms_only(Type, I+1); +atoms_only(Type, I) when I > tuple_size(Type), ?IS_RELATION(Type) -> + true; +atoms_only(_Type, _I) -> + false. + +rel(Ts, Sz, Type) when Sz >= 1 -> + SL = usort(Ts), + rel(SL, SL, Sz, Type). + +rel([T | Ts], L, Sz, Type) when tuple_size(T) =:= Sz -> + rel(Ts, L, Sz, Type); +rel([], L, _Sz, Type) -> + ?SET(L, Type). + +rel_type([E | Ts], L, Type) -> + {NType, NE} = make_element(E, Type, Type), + rel_type(Ts, [NE | L], NType); +rel_type([], [], ?ANYTYPE) -> + empty_set(); +rel_type([], SL, Type) when ?IS_RELATION(Type) -> + ?SET(usort(SL), Type). + +%% Inlined. +a_func(Ts, T) -> + case {T, is_type(T)} of + {[?BINREL(DT, RT) = Type], true} when ?IS_ATOM_TYPE(DT), + ?IS_ATOM_TYPE(RT) -> + func(Ts, Type); + {[Type], true} -> + func_type(Ts, [], Type, fun(?BINREL(_,_)) -> true end) + end. + +func(L0, Type) -> + L = usort(L0), + func(L, L, L, Type). + +func([{X,_} | Ts], X0, L, Type) when X /= X0 -> + func(Ts, X, L, Type); +func([{X,_} | _Ts], X0, _L, _Type) when X == X0 -> + bad_function; +func([], _X0, L, Type) -> + ?SET(L, Type). + +%% Inlined. +fam(Ts, T) -> + case {T, is_type(T)} of + {[?FAMILY(DT, RT) = Type], true} when ?IS_ATOM_TYPE(DT), + ?IS_ATOM_TYPE(RT) -> + fam2(Ts, Type); + {[Type], true} -> + func_type(Ts, [], Type, fun(?FAMILY(_,_)) -> true end) + end. + +fam2([], Type) -> + ?SET([], Type); +fam2(Ts, Type) -> + fam2(sort(Ts), Ts, [], Type). + +fam2([{I,L} | T], I0, SL, Type) when I /= I0 -> + fam2(T, I, [{I,usort(L)} | SL], Type); +fam2([{I,L} | T], I0, SL, Type) when I == I0 -> + case {usort(L), SL} of + {NL, [{_I,NL1} | _]} when NL == NL1 -> + fam2(T, I0, SL, Type); + _ -> + bad_function + end; +fam2([], _I0, SL, Type) -> + ?SET(reverse(SL), Type). + +func_type([E | T], SL, Type, F) -> + {NType, NE} = make_element(E, Type, Type), + func_type(T, [NE | SL], NType, F); +func_type([], [], ?ANYTYPE, _F) -> + empty_set(); +func_type([], SL, Type, F) -> + true = F(Type), + NL = usort(SL), + check_function(NL, ?SET(NL, Type)). + +setify(L, ?SET_OF(Atom)) when ?IS_ATOM_TYPE(Atom), Atom =/= ?ANYTYPE -> + ?SET(usort(L), Atom); +setify(L, ?SET_OF(Type0)) -> + try is_no_lists(Type0) of + N when is_integer(N) -> + rel(L, N, Type0); + Sizes -> + make_oset(L, Sizes, L, Type0) + catch + _:_ -> + {?SET_OF(Type), Set} = create(L, Type0, Type0, []), + ?SET(Set, Type) + end; +setify(E, Type0) -> + {Type, OrdSet} = make_element(E, Type0, Type0), + ?ORDSET(OrdSet, Type). + +is_no_lists(T) when is_tuple(T) -> + Sz = tuple_size(T), + is_no_lists(T, Sz, Sz, []). + +is_no_lists(_T, 0, Sz, []) -> + Sz; +is_no_lists(_T, 0, Sz, L) -> + {Sz, L}; +is_no_lists(T, I, Sz, L) when ?IS_ATOM_TYPE(?REL_TYPE(I, T)) -> + is_no_lists(T, I-1, Sz, L); +is_no_lists(T, I, Sz, L) -> + is_no_lists(T, I-1, Sz, [{I,is_no_lists(?REL_TYPE(I, T))} | L]). + +create([E | Es], T, T0, L) -> + {NT, S} = make_element(E, T, T0), + create(Es, NT, T0, [S | L]); +create([], T, _T0, L) -> + {?SET_OF(T), usort(L)}. + +make_element(C, ?ANYTYPE, _T0) -> + make_element(C); +make_element(C, Atom, ?ANYTYPE) when ?IS_ATOM_TYPE(Atom), + not is_list(C), not is_tuple(C) -> + {Atom, C}; +make_element(C, Atom, Atom) when ?IS_ATOM_TYPE(Atom) -> + {Atom, C}; +make_element(T, TT, ?ANYTYPE) when tuple_size(T) =:= tuple_size(TT) -> + make_tuple(tuple_to_list(T), tuple_to_list(TT), [], [], ?ANYTYPE); +make_element(T, TT, T0) when tuple_size(T) =:= tuple_size(TT) -> + make_tuple(tuple_to_list(T), tuple_to_list(TT), [], [], tuple_to_list(T0)); +make_element(L, [LT], ?ANYTYPE) when is_list(L) -> + create(L, LT, ?ANYTYPE, []); +make_element(L, [LT], [T0]) when is_list(L) -> + create(L, LT, T0, []). + +make_tuple([E | Es], [T | Ts], NT, L, T0) when T0 =:= ?ANYTYPE -> + {ET, ES} = make_element(E, T, T0), + make_tuple(Es, Ts, [ET | NT], [ES | L], T0); +make_tuple([E | Es], [T | Ts], NT, L, [T0 | T0s]) -> + {ET, ES} = make_element(E, T, T0), + make_tuple(Es, Ts, [ET | NT], [ES | L], T0s); +make_tuple([], [], NT, L, _T0s) when NT =/= [] -> + {list_to_tuple(reverse(NT)), list_to_tuple(reverse(L))}. + +%% Derive type. +make_element(C) when not is_list(C), not is_tuple(C) -> + {?ATOM_TYPE, C}; +make_element(T) when is_tuple(T) -> + make_tuple(tuple_to_list(T), [], []); +make_element(L) when is_list(L) -> + create(L, ?ANYTYPE, ?ANYTYPE, []). + +make_tuple([E | Es], T, L) -> + {ET, ES} = make_element(E), + make_tuple(Es, [ET | T], [ES | L]); +make_tuple([], T, L) when T =/= [] -> + {list_to_tuple(reverse(T)), list_to_tuple(reverse(L))}. + +make_oset([T | Ts], Szs, L, Type) -> + true = test_oset(Szs, T, T), + make_oset(Ts, Szs, L, Type); +make_oset([], _Szs, L, Type) -> + ?SET(usort(L), Type). + +%% Optimization. Avoid re-building (nested) tuples. +test_oset({Sz,Args}, T, T0) when tuple_size(T) =:= Sz -> + test_oset_args(Args, T, T0); +test_oset(Sz, T, _T0) when tuple_size(T) =:= Sz -> + true. + +test_oset_args([{Arg,Szs} | Ss], T, T0) -> + true = test_oset(Szs, ?REL_TYPE(Arg, T), T0), + test_oset_args(Ss, T, T0); +test_oset_args([], _T, _T0) -> + true. + +list_of_sets([S | Ss], Type, L) -> + list_of_sets(Ss, Type, [?SET(S, Type) | L]); +list_of_sets([], _Type, L) -> + reverse(L). + +list_of_ordsets([S | Ss], Type, L) -> + list_of_ordsets(Ss, Type, [?ORDSET(S, Type) | L]); +list_of_ordsets([], _Type, L) -> + reverse(L). + +tuple_of_sets([S | Ss], [?SET_OF(Type) | Types], L) -> + tuple_of_sets(Ss, Types, [?SET(S, Type) | L]); +tuple_of_sets([S | Ss], [Type | Types], L) -> + tuple_of_sets(Ss, Types, [?ORDSET(S, Type) | L]); +tuple_of_sets([], [], L) -> + list_to_tuple(reverse(L)). + +spec([E | Es], Fun, Type, L) -> + case Fun(term2set(E, Type)) of + true -> + spec(Es, Fun, Type, [E | L]); + false -> + spec(Es, Fun, Type, L); + _ -> + badarg + end; +spec([], _Fun, _Type, L) -> + reverse(L). + +specification([E | Es], Fun, L) -> + case Fun(E) of + true -> + specification(Es, Fun, [E | L]); + false -> + specification(Es, Fun, L); + _ -> + badarg + end; +specification([], _Fun, L) -> + reverse(L). + +%% Elements from the first list are kept. +intersection([H1 | T1], [H2 | T2], L) when H1 < H2 -> + intersection1(T1, T2, L, H2); +intersection([H1 | T1], [H2 | T2], L) when H1 == H2 -> + intersection(T1, T2, [H1 | L]); +intersection([H1 | T1], [_H2 | T2], L) -> + intersection2(T1, T2, L, H1); +intersection(_, _, L) -> + reverse(L). + +intersection1([H1 | T1], T2, L, H2) when H1 < H2 -> + intersection1(T1, T2, L, H2); +intersection1([H1 | T1], T2, L, H2) when H1 == H2 -> + intersection(T1, T2, [H1 | L]); +intersection1([H1 | T1], T2, L, _H2) -> + intersection2(T1, T2, L, H1); +intersection1(_, _, L, _) -> + reverse(L). + +intersection2(T1, [H2 | T2], L, H1) when H1 > H2 -> + intersection2(T1, T2, L, H1); +intersection2(T1, [H2 | T2], L, H1) when H1 == H2 -> + intersection(T1, T2, [H1 | L]); +intersection2(T1, [H2 | T2], L, _H1) -> + intersection1(T1, T2, L, H2); +intersection2(_, _, L, _) -> + reverse(L). + +difference([H1 | T1], [H2 | T2], L) when H1 < H2 -> + diff(T1, T2, [H1 | L], H2); +difference([H1 | T1], [H2 | T2], L) when H1 == H2 -> + difference(T1, T2, L); +difference([H1 | T1], [_H2 | T2], L) -> + diff2(T1, T2, L, H1); +difference(L1, _, L) -> + reverse(L, L1). + +diff([H1 | T1], T2, L, H2) when H1 < H2 -> + diff(T1, T2, [H1 | L], H2); +diff([H1 | T1], T2, L, H2) when H1 == H2 -> + difference(T1, T2, L); +diff([H1 | T1], T2, L, _H2) -> + diff2(T1, T2, L, H1); +diff(_, _, L, _) -> + reverse(L). + +diff2(T1, [H2 | T2], L, H1) when H1 > H2 -> + diff2(T1, T2, L, H1); +diff2(T1, [H2 | T2], L, H1) when H1 == H2 -> + difference(T1, T2, L); +diff2(T1, [H2 | T2], L, H1) -> + diff(T1, T2, [H1 | L], H2); +diff2(T1, _, L, H1) -> + reverse(L, [H1 | T1]). + +symdiff([H1 | T1], T2, L) -> + symdiff2(T1, T2, L, H1); +symdiff(_, T2, L) -> + reverse(L, T2). + +symdiff1([H1 | T1], T2, L, H2) when H1 < H2 -> + symdiff1(T1, T2, [H1 | L], H2); +symdiff1([H1 | T1], T2, L, H2) when H1 == H2 -> + symdiff(T1, T2, L); +symdiff1([H1 | T1], T2, L, H2) -> + symdiff2(T1, T2, [H2 | L], H1); +symdiff1(_, T2, L, H2) -> + reverse(L, [H2 | T2]). + +symdiff2(T1, [H2 | T2], L, H1) when H1 > H2 -> + symdiff2(T1, T2, [H2 | L], H1); +symdiff2(T1, [H2 | T2], L, H1) when H1 == H2 -> + symdiff(T1, T2, L); +symdiff2(T1, [H2 | T2], L, H1) -> + symdiff1(T1, T2, [H1 | L], H2); +symdiff2(T1, _, L, H1) -> + reverse(L, [H1 | T1]). + +sympart([H1 | T1], [H2 | T2], L1, L12, L2, T) when H1 < H2 -> + sympart1(T1, T2, [H1 | L1], L12, L2, T, H2); +sympart([H1 | T1], [H2 | T2], L1, L12, L2, T) when H1 == H2 -> + sympart(T1, T2, L1, [H1 | L12], L2, T); +sympart([H1 | T1], [H2 | T2], L1, L12, L2, T) -> + sympart2(T1, T2, L1, L12, [H2 | L2], T, H1); +sympart(S1, [], L1, L12, L2, T) -> + {?SET(reverse(L1, S1), T), + ?SET(reverse(L12), T), + ?SET(reverse(L2), T)}; +sympart(_, S2, L1, L12, L2, T) -> + {?SET(reverse(L1), T), + ?SET(reverse(L12), T), + ?SET(reverse(L2, S2), T)}. + +sympart1([H1 | T1], T2, L1, L12, L2, T, H2) when H1 < H2 -> + sympart1(T1, T2, [H1 | L1], L12, L2, T, H2); +sympart1([H1 | T1], T2, L1, L12, L2, T, H2) when H1 == H2 -> + sympart(T1, T2, L1, [H1 | L12], L2, T); +sympart1([H1 | T1], T2, L1, L12, L2, T, H2) -> + sympart2(T1, T2, L1, L12, [H2 | L2], T, H1); +sympart1(_, T2, L1, L12, L2, T, H2) -> + {?SET(reverse(L1), T), + ?SET(reverse(L12), T), + ?SET(reverse(L2, [H2 | T2]), T)}. + +sympart2(T1, [H2 | T2], L1, L12, L2, T, H1) when H1 > H2 -> + sympart2(T1, T2, L1, L12, [H2 | L2], T, H1); +sympart2(T1, [H2 | T2], L1, L12, L2, T, H1) when H1 == H2 -> + sympart(T1, T2, L1, [H1 | L12], L2, T); +sympart2(T1, [H2 | T2], L1, L12, L2, T, H1) -> + sympart1(T1, T2, [H1 | L1], L12, L2, T, H2); +sympart2(T1, _, L1, L12, L2, T, H1) -> + {?SET(reverse(L1, [H1 | T1]), T), + ?SET(reverse(L12), T), + ?SET(reverse(L2), T)}. + +prod([[E | Es] | Xs], T, L) -> + prod(Es, Xs, T, prod(Xs, [E | T], L)); +prod([], T, L) -> + [list_to_tuple(reverse(T)) | L]. + +prod([E | Es], Xs, T, L) -> + prod(Es, Xs, T, prod(Xs, [E | T], L)); +prod([], _Xs, _E, L) -> + L. + +constant_function([E | Es], X, L) -> + constant_function(Es, X, [{E,X} | L]); +constant_function([], _X, L) -> + reverse(L). + +subset([H1 | T1], [H2 | T2]) when H1 > H2 -> + subset(T1, T2, H1); +subset([H1 | T1], [H2 | T2]) when H1 == H2 -> + subset(T1, T2); +subset(L1, _) -> + L1 =:= []. + +subset(T1, [H2 | T2], H1) when H1 > H2 -> + subset(T1, T2, H1); +subset(T1, [H2 | T2], H1) when H1 == H2 -> + subset(T1, T2); +subset(_, _, _) -> + false. + +disjoint([B | Bs], A, As) when A < B -> + disjoint(As, B, Bs); +disjoint([B | _Bs], A, _As) when A == B -> + false; +disjoint([_B | Bs], A, As) -> + disjoint(Bs, A, As); +disjoint(_Bs, _A, _As) -> + true. + +%% Append sets that come in order, then "merge". +lunion([[_] = S]) -> % optimization + S; +lunion([[] | Ls]) -> + lunion(Ls); +lunion([S | Ss]) -> + umerge(lunion(Ss, last(S), [S], [])); +lunion([]) -> + []. + +lunion([[E] = S | Ss], Last, SL, Ls) when E > Last -> % optimization + lunion(Ss, E, [S | SL], Ls); +lunion([S | Ss], Last, SL, Ls) when hd(S) > Last -> + lunion(Ss, last(S), [S | SL], Ls); +lunion([S | Ss], _Last, SL, Ls) -> + lunion(Ss, last(S), [S], [append(reverse(SL)) | Ls]); +lunion([], _Last, SL, Ls) -> + [append(reverse(SL)) | Ls]. + +%% The empty list is always the first list, if present. +lintersection(_, []) -> + []; +lintersection([S | Ss], S0) -> + lintersection(Ss, intersection(S, S0, [])); +lintersection([], S) -> + S. + +can_rel([S | Ss], L) -> + can_rel(Ss, L, S, S); +can_rel([], L) -> + sort(L). + +can_rel(Ss, L, [E | Es], S) -> + can_rel(Ss, [{E, S} | L], Es, S); +can_rel(Ss, L, _, _S) -> + can_rel(Ss, L). + +rel2family([{X,Y} | S]) -> + rel2fam(S, X, [Y], []); +rel2family([]) -> + []. + +rel2fam([{X,Y} | S], X0, YL, L) when X0 == X -> + rel2fam(S, X0, [Y | YL], L); +rel2fam([{X,Y} | S], X0, [A,B | YL], L) -> % optimization + rel2fam(S, X, [Y], [{X0,reverse(YL,[B,A])} | L]); +rel2fam([{X,Y} | S], X0, YL, L) -> + rel2fam(S, X, [Y], [{X0,YL} | L]); +rel2fam([], X, YL, L) -> + reverse([{X,reverse(YL)} | L]). + +dom([{X,_} | Es]) -> + dom([], X, Es); +dom([] = L) -> + L. + +dom(L, X, [{X1,_} | Es]) when X == X1 -> + dom(L, X, Es); +dom(L, X, [{Y,_} | Es]) -> + dom([X | L], Y, Es); +dom(L, X, []) -> + reverse(L, [X]). + +ran([{_,Y} | Es], L) -> + ran(Es, [Y | L]); +ran([], L) -> + usort(L). + +relprod(A, B) -> + usort(relprod1(A, B)). + +relprod1([{Ay,Ax} | A], B) -> + relprod1(B, Ay, Ax, A, []); +relprod1(_A, _B) -> + []. + +relprod1([{Bx,_By} | B], Ay, Ax, A, L) when Ay > Bx -> + relprod1(B, Ay, Ax, A, L); +relprod1([{Bx,By} | B], Ay, Ax, A, L) when Ay == Bx -> + relprod(B, Bx, By, A, [{Ax,By} | L], Ax, B, Ay); +relprod1([{Bx,By} | B], _Ay, _Ax, A, L) -> + relprod2(B, Bx, By, A, L); +relprod1(_B, _Ay, _Ax, _A, L) -> + L. + +relprod2(B, Bx, By, [{Ay, _Ax} | A], L) when Ay < Bx -> + relprod2(B, Bx, By, A, L); +relprod2(B, Bx, By, [{Ay, Ax} | A], L) when Ay == Bx -> + relprod(B, Bx, By, A, [{Ax,By} | L], Ax, B, Ay); +relprod2(B, _Bx, _By, [{Ay, Ax} | A], L) -> + relprod1(B, Ay, Ax, A, L); +relprod2(_, _, _, _, L) -> + L. + +relprod(B0, Bx0, By0, A0, L, Ax, [{Bx,By} | B], Ay) when Ay == Bx -> + relprod(B0, Bx0, By0, A0, [{Ax,By} | L], Ax, B, Ay); +relprod(B0, Bx0, By0, A0, L, _Ax, _B, _Ay) -> + relprod2(B0, Bx0, By0, A0, L). + +relprod_n([], _R, _EmptyG, _IsR) -> + {error, badarg}; +relprod_n(RL, R, EmptyR, IsR) -> + case domain_type(RL, ?ANYTYPE) of + Error = {error, _Reason} -> + Error; + DType -> + Empty = any(fun is_empty_set/1, RL) or EmptyR, + RType = range_type(RL, []), + Type = ?BINREL(DType, RType), + Prod = + case Empty of + true when DType =:= ?ANYTYPE; RType =:= ?ANYTYPE -> + empty_set(); + true -> + ?SET([], Type); + false -> + TL = ?LIST((relprod_n(RL))), + Sz = length(RL), + Fun = fun({X,A}) -> {X, flat(Sz, A, [])} end, + ?SET(map(Fun, TL), Type) + end, + case IsR of + true -> relative_product(Prod, R); + false -> Prod + end + end. + +relprod_n([R | Rs]) -> + relprod_n(Rs, R). + +relprod_n([], R) -> + R; +relprod_n([R | Rs], R0) -> + T = raise_element(R0, 1), + R1 = relative_product1(T, R), + NR = projection({external, fun({{X,A},AS}) -> {X,{A,AS}} end}, R1), + relprod_n(Rs, NR). + +flat(1, A, L) -> + list_to_tuple([A | L]); +flat(N, {T,A}, L) -> + flat(N-1, T, [A | L]). + +domain_type([T | Ts], T0) when ?IS_SET(T) -> + case ?TYPE(T) of + ?BINREL(DT, _RT) -> + case unify_types(DT, T0) of + [] -> {error, type_mismatch}; + T1 -> domain_type(Ts, T1) + end; + ?ANYTYPE -> + domain_type(Ts, T0); + _ -> {error, badarg} + end; +domain_type([], T0) -> + T0. + +range_type([T | Ts], L) -> + case ?TYPE(T) of + ?BINREL(_DT, RT) -> + range_type(Ts, [RT | L]); + ?ANYTYPE -> + ?ANYTYPE + end; +range_type([], L) -> + list_to_tuple(reverse(L)). + +converse([{A,B} | X], L) -> + converse(X, [{B,A} | L]); +converse([], L) -> + sort(L). + +strict([{E1,E2} | Es], L) when E1 == E2 -> + strict(Es, L); +strict([E | Es], L) -> + strict(Es, [E | L]); +strict([], L) -> + reverse(L). + +weak(Es) -> + %% Not very efficient... + weak(Es, ran(Es, []), []). + +weak(Es=[{X,_} | _], [Y | Ys], L) when X > Y -> + weak(Es, Ys, [{Y,Y} | L]); +weak(Es=[{X,_} | _], [Y | Ys], L) when X == Y -> + weak(Es, Ys, L); +weak([E={X,Y} | Es], Ys, L) when X > Y -> + weak1(Es, Ys, [E | L], X); +weak([E={X,Y} | Es], Ys, L) when X == Y -> + weak2(Es, Ys, [E | L], X); +weak([E={X,_Y} | Es], Ys, L) -> % when X < _Y + weak2(Es, Ys, [E, {X,X} | L], X); +weak([], [Y | Ys], L) -> + weak([], Ys, [{Y,Y} | L]); +weak([], [], L) -> + reverse(L). + +weak1([E={X,Y} | Es], Ys, L, X0) when X > Y, X == X0 -> + weak1(Es, Ys, [E | L], X); +weak1([E={X,Y} | Es], Ys, L, X0) when X == Y, X == X0 -> + weak2(Es, Ys, [E | L], X); +weak1([E={X,_Y} | Es], Ys, L, X0) when X == X0 -> % when X < Y + weak2(Es, Ys, [E, {X,X} | L], X); +weak1(Es, Ys, L, X) -> + weak(Es, Ys, [{X,X} | L]). + +weak2([E={X,_Y} | Es], Ys, L, X0) when X == X0 -> % when X < _Y + weak2(Es, Ys, [E | L], X); +weak2(Es, Ys, L, _X) -> + weak(Es, Ys, L). + +extc(L, [D | Ds], C, Ts) -> + extc(L, Ds, C, Ts, D); +extc(L, [], _C, _Ts) -> + L. + +extc(L, Ds, C, [{X,_Y} | Ts], D) when X < D -> + extc(L, Ds, C, Ts, D); +extc(L, Ds, C, [{X,_Y} | Ts], D) when X == D -> + extc(L, Ds, C, Ts); +extc(L, Ds, C, [{X,_Y} | Ts], D) -> + extc2([{D,C} | L], Ds, C, Ts, X); +extc(L, Ds, C, [], D) -> + extc_tail([{D,C} | L], Ds, C). + +extc2(L, [D | Ds], C, Ts, X) when X > D -> + extc2([{D,C} | L], Ds, C, Ts, X); +extc2(L, [D | Ds], C, Ts, X) when X == D -> + extc(L, Ds, C, Ts); +extc2(L, [D | Ds], C, Ts, _X) -> + extc(L, Ds, C, Ts, D); +extc2(L, [], _C, _Ts, _X) -> + L. + +extc_tail(L, [D | Ds], C) -> + extc_tail([{D,C} | L], Ds, C); +extc_tail(L, [], _C) -> + L. + +is_a_func([{E,_} | Es], E0) when E /= E0 -> + is_a_func(Es, E); +is_a_func(L, _E) -> + L =:= []. + +restrict_n(I, [T | Ts], Key, Keys, L) -> + case element(I, T) of + K when K < Key -> + restrict_n(I, Ts, Key, Keys, L); + K when K == Key -> + restrict_n(I, Ts, Key, Keys, [T | L]); + K -> + restrict_n(I, K, Ts, Keys, L, T) + end; +restrict_n(_I, _Ts, _Key, _Keys, L) -> + L. + +restrict_n(I, K, Ts, [Key | Keys], L, E) when K > Key -> + restrict_n(I, K, Ts, Keys, L, E); +restrict_n(I, K, Ts, [Key | Keys], L, E) when K == Key -> + restrict_n(I, Ts, Key, Keys, [E | L]); +restrict_n(I, _K, Ts, [Key | Keys], L, _E) -> + restrict_n(I, Ts, Key, Keys, L); +restrict_n(_I, _K, _Ts, _Keys, L, _E) -> + L. + +restrict([Key | Keys], Tuples) -> + restrict(Tuples, Key, Keys, []); +restrict(_Keys, _Tuples) -> + []. + +restrict([{K,_E} | Ts], Key, Keys, L) when K < Key -> + restrict(Ts, Key, Keys, L); +restrict([{K,E} | Ts], Key, Keys, L) when K == Key -> + restrict(Ts, Key, Keys, [E | L]); +restrict([{K,E} | Ts], _Key, Keys, L) -> + restrict(Ts, K, Keys, L, E); +restrict(_Ts, _Key, _Keys, L) -> + L. + +restrict(Ts, K, [Key | Keys], L, E) when K > Key -> + restrict(Ts, K, Keys, L, E); +restrict(Ts, K, [Key | Keys], L, E) when K == Key -> + restrict(Ts, Key, Keys, [E | L]); +restrict(Ts, _K, [Key | Keys], L, _E) -> + restrict(Ts, Key, Keys, L); +restrict(_Ts, _K, _Keys, L, _E) -> + L. + +diff_restrict_n(I, [T | Ts], Key, Keys, L) -> + case element(I, T) of + K when K < Key -> + diff_restrict_n(I, Ts, Key, Keys, [T | L]); + K when K == Key -> + diff_restrict_n(I, Ts, Key, Keys, L); + K -> + diff_restrict_n(I, K, Ts, Keys, L, T) + end; +diff_restrict_n(I, _Ts, _Key, _Keys, L) when I =:= 1 -> + reverse(L); +diff_restrict_n(_I, _Ts, _Key, _Keys, L) -> + sort(L). + +diff_restrict_n(I, K, Ts, [Key | Keys], L, T) when K > Key -> + diff_restrict_n(I, K, Ts, Keys, L, T); +diff_restrict_n(I, K, Ts, [Key | Keys], L, _T) when K == Key -> + diff_restrict_n(I, Ts, Key, Keys, L); +diff_restrict_n(I, _K, Ts, [Key | Keys], L, T) -> + diff_restrict_n(I, Ts, Key, Keys, [T | L]); +diff_restrict_n(I, _K, Ts, _Keys, L, T) when I =:= 1 -> + reverse(L, [T | Ts]); +diff_restrict_n(_I, _K, Ts, _Keys, L, T) -> + sort([T | Ts ++ L]). + +diff_restrict([Key | Keys], Tuples) -> + diff_restrict(Tuples, Key, Keys, []); +diff_restrict(_Keys, Tuples) -> + diff_restrict_tail(Tuples, []). + +diff_restrict([{K,E} | Ts], Key, Keys, L) when K < Key -> + diff_restrict(Ts, Key, Keys, [E | L]); +diff_restrict([{K,_E} | Ts], Key, Keys, L) when K == Key -> + diff_restrict(Ts, Key, Keys, L); +diff_restrict([{K,E} | Ts], _Key, Keys, L) -> + diff_restrict(Ts, K, Keys, L, E); +diff_restrict(_Ts, _Key, _Keys, L) -> + L. + +diff_restrict(Ts, K, [Key | Keys], L, E) when K > Key -> + diff_restrict(Ts, K, Keys, L, E); +diff_restrict(Ts, K, [Key | Keys], L, _E) when K == Key -> + diff_restrict(Ts, Key, Keys, L); +diff_restrict(Ts, _K, [Key | Keys], L, E) -> + diff_restrict(Ts, Key, Keys, [E | L]); +diff_restrict(Ts, _K, _Keys, L, E) -> + diff_restrict_tail(Ts, [E | L]). + +diff_restrict_tail([{_K,E} | Ts], L) -> + diff_restrict_tail(Ts, [E | L]); +diff_restrict_tail(_Ts, L) -> + L. + +comp([], B) -> + check_function(B, []); +comp(_A, []) -> + bad_function; +comp(A0, [{Bx,By} | B]) -> + A = converse(A0, []), + check_function(A0, comp1(A, B, [], Bx, By)). + +comp1([{Ay,Ax} | A], B, L, Bx, By) when Ay == Bx -> + comp1(A, B, [{Ax,By} | L], Bx, By); +comp1([{Ay,Ax} | A], B, L, Bx, _By) when Ay > Bx -> + comp2(A, B, L, Bx, Ay, Ax); +comp1([{Ay,_Ax} | _A], _B, _L, Bx, _By) when Ay < Bx -> + bad_function; +comp1([], B, L, Bx, _By) -> + check_function(Bx, B, L). + +comp2(A, [{Bx,_By} | B], L, Bx0, Ay, Ax) when Ay > Bx, Bx /= Bx0 -> + comp2(A, B, L, Bx, Ay, Ax); +comp2(A, [{Bx,By} | B], L, _Bx0, Ay, Ax) when Ay == Bx -> + comp1(A, B, [{Ax,By} | L], Bx, By); +comp2(_A, _B, _L, _Bx0, _Ay, _Ax) -> + bad_function. + +inverse1([{A,B} | X]) -> + inverse(X, A, [{B,A}]); +inverse1([]) -> + []. + +inverse([{A,B} | X], A0, L) when A0 /= A -> + inverse(X, A, [{B,A} | L]); +inverse([{A,_B} | _X], A0, _L) when A0 == A -> + bad_function; +inverse([], _A0, L) -> + SL = [{V,_} | Es] = sort(L), + case is_a_func(Es, V) of + true -> SL; + false -> bad_function + end. + +%% Inlined. +external_fun({external, Function}) when is_atom(Function) -> + false; +external_fun({external, Fun}) -> + Fun; +external_fun(_) -> + false. + +%% Inlined. +element_type(?SET_OF(Type)) -> Type; +element_type(Type) -> Type. + +subst(Ts, Fun, Type) -> + subst(Ts, Fun, Type, ?ANYTYPE, []). + +subst([T | Ts], Fun, Type, NType, L) -> + case setfun(T, Fun, Type, NType) of + {SD, ST} -> subst(Ts, Fun, Type, ST, [{T, SD} | L]); + Bad -> Bad + end; +subst([], _Fun, _Type, NType, L) -> + {L, NType}. + +projection1([E | Es]) -> + projection1([], element(1, E), Es); +projection1([] = L) -> + L. + +projection1(L, X, [E | Es]) -> + case element(1, E) of + X1 when X == X1 -> projection1(L, X, Es); + X1 -> projection1([X | L], X1, Es) + end; +projection1(L, X, []) -> + reverse(L, [X]). + +projection_n([E | Es], I, L) -> + projection_n(Es, I, [element(I, E) | L]); +projection_n([], _I, L) -> + usort(L). + +substitute_element([T | Ts], I, L) -> + substitute_element(Ts, I, [{T, element(I, T)} | L]); +substitute_element(_, _I, L) -> + reverse(L). + +substitute([T | Ts], Fun, L) -> + substitute(Ts, Fun, [{T, Fun(T)} | L]); +substitute(_, _Fun, L) -> + reverse(L). + +partition_n(I, [E | Ts]) -> + partition_n(I, Ts, element(I, E), [E], []); +partition_n(_I, []) -> + []. + +partition_n(I, [E | Ts], K, Es, P) -> + case {element(I, E), Es} of + {K1, _} when K == K1 -> + partition_n(I, Ts, K, [E | Es], P); + {K1, [_]} -> % optimization + partition_n(I, Ts, K1, [E], [Es | P]); + {K1, _} -> + partition_n(I, Ts, K1, [E], [reverse(Es) | P]) + end; +partition_n(I, [], _K, Es, P) when I > 1 -> + sort([reverse(Es) | P]); +partition_n(_I, [], _K, [_] = Es, P) -> % optimization + reverse(P, [Es]); +partition_n(_I, [], _K, Es, P) -> + reverse(P, [reverse(Es)]). + +partition3_n(I, [T | Ts], Key, Keys, L1, L2) -> + case element(I, T) of + K when K < Key -> + partition3_n(I, Ts, Key, Keys, L1, [T | L2]); + K when K == Key -> + partition3_n(I, Ts, Key, Keys, [T | L1], L2); + K -> + partition3_n(I, K, Ts, Keys, L1, L2, T) + end; +partition3_n(I, _Ts, _Key, _Keys, L1, L2) when I =:= 1 -> + [reverse(L1) | reverse(L2)]; +partition3_n(_I, _Ts, _Key, _Keys, L1, L2) -> + [sort(L1) | sort(L2)]. + +partition3_n(I, K, Ts, [Key | Keys], L1, L2, T) when K > Key -> + partition3_n(I, K, Ts, Keys, L1, L2, T); +partition3_n(I, K, Ts, [Key | Keys], L1, L2, T) when K == Key -> + partition3_n(I, Ts, Key, Keys, [T | L1], L2); +partition3_n(I, _K, Ts, [Key | Keys], L1, L2, T) -> + partition3_n(I, Ts, Key, Keys, L1, [T | L2]); +partition3_n(I, _K, Ts, _Keys, L1, L2, T) when I =:= 1 -> + [reverse(L1) | reverse(L2, [T | Ts])]; +partition3_n(_I, _K, Ts, _Keys, L1, L2, T) -> + [sort(L1) | sort([T | Ts ++ L2])]. + +partition3([Key | Keys], Tuples) -> + partition3(Tuples, Key, Keys, [], []); +partition3(_Keys, Tuples) -> + partition3_tail(Tuples, [], []). + +partition3([{K,E} | Ts], Key, Keys, L1, L2) when K < Key -> + partition3(Ts, Key, Keys, L1, [E | L2]); +partition3([{K,E} | Ts], Key, Keys, L1, L2) when K == Key -> + partition3(Ts, Key, Keys, [E | L1], L2); +partition3([{K,E} | Ts], _Key, Keys, L1, L2) -> + partition3(Ts, K, Keys, L1, L2, E); +partition3(_Ts, _Key, _Keys, L1, L2) -> + [L1 | L2]. + +partition3(Ts, K, [Key | Keys], L1, L2, E) when K > Key -> + partition3(Ts, K, Keys, L1, L2, E); +partition3(Ts, K, [Key | Keys], L1, L2, E) when K == Key -> + partition3(Ts, Key, Keys, [E | L1], L2); +partition3(Ts, _K, [Key | Keys], L1, L2, E) -> + partition3(Ts, Key, Keys, L1, [E | L2]); +partition3(Ts, _K, _Keys, L1, L2, E) -> + partition3_tail(Ts, L1, [E | L2]). + +partition3_tail([{_K,E} | Ts], L1, L2) -> + partition3_tail(Ts, L1, [E | L2]); +partition3_tail(_Ts, L1, L2) -> + [L1 | L2]. + +replace([E | Es], F, L) -> + replace(Es, F, [F(E) | L]); +replace(_, _F, L) -> + sort(L). + +mul_relprod([T | Ts], I, R) when ?IS_SET(T) -> + P = raise_element(R, I), + F = relative_product1(P, T), + [F | mul_relprod(Ts, I+1, R)]; +mul_relprod([], _I, _R) -> + []. + +raise_element(R, I) -> + L = sort(I =/= 1, rearr(?LIST(R), I, [])), + Type = ?TYPE(R), + ?SET(L, ?BINREL(?REL_TYPE(I, Type), Type)). + +rearr([E | Es], I, L) -> + rearr(Es, I, [{element(I, E), E} | L]); +rearr([], _I, L) -> + L. + +join_element(E1, E2) -> + [_ | L2] = tuple_to_list(E2), + list_to_tuple(tuple_to_list(E1) ++ L2). + +join_element(E1, E2, I2) -> + tuple_to_list(E1) ++ join_element2(tuple_to_list(E2), 1, I2). + +join_element2([B | Bs], C, I2) when C =/= I2 -> + [B | join_element2(Bs, C+1, I2)]; +join_element2([_ | Bs], _C, _I2) -> + Bs. + +family2rel([{X,S} | F], L) -> + fam2rel(F, L, X, S); +family2rel([], L) -> + reverse(L). + +fam2rel(F, L, X, [Y | Ys]) -> + fam2rel(F, [{X,Y} | L], X, Ys); +fam2rel(F, L, _X, _) -> + family2rel(F, L). + +fam_spec([{_,S}=E | F], Fun, Type, L) -> + case Fun(?SET(S, Type)) of + true -> + fam_spec(F, Fun, Type, [E | L]); + false -> + fam_spec(F, Fun, Type, L); + _ -> + badarg + end; +fam_spec([], _Fun, _Type, L) -> + reverse(L). + +fam_specification([{_,S}=E | F], Fun, L) -> + case Fun(S) of + true -> + fam_specification(F, Fun, [E | L]); + false -> + fam_specification(F, Fun, L); + _ -> + badarg + end; +fam_specification([], _Fun, L) -> + reverse(L). + +un_of_fam([{_X,S} | F], L) -> + un_of_fam(F, [S | L]); +un_of_fam([], L) -> + lunion(sort(L)). + +int_of_fam([{_,S} | F]) -> + int_of_fam(F, [S]); +int_of_fam([]) -> + badarg. + +int_of_fam([{_,S} | F], L) -> + int_of_fam(F, [S | L]); +int_of_fam([], [L | Ls]) -> + lintersection(Ls, L). + +fam_un([{X,S} | F], L) -> + fam_un(F, [{X, lunion(S)} | L]); +fam_un([], L) -> + reverse(L). + +fam_int([{X, [S | Ss]} | F], L) -> + fam_int(F, [{X, lintersection(Ss, S)} | L]); +fam_int([{_X,[]} | _F], _L) -> + badarg; +fam_int([], L) -> + reverse(L). + +fam_dom([{X,S} | F], L) -> + fam_dom(F, [{X, dom(S)} | L]); +fam_dom([], L) -> + reverse(L). + +fam_ran([{X,S} | F], L) -> + fam_ran(F, [{X, ran(S, [])} | L]); +fam_ran([], L) -> + reverse(L). + +fam_union(F1 = [{A,_AS} | _AL], [B1={B,_BS} | BL], L) when A > B -> + fam_union(F1, BL, [B1 | L]); +fam_union([{A,AS} | AL], [{B,BS} | BL], L) when A == B -> + fam_union(AL, BL, [{A, umerge(AS, BS)} | L]); +fam_union([A1 | AL], F2, L) -> + fam_union(AL, F2, [A1 | L]); +fam_union(_, F2, L) -> + reverse(L, F2). + +fam_intersect(F1 = [{A,_AS} | _AL], [{B,_BS} | BL], L) when A > B -> + fam_intersect(F1, BL, L); +fam_intersect([{A,AS} | AL], [{B,BS} | BL], L) when A == B -> + fam_intersect(AL, BL, [{A, intersection(AS, BS, [])} | L]); +fam_intersect([_A1 | AL], F2, L) -> + fam_intersect(AL, F2, L); +fam_intersect(_, _, L) -> + reverse(L). + +fam_difference(F1 = [{A,_AS} | _AL], [{B,_BS} | BL], L) when A > B -> + fam_difference(F1, BL, L); +fam_difference([{A,AS} | AL], [{B,BS} | BL], L) when A == B -> + fam_difference(AL, BL, [{A, difference(AS, BS, [])} | L]); +fam_difference([A1 | AL], F2, L) -> + fam_difference(AL, F2, [A1 | L]); +fam_difference(F1, _, L) -> + reverse(L, F1). + +check_function([{X,_} | XL], R) -> + check_function(X, XL, R); +check_function([], R) -> + R. + +check_function(X0, [{X,_} | XL], R) when X0 /= X -> + check_function(X, XL, R); +check_function(X0, [{X,_} | _XL], _R) when X0 == X -> + bad_function; +check_function(_X0, [], R) -> + R. + +fam_partition_n(I, [E | Ts]) -> + fam_partition_n(I, Ts, element(I, E), [E], []); +fam_partition_n(_I, []) -> + []. + +fam_partition_n(I, [E | Ts], K, Es, P) -> + case {element(I, E), Es} of + {K1, _} when K == K1 -> + fam_partition_n(I, Ts, K, [E | Es], P); + {K1, [_]} -> % optimization + fam_partition_n(I, Ts, K1, [E], [{K,Es} | P]); + {K1, _} -> + fam_partition_n(I, Ts, K1, [E], [{K,reverse(Es)} | P]) + end; +fam_partition_n(_I, [], K, [_] = Es, P) -> % optimization + reverse(P, [{K,Es}]); +fam_partition_n(_I, [], K, Es, P) -> + reverse(P, [{K,reverse(Es)}]). + +fam_partition([{K,Vs} | Ts], Sort) -> + fam_partition(Ts, K, [Vs], [], Sort); +fam_partition([], _Sort) -> + []. + +fam_partition([{K1,V} | Ts], K, Vs, P, S) when K1 == K -> + fam_partition(Ts, K, [V | Vs], P, S); +fam_partition([{K1,V} | Ts], K, [_] = Vs, P, S) -> % optimization + fam_partition(Ts, K1, [V], [{K, Vs} | P], S); +fam_partition([{K1,V} | Ts], K, Vs, P, S) -> + fam_partition(Ts, K1, [V], [{K, sort(S, Vs)} | P], S); +fam_partition([], K, [_] = Vs, P, _S) -> % optimization + [{K, Vs} | P]; +fam_partition([], K, Vs, P, S) -> + [{K, sort(S, Vs)} | P]. + +fam_proj([{X,S} | F], Fun, Type, NType, L) -> + case setfun(S, Fun, Type, NType) of + {SD, ST} -> fam_proj(F, Fun, Type, ST, [{X, SD} | L]); + Bad -> Bad + end; +fam_proj([], _Fun, _Type, NType, L) -> + {reverse(L), NType}. + +setfun(T, Fun, Type, NType) -> + case Fun(term2set(T, Type)) of + NS when ?IS_SET(NS) -> + case unify_types(NType, ?SET_OF(?TYPE(NS))) of + [] -> type_mismatch; + NT -> {?LIST(NS), NT} + end; + NS when ?IS_ORDSET(NS) -> + case unify_types(NType, NT = ?ORDTYPE(NS)) of + [] -> type_mismatch; + NT -> {?ORDDATA(NS), NT} + end; + _ -> + badarg + end. + +%% Inlined. +term2set(L, Type) when is_list(L) -> + ?SET(L, Type); +term2set(T, Type) -> + ?ORDSET(T, Type). + +fam2digraph(F, G) -> + Fun = fun({From, ToL}) -> + digraph:add_vertex(G, From), + Fun2 = fun(To) -> + digraph:add_vertex(G, To), + case digraph:add_edge(G, From, To) of + {error, {bad_edge, _}} -> + throw({error, cyclic}); + _ -> + true + end + end, + foreach(Fun2, ToL) + end, + foreach(Fun, to_external(F)), + G. + +digraph_family(G) -> + Vs = sort(digraph:vertices(G)), + digraph_fam(Vs, Vs, G, []). + +digraph_fam([V | Vs], V0, G, L) when V /= V0 -> + Ns = sort(digraph:out_neighbours(G, V)), + digraph_fam(Vs, V, G, [{V,Ns} | L]); +digraph_fam([], _V0, _G, L) -> + reverse(L). + +%% -> boolean() +check_fun(T, F, FunT) -> + true = is_type(FunT), + {NT, _MaxI} = number_tuples(T, 1), + L = flatten(tuple2list(F(NT))), + has_hole(L, 1). + +number_tuples(T, N) when is_tuple(T) -> + {L, NN} = mapfoldl(fun number_tuples/2, N, tuple_to_list(T)), + {list_to_tuple(L), NN}; +number_tuples(_, N) -> + {N, N+1}. + +tuple2list(T) when is_tuple(T) -> + map(fun tuple2list/1, tuple_to_list(T)); +tuple2list(C) -> + [C]. + +has_hole([I | Is], I0) when I =< I0 -> has_hole(Is, erlang:max(I+1, I0)); +has_hole(Is, _I) -> Is =/= []. + +%% Optimization. Same as check_fun/3, but for integers. +check_for_sort(T, _I) when T =:= ?ANYTYPE -> + empty; +check_for_sort(T, I) when ?IS_RELATION(T), I =< ?REL_ARITY(T), I >= 1 -> + I > 1; +check_for_sort(_T, _I) -> + error. + +inverse_substitution(L, Fun, Sort) -> + %% One easily sees that the inverse of the tuples created by + %% applying Fun need to be sorted iff the tuples created by Fun + %% need to be sorted. + sort(Sort, fun_rearr(L, Fun, [])). + +fun_rearr([E | Es], Fun, L) -> + fun_rearr(Es, Fun, [{Fun(E), E} | L]); +fun_rearr([], _Fun, L) -> + L. + +sets_to_list(Ss) -> + map(fun(S) when ?IS_SET(S) -> ?LIST(S) end, Ss). + +types([], L) -> + list_to_tuple(reverse(L)); +types([S | _Ss], _L) when ?TYPE(S) =:= ?ANYTYPE -> + ?ANYTYPE; +types([S | Ss], L) -> + types(Ss, [?TYPE(S) | L]). + +%% Inlined. +unify_types(T, T) -> T; +unify_types(Type1, Type2) -> + catch unify_types1(Type1, Type2). + +unify_types1(Atom, Atom) when ?IS_ATOM_TYPE(Atom) -> + Atom; +unify_types1(?ANYTYPE, Type) -> + Type; +unify_types1(Type, ?ANYTYPE) -> + Type; +unify_types1(?SET_OF(Type1), ?SET_OF(Type2)) -> + [unify_types1(Type1, Type2)]; +unify_types1(T1, T2) when tuple_size(T1) =:= tuple_size(T2) -> + unify_typesl(tuple_size(T1), T1, T2, []); +unify_types1(_T1, _T2) -> + throw([]). + +unify_typesl(0, _T1, _T2, L) -> + list_to_tuple(L); +unify_typesl(N, T1, T2, L) -> + T = unify_types1(?REL_TYPE(N, T1), ?REL_TYPE(N, T2)), + unify_typesl(N-1, T1, T2, [T | L]). + +%% inlined. +match_types(T, T) -> true; +match_types(Type1, Type2) -> match_types1(Type1, Type2). + +match_types1(Atom, Atom) when ?IS_ATOM_TYPE(Atom) -> + true; +match_types1(?ANYTYPE, _) -> + true; +match_types1(_, ?ANYTYPE) -> + true; +match_types1(?SET_OF(Type1), ?SET_OF(Type2)) -> + match_types1(Type1, Type2); +match_types1(T1, T2) when tuple_size(T1) =:= tuple_size(T2) -> + match_typesl(tuple_size(T1), T1, T2); +match_types1(_T1, _T2) -> + false. + +match_typesl(0, _T1, _T2) -> + true; +match_typesl(N, T1, T2) -> + case match_types1(?REL_TYPE(N, T1), ?REL_TYPE(N, T2)) of + true -> match_typesl(N-1, T1, T2); + false -> false + end. + +sort(true, L) -> + sort(L); +sort(false, L) -> + reverse(L). diff --git a/lib/tools/test/tools_bench.spec b/lib/tools/test/tools_bench.spec new file mode 100644 index 0000000000..ef08fd68a8 --- /dev/null +++ b/lib/tools/test/tools_bench.spec @@ -0,0 +1 @@ +{suites,"../tools_test",[prof_bench_SUITE]}. diff --git a/lib/wx/doc/overview.edoc b/lib/wx/doc/overview.edoc index 054016f515..843a9c1320 100644 --- a/lib/wx/doc/overview.edoc +++ b/lib/wx/doc/overview.edoc @@ -218,7 +218,7 @@ the <em>fun</em> returns. The callbacks are always invoked by another process and have exclusive usage of the GUI when invoked. This means that a callback <em>fun</em> -can not use the process dictionary and should not make calls to other +cannot use the process dictionary and should not make calls to other processes. Calls to another process inside a callback <em>fun</em> may cause a deadlock if the other process is waiting on completion of his call to the GUI. diff --git a/lib/wx/examples/demo/ex_graphicsContext.erl b/lib/wx/examples/demo/ex_graphicsContext.erl index 1193578037..1e6ffce18d 100644 --- a/lib/wx/examples/demo/ex_graphicsContext.erl +++ b/lib/wx/examples/demo/ex_graphicsContext.erl @@ -74,7 +74,7 @@ do_init(Config) -> pen = Pen, brush = Brush, font = Font}}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Sync events i.e. from callbacks must return ok, it can not return a new state. +%% Sync events i.e. from callbacks must return ok, it cannot return a new state. %% Do the redrawing here. handle_sync_event(#wx{event = #wxPaint{}},_, #state{win=Win, pen = Pen, brush = Brush, font = Font}) -> diff --git a/lib/wx/examples/simple/hello2.erl b/lib/wx/examples/simple/hello2.erl index 656c056d9a..07a9a56b7d 100644 --- a/lib/wx/examples/simple/hello2.erl +++ b/lib/wx/examples/simple/hello2.erl @@ -33,7 +33,7 @@ init/1, handle_info/2, handle_event/2, handle_call/3, code_change/3, terminate/2]). --behavoiur(wx_object). +-behaviour(wx_object). -record(state, {win}). diff --git a/lib/wx/src/Makefile b/lib/wx/src/Makefile index 21b45af2c4..52f4008e0a 100644 --- a/lib/wx/src/Makefile +++ b/lib/wx/src/Makefile @@ -111,10 +111,10 @@ $(APPUP_TARGET): $(APPUP_SRC) ../vsn.mk Makefile # Rules $(EBIN)/%.beam: $(ESRC)/%.erl $(HEADER_FILES) - $(V_ERLC) -W -bbeam $(ERL_FLAGS) $(ERL_COMPILE_FLAGS) -o$(EBIN) $< + $(V_ERLC) -W -bbeam $(ERL_COMPILE_FLAGS) -o$(EBIN) $< $(EBIN)/%.beam: $(EGEN)/%.erl $(HEADER_FILES) - $(V_ERLC) -W -bbeam $(ERL_FLAGS) $(ERL_COMPILE_FLAGS) -o$(EBIN) $< + $(V_ERLC) -W -bbeam $(ERL_COMPILE_FLAGS) -o$(EBIN) $< # ---------------------------------------------------- # Release Target diff --git a/lib/wx/test/wx_event_SUITE.erl b/lib/wx/test/wx_event_SUITE.erl index 7ecd9b7283..1cc194d569 100644 --- a/lib/wx/test/wx_event_SUITE.erl +++ b/lib/wx/test/wx_event_SUITE.erl @@ -347,9 +347,9 @@ connect_in_callback(Config) -> %% such that new events are not fired until the previous %% callback have returned. - %% That means that a callback can not wait for other events + %% That means that a callback cannot wait for other events %% in receive since they will not come. - %% It also means that you can not attach a new callback directly from + %% It also means that you cannot attach a new callback directly from %% the callback since that callback will be removed when the temporary %% process that executes the outer callback (may) die(s) before the callback %% is invoked diff --git a/lib/xmerl/src/xmerl_sax_parser_base.erlsrc b/lib/xmerl/src/xmerl_sax_parser_base.erlsrc index 1dca9608cb..ef753c7148 100644 --- a/lib/xmerl/src/xmerl_sax_parser_base.erlsrc +++ b/lib/xmerl/src/xmerl_sax_parser_base.erlsrc @@ -3679,7 +3679,7 @@ create_tempfile(Template) -> false -> case os:getenv("TEMP") of false -> - throw({error, "Variabel TMP or TEMP doesn't exist"}); + throw({error, "Variable TMP or TEMP doesn't exist"}); P2 -> P2 end; diff --git a/lib/xmerl/test/xmerl_SUITE_data/eventp/CMOM.xml b/lib/xmerl/test/xmerl_SUITE_data/eventp/CMOM.xml index c2533248d1..0379c18214 100644 --- a/lib/xmerl/test/xmerl_SUITE_data/eventp/CMOM.xml +++ b/lib/xmerl/test/xmerl_SUITE_data/eventp/CMOM.xml @@ -276,7 +276,7 @@ </exception> <exception name="UpgradeNotPossibleException"> - <description>Before an upgrade is started it was found that the upgrade can not take place. A possible reason is that the upgrade package that is running in the node is not in the upgrade window of this upgrade package. </description> + <description>Before an upgrade is started it was found that the upgrade cannot take place. A possible reason is that the upgrade package that is running in the node is not in the upgrade window of this upgrade package. </description> <exceptionParameter name="message"> <dataType> <string/> @@ -1703,7 +1703,7 @@ Example: <exception name="NoSuchAttributeException"> <description>Exception thrown when an MO attribute is requested to be accessed but the access -method for the is not defined (the attribute can not be accessed)</description> +method for the is not defined (the attribute cannot be accessed)</description> </exception> <exception name="MoCardinalityViolationException"> @@ -3125,7 +3125,7 @@ active = Synchronization is used in system clock generation. <description>RefState can have the following values: failed = synchronization reference is not capable to perform its required tasks. degraded = capability of synchronization reference to perform its required tasks is degraded e.g. because of signal level degradation. This value is only applicable for traffic carrying (ET physical path termination) synchronization references. Note: attribute degradationIsFault controls whether synchronization reference degradation is interpreted as a synchronization reference fault or not. -lossOfTracking = system clock regulation algorithm on TU board can not follow the 8kHz synchronization reference signal either because of the poor quality of the signal or because of a HW fault at TU board. If all synchronization references repeatedly end up to state lossOfTracking, fault is likely in TU HW. +lossOfTracking = system clock regulation algorithm on TU board cannot follow the 8kHz synchronization reference signal either because of the poor quality of the signal or because of a HW fault at TU board. If all synchronization references repeatedly end up to state lossOfTracking, fault is likely in TU HW. ok = synchronization reference is capable of performing its required tasks. </description> <enumMember name="failed"> @@ -11270,7 +11270,7 @@ Note! This action requires a transaction.</description> The changing of the IP address with operation assignIpAddress might cause interruption of the communication if the network management tool is connected via the ethernet link. -Note: The EthernetLink MO can not be deleted! +Note: The EthernetLink MO cannot be deleted! Note: The performance monitoring counters in the EthernetLink MO has a "Wrap-around time" of approximately 2 hours. </description> @@ -11532,8 +11532,8 @@ Note! This action requires a transaction.</description> <description>This MO holds the IP routing table. The IpRoutingTable MO is automatically created when the Ip MO is created. -The IpRoutingTable MO can not be created manually. -The IpRoutingTable MO can not be deleted. +The IpRoutingTable MO cannot be created manually. +The IpRoutingTable MO cannot be deleted. </description> <systemCreated/> <attribute name="userLabel"> @@ -12967,7 +12967,7 @@ Note! When using fractional atm, timeslot 1 must be a part of the fraction. Note! ETM1 does not support EPD and PPD -Note! ETM1 does not have a proper buffer management. Thus fairness of UBR+ traffic can not be guaranteed and shaping on UBR+ traffic is not possible. +Note! ETM1 does not have a proper buffer management. Thus fairness of UBR+ traffic cannot be guaranteed and shaping on UBR+ traffic is not possible. Note! The number of VCC TP+VPC TP with performance monitoring enabled (i.e. PM mode <> off) is restricted to 1 per port. </description> @@ -13371,7 +13371,7 @@ Struct element description : -Date is in string format, max length is 40. Format for date is: weekday month date hour:min:seconds year. -Status is in string format, max length is 40. -Note! The identity can not be the same as name. Identity should contain the product identity. +Note! The identity cannot be the same as name. Identity should contain the product identity. </description> @@ -13562,7 +13562,7 @@ Note! For comment and operatorName spaces (' ') are also allowed within the stri This action does not require a transaction. -Note! The configurationVersionName and identity can not be the same. The identity should be the product identity. +Note! The configurationVersionName and identity cannot be the same. The identity should be the product identity. </description> <returnType> <void/> @@ -17367,7 +17367,7 @@ NOTE: There is a restriction of a maximum of 32 Mtp2Tp's per MP.</description> <attribute name="mtp2ProfileItuId"> <description>Reference to a Mtp2ProfileItu MO. -Note: The bitRate can not be changed.</description> +Note: The bitRate cannot be changed.</description> <mandatory/> <noNotification/> <dataType> @@ -17566,7 +17566,7 @@ NOTE: There is a restriction of a maximum of 32 Mtp2Tp's per MP.</description> <attribute name="mtp2ProfileAnsiId"> <description>Reference to a Mtp2ProfileAnsi MO. -Note: The bitRate can not be changed.</description> +Note: The bitRate cannot be changed.</description> <mandatory/> <noNotification/> <dataType> @@ -21777,7 +21777,7 @@ Each E1/DS1/J1 channel can suport up to 2 VP connections. The ET-MC41 board support one biderectional F4/F5 PM flow per E1 channel. -Note! The ETMC41 supports IMA. However the E1 ports being part of the same IMA group can not be selected randomly. +Note! The ETMC41 supports IMA. However the E1 ports being part of the same IMA group cannot be selected randomly. Note! The number of VCC TP+VPC TP with performance monitoring enabled (i.e. PM mode <> off) is restricted to 1 / port @@ -21883,7 +21883,7 @@ NOTE: There is a restriction of a maximum of 32 Mtp2Tp's per MP. <attribute name="mtp2ProfileChinaId"> <description>Reference to a Mtp2ProfileChina MO. -Note: The bitRate can not be changed.</description> +Note: The bitRate cannot be changed.</description> <mandatory/> <noNotification/> <dataType> diff --git a/lib/xmerl/test/xmerl_SUITE_data/eventp/CelloMOM.xml b/lib/xmerl/test/xmerl_SUITE_data/eventp/CelloMOM.xml index 3b5d8ae2ad..3b9ccac0f4 100644 --- a/lib/xmerl/test/xmerl_SUITE_data/eventp/CelloMOM.xml +++ b/lib/xmerl/test/xmerl_SUITE_data/eventp/CelloMOM.xml @@ -276,7 +276,7 @@ </exception>
<exception name="UpgradeNotPossibleException">
- <description>Before an upgrade is started it was found that the upgrade can not take place. A possible reason is that the upgrade package that is running in the node is not in the upgrade window of this upgrade package. </description>
+ <description>Before an upgrade is started it was found that the upgrade cannot take place. A possible reason is that the upgrade package that is running in the node is not in the upgrade window of this upgrade package. </description>
<exceptionParameter name="message">
<dataType>
<string/>
@@ -1703,7 +1703,7 @@ Example: <exception name="NoSuchAttributeException">
<description>Exception thrown when an MO attribute is requested to be accessed but the access
-method for the is not defined (the attribute can not be accessed)</description>
+method for the is not defined (the attribute cannot be accessed)</description>
</exception>
<exception name="MoCardinalityViolationException">
@@ -3125,7 +3125,7 @@ active = Synchronization is used in system clock generation. <description>RefState can have the following values:
failed = synchronization reference is not capable to perform its required tasks.
degraded = capability of synchronization reference to perform its required tasks is degraded e.g. because of signal level degradation. This value is only applicable for traffic carrying (ET physical path termination) synchronization references. Note: attribute degradationIsFault controls whether synchronization reference degradation is interpreted as a synchronization reference fault or not.
-lossOfTracking = system clock regulation algorithm on TU board can not follow the 8kHz synchronization reference signal either because of the poor quality of the signal or because of a HW fault at TU board. If all synchronization references repeatedly end up to state lossOfTracking, fault is likely in TU HW.
+lossOfTracking = system clock regulation algorithm on TU board cannot follow the 8kHz synchronization reference signal either because of the poor quality of the signal or because of a HW fault at TU board. If all synchronization references repeatedly end up to state lossOfTracking, fault is likely in TU HW.
ok = synchronization reference is capable of performing its required tasks.
</description>
<enumMember name="failed">
@@ -11270,7 +11270,7 @@ Note! This action requires a transaction.</description> The changing of the IP address with operation assignIpAddress might cause interruption of the communication if the network management tool is connected via the ethernet link.
-Note: The EthernetLink MO can not be deleted!
+Note: The EthernetLink MO cannot be deleted!
Note: The performance monitoring counters in the EthernetLink MO has a "Wrap-around time" of approximately 2 hours.
</description>
@@ -11532,8 +11532,8 @@ Note! This action requires a transaction.</description> <description>This MO holds the IP routing table.
The IpRoutingTable MO is automatically created when the Ip MO is created.
-The IpRoutingTable MO can not be created manually.
-The IpRoutingTable MO can not be deleted.
+The IpRoutingTable MO cannot be created manually.
+The IpRoutingTable MO cannot be deleted.
</description>
<systemCreated/>
<attribute name="userLabel">
@@ -12967,7 +12967,7 @@ Note! When using fractional atm, timeslot 1 must be a part of the fraction. Note! ETM1 does not support EPD and PPD
-Note! ETM1 does not have a proper buffer management. Thus fairness of UBR+ traffic can not be guaranteed and shaping on UBR+ traffic is not possible.
+Note! ETM1 does not have a proper buffer management. Thus fairness of UBR+ traffic cannot be guaranteed and shaping on UBR+ traffic is not possible.
Note! The number of VCC TP+VPC TP with performance monitoring enabled (i.e. PM mode <> off) is restricted to 1 per port.
</description>
@@ -13371,7 +13371,7 @@ Struct element description : -Date is in string format, max length is 40. Format for date is: weekday month date hour:min:seconds year.
-Status is in string format, max length is 40.
-Note! The identity can not be the same as name. Identity should contain the product identity.
+Note! The identity cannot be the same as name. Identity should contain the product identity.
</description>
@@ -13562,7 +13562,7 @@ Note! For comment and operatorName spaces (' ') are also allowed within the stri This action does not require a transaction.
-Note! The configurationVersionName and identity can not be the same. The identity should be the product identity.
+Note! The configurationVersionName and identity cannot be the same. The identity should be the product identity.
</description>
<returnType>
<void/>
@@ -17367,7 +17367,7 @@ NOTE: There is a restriction of a maximum of 32 Mtp2Tp's per MP.</description> <attribute name="mtp2ProfileItuId">
<description>Reference to a Mtp2ProfileItu MO.
-Note: The bitRate can not be changed.</description>
+Note: The bitRate cannot be changed.</description>
<mandatory/>
<noNotification/>
<dataType>
@@ -17566,7 +17566,7 @@ NOTE: There is a restriction of a maximum of 32 Mtp2Tp's per MP.</description> <attribute name="mtp2ProfileAnsiId">
<description>Reference to a Mtp2ProfileAnsi MO.
-Note: The bitRate can not be changed.</description>
+Note: The bitRate cannot be changed.</description>
<mandatory/>
<noNotification/>
<dataType>
@@ -21777,7 +21777,7 @@ Each E1/DS1/J1 channel can suport up to 2 VP connections. The ET-MC41 board support one biderectional F4/F5 PM flow per E1 channel.
-Note! The ETMC41 supports IMA. However the E1 ports being part of the same IMA group can not be selected randomly.
+Note! The ETMC41 supports IMA. However the E1 ports being part of the same IMA group cannot be selected randomly.
Note! The number of VCC TP+VPC TP with performance monitoring enabled (i.e. PM mode <> off) is restricted to 1 / port
@@ -21883,7 +21883,7 @@ NOTE: There is a restriction of a maximum of 32 Mtp2Tp's per MP. <attribute name="mtp2ProfileChinaId">
<description>Reference to a Mtp2ProfileChina MO.
-Note: The bitRate can not be changed.</description>
+Note: The bitRate cannot be changed.</description>
<mandatory/>
<noNotification/>
<dataType>
diff --git a/lib/xmerl/test/xmerl_xsd_lib.erl b/lib/xmerl/test/xmerl_xsd_lib.erl index 6006cf500f..39300cd6cb 100644 --- a/lib/xmerl/test/xmerl_xsd_lib.erl +++ b/lib/xmerl/test/xmerl_xsd_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2016. All Rights Reserved. +%% Copyright Ericsson AB 2006-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -94,8 +94,9 @@ return_results2({NewFail, NewSuccess, NewMal, NewNotMal}, NumSucc, SkippedN, Tot _ -> io_lib:format("These ~p skipped tests were malicious, but succeeds now: ~p~n", [length(NewNotMal), NewNotMal]) end, - ct:comment(io_lib:format("~p successful tests, ~p skipped tests of totally ~p test cases. ~n" ++ - NFComm ++ NSComm ++ NMComm ++ NNMComm, [NumSucc, SkippedN, TotN])), + ct:comment(io_lib:format("~p successful tests, ~p skipped tests of totally ~p test cases. ~n~ts", + [NumSucc, SkippedN, TotN, + NFComm ++ NSComm ++ NMComm ++ NNMComm])), [] = NewFail. %% return_results2(Diff,{STErrs,STOther},{ITErrs,ITOther},TotN) -> |