diff options
55 files changed, 917 insertions, 1468 deletions
diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex 76fdce52f9..d0ac42d595 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex 76fdce52f9..d0ac42d595 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/bootstrap/lib/compiler/ebin/compile.beam b/bootstrap/lib/compiler/ebin/compile.beam Binary files differindex bead5a1399..3294f7218d 100644 --- a/bootstrap/lib/compiler/ebin/compile.beam +++ b/bootstrap/lib/compiler/ebin/compile.beam diff --git a/bootstrap/lib/compiler/ebin/sys_expand_pmod.beam b/bootstrap/lib/compiler/ebin/sys_expand_pmod.beam Binary files differindex 5142cdde01..fbe8079575 100644 --- a/bootstrap/lib/compiler/ebin/sys_expand_pmod.beam +++ b/bootstrap/lib/compiler/ebin/sys_expand_pmod.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_lint.beam b/bootstrap/lib/stdlib/ebin/erl_lint.beam Binary files differindex 7b1eaf1f15..55ec220d55 100644 --- a/bootstrap/lib/stdlib/ebin/erl_lint.beam +++ b/bootstrap/lib/stdlib/ebin/erl_lint.beam diff --git a/bootstrap/lib/stdlib/ebin/otp_internal.beam b/bootstrap/lib/stdlib/ebin/otp_internal.beam Binary files differindex f22f936f42..6f21db8a48 100644 --- a/bootstrap/lib/stdlib/ebin/otp_internal.beam +++ b/bootstrap/lib/stdlib/ebin/otp_internal.beam diff --git a/bootstrap/lib/stdlib/ebin/regexp.beam b/bootstrap/lib/stdlib/ebin/regexp.beam Binary files differdeleted file mode 100644 index 023f2fb9b2..0000000000 --- a/bootstrap/lib/stdlib/ebin/regexp.beam +++ /dev/null diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index 4d0c87bf12..08d2066da3 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -61,7 +61,7 @@ MODULES= \ exception_SUITE \ float_SUITE \ fun_SUITE \ - fun_r12_SUITE \ + fun_r13_SUITE \ gc_SUITE \ guard_SUITE \ hash_SUITE \ diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 19281f6d58..08308629fe 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -18,7 +18,7 @@ %% -module(distribution_SUITE). --compile(r12). +-compile(r13). %% Tests distribution and the tcp driver. @@ -37,7 +37,7 @@ dist_auto_connect_never/1, dist_auto_connect_once/1, dist_parallel_send/1, atom_roundtrip/1, - atom_roundtrip_r12b/1, + atom_roundtrip_r13b/1, contended_atom_cache_entry/1, bad_dist_structure/1, bad_dist_ext_receive/1, @@ -62,7 +62,7 @@ all() -> link_to_dead_new_node, applied_monitor_node, ref_port_roundtrip, nil_roundtrip, stop_dist, {group, trap_bif}, {group, dist_auto_connect}, - dist_parallel_send, atom_roundtrip, atom_roundtrip_r12b, + dist_parallel_send, atom_roundtrip, atom_roundtrip_r13b, contended_atom_cache_entry, bad_dist_structure, {group, bad_dist_ext}]. groups() -> @@ -1100,17 +1100,17 @@ atom_roundtrip(Config) when is_list(Config) -> ?line stop_node(Node), ?line ok. -atom_roundtrip_r12b(Config) when is_list(Config) -> - case ?t:is_release_available("r12b") of +atom_roundtrip_r13b(Config) when is_list(Config) -> + case ?t:is_release_available("r13b") of true -> ?line AtomData = atom_data(), ?line verify_atom_data(AtomData), - ?line {ok, Node} = start_node(Config, [], "r12b"), + ?line {ok, Node} = start_node(Config, [], "r13b"), ?line do_atom_roundtrip(Node, AtomData), ?line stop_node(Node), ?line ok; false -> - ?line {skip,"No OTP R12B available"} + ?line {skip,"No OTP R13B available"} end. do_atom_roundtrip(Node, AtomData) -> diff --git a/erts/emulator/test/fun_r12_SUITE.erl b/erts/emulator/test/fun_r13_SUITE.erl index 3b1dfc9825..76ddf9fec9 100644 --- a/erts/emulator/test/fun_r12_SUITE.erl +++ b/erts/emulator/test/fun_r13_SUITE.erl @@ -17,10 +17,10 @@ %% %CopyrightEnd% %% --module(fun_r12_SUITE). --compile(r12). +-module(fun_r13_SUITE). +-compile(r13). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2,dist_old_release/1]). @@ -29,10 +29,10 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> +all() -> [dist_old_release]. -groups() -> +groups() -> []. init_per_suite(Config) -> diff --git a/erts/start_scripts/Makefile b/erts/start_scripts/Makefile index 4df7568484..c2576ee4cc 100644 --- a/erts/start_scripts/Makefile +++ b/erts/start_scripts/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2010. All Rights Reserved. +# Copyright Ericsson AB 1997-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -74,7 +74,7 @@ $(SS_ROOT)/start_clean.script \ $(SS_ROOT)/start_clean.boot: $(SS_ROOT)/start_clean.rel $(INSTALL_DIR) $(SS_TMP) ( cd $(SS_TMP) && \ - $(ERLC) $(SASL_FLAGS) $(SCRIPT_PATH) -o $(SS_ROOT) $< ) + $(ERLC) $(SASL_FLAGS) $(SCRIPT_PATH) +no_warn_sasl -o $(SS_ROOT) $< ) $(SS_ROOT)/start_sasl.script \ $(SS_ROOT)/start_sasl.boot: $(SS_ROOT)/start_sasl.rel @@ -128,7 +128,7 @@ $(SS_ROOT)/start_all_example.rel: $(SS_ROOT)/start_all_example.rel.src \ $(ERL_TOP)/bin/start.script: $(INSTALL_DIR) $(SS_TMP) ( cd $(SS_TMP) && \ - $(ERLC) $(SCRIPT_PATH) +otp_build -o $@ $(SS_ROOT)/start_clean.rel ) + $(ERLC) $(SCRIPT_PATH) +no_warn_sasl +otp_build -o $@ $(SS_ROOT)/start_clean.rel ) $(ERL_TOP)/bin/start_sasl.script: $(INSTALL_DIR) $(SS_TMP) @@ -138,18 +138,20 @@ $(ERL_TOP)/bin/start_sasl.script: $(ERL_TOP)/bin/start_clean.script: $(INSTALL_DIR) $(SS_TMP) ( cd $(SS_TMP) && \ - $(ERLC) $(SCRIPT_PATH) +otp_build -o $@ $(SS_ROOT)/start_clean.rel ) + $(ERLC) $(SCRIPT_PATH) +no_warn_sasl +otp_build -o $@ $(SS_ROOT)/start_clean.rel ) ## Special target used from system/build/Makefile for source code release bootstrap. bootstrap_scripts: $(SS_ROOT)/start_clean.rel $(INSTALL_DIR) $(TESTROOT)/bin $(INSTALL_DIR) $(SS_TMP) ( cd $(SS_TMP) && \ - $(ERLC) $(BOOTSTRAP_SCRIPT_PATH) +otp_build +no_module_tests \ - -o $(TESTROOT)/bin/start.script $(SS_ROOT)/start_clean.rel ) + $(ERLC) $(BOOTSTRAP_SCRIPT_PATH) +no_warn_sasl +otp_build \ + +no_module_tests -o $(TESTROOT)/bin/start.script \ + $(SS_ROOT)/start_clean.rel ) ( cd $(SS_TMP) && \ - $(ERLC) $(BOOTSTRAP_SCRIPT_PATH) +otp_build +no_module_tests \ - -o $(TESTROOT)/bin/start_clean.script $(SS_ROOT)/start_clean.rel ) + $(ERLC) $(BOOTSTRAP_SCRIPT_PATH) +no_warn_sasl +otp_build \ + +no_module_tests -o $(TESTROOT)/bin/start_clean.script \ + $(SS_ROOT)/start_clean.rel ) clean: $(RM) $(REL_SCRIPTS) $(INSTALL_SCRIPTS) diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index bfa7c6cedd..b4aa8f04bf 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -175,6 +175,8 @@ expand_opt(r12, Os) -> [no_recv_opt,no_line_info|Os]; expand_opt(r13, Os) -> [no_recv_opt,no_line_info|Os]; +expand_opt(r14, Os) -> + [no_line_info|Os]; expand_opt({debug_info_key,_}=O, Os) -> [encrypt_debug_info,O|Os]; expand_opt(no_float_opt, Os) -> diff --git a/lib/compiler/src/sys_expand_pmod.erl b/lib/compiler/src/sys_expand_pmod.erl index 4fee26f2a6..4576dfbf12 100644 --- a/lib/compiler/src/sys_expand_pmod.erl +++ b/lib/compiler/src/sys_expand_pmod.erl @@ -317,6 +317,8 @@ expr({'try',Line,Es0,Scs0,Ccs0,As0},St) -> Ccs1 = icr_clauses(Ccs0,St), As1 = exprs(As0,St), {'try',Line,Es1,Scs1,Ccs1,As1}; +expr({'fun',_,{function,_,_,_}}=ExtFun,_St) -> + ExtFun; expr({'fun',Line,Body,Info},St) -> case Body of {clauses,Cs0} -> diff --git a/lib/compiler/test/pmod_SUITE.erl b/lib/compiler/test/pmod_SUITE.erl index 3d02adaf52..5dd09a7245 100644 --- a/lib/compiler/test/pmod_SUITE.erl +++ b/lib/compiler/test/pmod_SUITE.erl @@ -100,6 +100,7 @@ basic_1(Config, Opts) -> Fun = fun(Arg) -> Prop4:bar(Arg) end, ?line ok = Fun({s,0}), + [{y,[1,2]},{x,[5,19]}] = Prop4:collapse([{y,[2,1]},{x,[19,5]}]), ok. otp_8447(Config) when is_list(Config) -> diff --git a/lib/compiler/test/pmod_SUITE_data/pmod_basic.erl b/lib/compiler/test/pmod_SUITE_data/pmod_basic.erl index 0d46cffe00..c6aa2d4655 100644 --- a/lib/compiler/test/pmod_SUITE_data/pmod_basic.erl +++ b/lib/compiler/test/pmod_SUITE_data/pmod_basic.erl @@ -21,6 +21,7 @@ -export([lookup/1,or_props/1,prepend/1,append/1,stupid_sum/0]). -export([bar/1,bar_bar/1]). -export([bc1/0, bc2/0]). +-export([collapse/1]). lookup(Key) -> proplists:lookup(Key, Props). @@ -77,3 +78,6 @@ bc1() -> bc2() -> << <<A:1>> || A <- [1,0,1,0] >>. + +collapse(L) -> + lists:keymap(fun lists:sort/1, 2, L). diff --git a/lib/observer/test/crashdump_helper.erl b/lib/observer/test/crashdump_helper.erl index d1c65f97e8..520fcdfd0d 100644 --- a/lib/observer/test/crashdump_helper.erl +++ b/lib/observer/test/crashdump_helper.erl @@ -19,7 +19,7 @@ -module(crashdump_helper). -export([n1_proc/2,remote_proc/2]). --compile(r12). +-compile(r13). -include("test_server.hrl"). n1_proc(N2,Creator) -> diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl index fdc4a2f1ff..79ece7edf5 100644 --- a/lib/observer/test/crashdump_viewer_SUITE.erl +++ b/lib/observer/test/crashdump_viewer_SUITE.erl @@ -70,7 +70,7 @@ init_per_suite(Config) when is_list(Config) -> application:start(inets), % will be using the http client later httpc:set_options([{ipfamily,inet6fb4}]), DataDir = ?config(data_dir,Config), - Rels = [R || R <- [r12b,r13b], ?t:is_release_available(R)] ++ [current], + Rels = [R || R <- [r13b,r14b], ?t:is_release_available(R)] ++ [current], io:format("Creating crash dumps for the following releases: ~p", [Rels]), AllDumps = create_dumps(DataDir,Rels), ?t:timetrap_cancel(Dog), @@ -722,7 +722,8 @@ dump_prefix(Rel) -> r11b -> "r11b_dump."; r12b -> "r12b_dump."; r13b -> "r13b_dump."; - current -> "r14b_dump." + r14b -> "r14b_dump."; + current -> "r15b_dump." end. compat_rel(Rel) -> @@ -733,5 +734,6 @@ compat_rel(Rel) -> r11b -> "+R11 "; r12b -> "+R12 "; r13b -> "+R13 "; + r14b -> "+R13 "; current -> "" end. diff --git a/lib/sasl/doc/src/release_handler.xml b/lib/sasl/doc/src/release_handler.xml index 7f32100d4b..bd4a513750 100644 --- a/lib/sasl/doc/src/release_handler.xml +++ b/lib/sasl/doc/src/release_handler.xml @@ -492,7 +492,7 @@ release_handler:set_unpacked(RelFile, [{myapp,"1.0","/home/user"},...]). </section> <funcs> <func> - <name>upgrade_app(App, Dir) -> {ok, Unpurged} | restart_new_emulator | {error, Reason}</name> + <name>upgrade_app(App, Dir) -> {ok, Unpurged} | restart_emulator | {error, Reason}</name> <fsummary>Upgrade to a new application version</fsummary> <type> <v>App = atom()</v> @@ -521,14 +521,21 @@ release_handler:set_unpacked(RelFile, [{myapp,"1.0","/home/user"},...]). does.</p> <p>Returns <c>{ok, Unpurged}</c> if evaluating the script is successful, where <c>Unpurged</c> is a list of unpurged - modules, or <c>restart_new_emulator</c> if this instruction is + modules, or <c>restart_emulator</c> if this instruction is encountered in the script, or <c>{error, Reason}</c> if an error occurred when finding or evaluating the script.</p> + <p>If the <c>restart_new_emulator</c> instruction is found in + the script, <c>upgrade_app/2</c> will return + <c>{error,restart_new_emulator}</c>. The reason for this is + that this instruction requires that a new version of the + emulator is started before the rest of the upgrade + instructions can be executed, and this can only be done by + <c>install_release/1,2</c>.</p> </desc> </func> <func> <name>downgrade_app(App, Dir) -></name> - <name>downgrade_app(App, OldVsn, Dir) -> {ok, Unpurged} | restart_new_emulator | {error, Reason}</name> + <name>downgrade_app(App, OldVsn, Dir) -> {ok, Unpurged} | restart_emulator | {error, Reason}</name> <fsummary>Downgrade to a previous application version</fsummary> <type> <v>App = atom()</v> @@ -562,7 +569,7 @@ release_handler:set_unpacked(RelFile, [{myapp,"1.0","/home/user"},...]). does.</p> <p>Returns <c>{ok, Unpurged}</c> if evaluating the script is successful, where <c>Unpurged</c> is a list of unpurged - modules, or <c>restart_new_emulator</c> if this instruction is + modules, or <c>restart_emulator</c> if this instruction is encountered in the script, or <c>{error, Reason}</c> if an error occurred when finding or evaluating the script.</p> </desc> @@ -638,7 +645,7 @@ release_handler:set_unpacked(RelFile, [{myapp,"1.0","/home/user"},...]). </desc> </func> <func> - <name>eval_appup_script(App, ToVsn, ToDir, Script) -> {ok, Unpurged} | restart_new_emulator | {error, Reason}</name> + <name>eval_appup_script(App, ToVsn, ToDir, Script) -> {ok, Unpurged} | restart_emulator | {error, Reason}</name> <fsummary>Evaluate an application upgrade or downgrade script</fsummary> <type> <v>App = atom()</v> @@ -651,8 +658,8 @@ release_handler:set_unpacked(RelFile, [{myapp,"1.0","/home/user"},...]). <desc> <p>Evaluates an application upgrade or downgrade script <c>Script</c>, the result from calling - <seealso marker="#upgrade_app/2">upgrade_app/2</seealso> or - <seealso marker="#downgrade_app/3">downgrade_app/2,3</seealso>, + <seealso marker="#upgrade_app/2">upgrade_script/2</seealso> or + <seealso marker="#downgrade_app/3">downgrade_script/3</seealso>, exactly in the same way as <seealso marker="#install_release/1">install_release/1,2</seealso> does.</p> @@ -663,9 +670,16 @@ release_handler:set_unpacked(RelFile, [{myapp,"1.0","/home/user"},...]). <c>.appup</c> files should be located under <c>Dir/ebin</c>.</p> <p>Returns <c>{ok, Unpurged}</c> if evaluating the script is successful, where <c>Unpurged</c> is a list of unpurged - modules, or <c>restart_new_emulator</c> if this instruction is + modules, or <c>restart_emulator</c> if this instruction is encountered in the script, or <c>{error, Reason}</c> if an error occurred when evaluating the script.</p> + <p>If the <c>restart_new_emulator</c> instruction is found in + the script, <c>eval_appup_script/4</c> will return + <c>{error,restart_new_emulator}</c>. The reason for this is + that this instruction requires that a new version of the + emulator is started before the rest of the upgrade + instructions can be executed, and this can only be done by + <c>install_release/1,2</c>.</p> </desc> </func> </funcs> diff --git a/lib/sasl/doc/src/systools.xml b/lib/sasl/doc/src/systools.xml index fa2fcbf534..1b90f0d4ee 100644 --- a/lib/sasl/doc/src/systools.xml +++ b/lib/sasl/doc/src/systools.xml @@ -138,8 +138,9 @@ <fsummary>Generate a boot script <c>.script/.boot</c>.</fsummary> <type> <v>Name = string()</v> - <v>Opt = src_tests | {path,[Dir]} | local | {variables,[Var]} | exref | {exref,[App]}] - | silent | {outdir,Dir} | warnings_as_errors</v> + <v>Opt = src_tests | {path,[Dir]} | local | {variables,[Var]} | exref | + {exref,[App]}] | silent | {outdir,Dir} | no_warn_sasl | + warnings_as_errors</v> <v> Dir = string()</v> <v> Var = {VarName,Prefix}</v> <v> VarName = Prefix = string()</v> @@ -195,6 +196,10 @@ <p>The applications are sorted according to the dependencies between the applications. Where there are no dependencies, the order in the <c>.rel</c> file is kept.</p> + <p>If <c>sasl</c> is not included as an application in + the <c>.rel</c> file, a warning is emitted because such a + release can not be used in an upgrade. To turn of this + warning, add the option <c>no_warn_sasl</c>.</p> <p>All files are searched for in the current path. It is assumed that the <c>.app</c> and <c>.beam</c> files for an application is located in the same directory. The <c>.erl</c> @@ -225,7 +230,7 @@ <p>Example: If the option <c>{variables,[{"TEST","lib"}]}</c> is supplied, and <c>myapp.app</c> is found in <c>lib/myapp/ebin</c>, then the path to this application in - the boot script will be <c>$TEST/myapp-1/ebin"</c>. If + the boot script will be <c>"$TEST/myapp-1/ebin"</c>. If <c>myapp.app</c> is found in <c>lib/test</c>, then the path will be <c>$TEST/test/myapp-1/ebin</c>.</p> <p>The checks performed before the boot script is generated can diff --git a/lib/sasl/src/release_handler.erl b/lib/sasl/src/release_handler.erl index 4e8cb4628c..b6ef8582c4 100644 --- a/lib/sasl/src/release_handler.erl +++ b/lib/sasl/src/release_handler.erl @@ -1050,38 +1050,50 @@ new_emulator_make_tmp_release(CurrentRelease,ToRelease,RelDir,Opts,Masters) -> CurrentVsn = CurrentRelease#release.vsn, ToVsn = ToRelease#release.vsn, TmpVsn = ?tmp_vsn(CurrentVsn), - BaseApps = [kernel,stdlib,sasl], - BaseLibs = [{App,Vsn,Lib} || {App,Vsn,Lib} <- ToRelease#release.libs, - lists:member(App,BaseApps)], - check_base_libs(BaseLibs,ToVsn), - OldBaseLibs = [{App,Vsn,Lib} || {App,Vsn,Lib} <- CurrentRelease#release.libs, - lists:member(App,BaseApps)], - check_base_libs(OldBaseLibs,CurrentVsn), - RestLibs = [{App,Vsn,Lib} || {App,Vsn,Lib} <- CurrentRelease#release.libs, - not lists:member(App,BaseApps)], - TmpRelease = CurrentRelease#release{vsn=TmpVsn, - erts_vsn=ToRelease#release.erts_vsn, - libs = BaseLibs ++ RestLibs, - status = unpacked}, - new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn,BaseLibs, - RelDir,Opts,Masters), - new_emulator_make_hybrid_config(CurrentVsn,ToVsn,TmpVsn,RelDir,Masters), - {TmpVsn,TmpRelease}. - -check_base_libs([_,_,_]=BaseLibs,_Vsn) -> - [Kernel,Sasl,Stdlib] = lists:keysort(1,BaseLibs), - [Kernel,Stdlib,Sasl]; -check_base_libs(SomeMissing,Vsn) -> - find_missing(SomeMissing,[kernel,stdlib,sasl],Vsn). - -find_missing(SomeMissing,[H|T],Vsn) -> - case lists:keymember(H,1,SomeMissing) of - true -> - find_missing(SomeMissing,T,Vsn); - false -> - throw({error,{missing_base_app,Vsn,H}}) + case get_base_libs(ToRelease#release.libs) of + {ok,{Kernel,Stdlib,Sasl}=BaseLibs,_} -> + case get_base_libs(ToRelease#release.libs) of + {ok,_,RestLibs} -> + TmpErtsVsn = ToRelease#release.erts_vsn, + TmpLibs = [Kernel,Stdlib,Sasl|RestLibs], + TmpRelease = CurrentRelease#release{vsn=TmpVsn, + erts_vsn=TmpErtsVsn, + libs = TmpLibs, + status = unpacked}, + new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn, + BaseLibs,RelDir,Opts,Masters), + new_emulator_make_hybrid_config(CurrentVsn,ToVsn,TmpVsn, + RelDir,Masters), + {TmpVsn,TmpRelease}; + {error,{missing,Missing}} -> + throw({error,{missing_base_app,CurrentVsn,Missing}}) + end; + {error,{missing,Missing}} -> + throw({error,{missing_base_app,ToVsn,Missing}}) end. +%% Get kernel, stdlib and sasl libs, +%% and also return the rest of the libs as a list. +%% Return error if any of kernel, stdlib or sasl does not exist. +get_base_libs(Libs) -> + get_base_libs(Libs,undefined,undefined,undefined,[]). +get_base_libs([{kernel,_,_}=Kernel|Libs],undefined,Stdlib,Sasl,Rest) -> + get_base_libs(Libs,Kernel,Stdlib,Sasl,Rest); +get_base_libs([{stdlib,_,_}=Stdlib|Libs],Kernel,undefined,Sasl,Rest) -> + get_base_libs(Libs,Kernel,Stdlib,Sasl,Rest); +get_base_libs([{sasl,_,_}=Sasl|Libs],Kernel,Stdlib,undefined,Rest) -> + get_base_libs(Libs,Kernel,Stdlib,Sasl,Rest); +get_base_libs([Lib|Libs],Kernel,Stdlib,Sasl,Rest) -> + get_base_libs(Libs,Kernel,Stdlib,Sasl,[Lib|Rest]); +get_base_libs([],undefined,_Stdlib,_Sasl,_Rest) -> + {error,{missing,kernel}}; +get_base_libs([],_Kernel,undefined,_Sasl,_Rest) -> + {error,{missing,stdlib}}; +get_base_libs([],_Kernel,_Stdlib,undefined,_Rest) -> + {error,{missing,sasl}}; +get_base_libs([],Kernel,Stdlib,Sasl,Rest) -> + {ok,{Kernel,Stdlib,Sasl},lists:reverse(Rest)}. + new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn,BaseLibs,RelDir,Opts,Masters) -> FromBootFile = filename:join([RelDir,CurrentVsn,"start.boot"]), ToBootFile = filename:join([RelDir,ToVsn,"start.boot"]), @@ -1090,9 +1102,10 @@ new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn,BaseLibs,RelDir,Opts,Maste Args = [ToVsn,Opts], {ok,FromBoot} = read_file(FromBootFile,Masters), {ok,ToBoot} = read_file(ToBootFile,Masters), - [KernelPath,SaslPath,StdlibPath] = - [filename:join(Path,ebin) || {_,_,Path} <- lists:keysort(1,BaseLibs)], - Paths = {KernelPath,StdlibPath,SaslPath}, + {{_,_,KernelPath},{_,_,SaslPath},{_,_,StdlibPath}} = BaseLibs, + Paths = {filename:join(KernelPath,"ebin"), + filename:join(StdlibPath,"ebin"), + filename:join(SaslPath,"ebin")}, case systools_make:make_hybrid_boot(TmpVsn,FromBoot,ToBoot,Paths,Args) of {ok,TmpBoot} -> write_file(TmpBootFile,TmpBoot,Masters); diff --git a/lib/sasl/src/release_handler_1.erl b/lib/sasl/src/release_handler_1.erl index b4b288646f..37275eff45 100644 --- a/lib/sasl/src/release_handler_1.erl +++ b/lib/sasl/src/release_handler_1.erl @@ -459,7 +459,9 @@ eval({apply, {M, F, A}}, EvalState) -> apply(M, F, A), EvalState; eval(restart_emulator, _EvalState) -> - throw(restart_emulator). + throw(restart_emulator); +eval(restart_new_emulator, _EvalState) -> + throw(restart_new_emulator). get_opt(Tag, EvalState, Default) -> case lists:keysearch(Tag, 1, EvalState#eval_state.opts) of diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index ce37f3c2ce..1c14027805 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -58,6 +58,7 @@ %% {variables,[{Name,AbsString}]} %% {machine, jam | beam | vee} %% exref | {exref, [AppName]} +%% no_warn_sasl %%----------------------------------------------------------------- make_script(RelName) when is_list(RelName) -> @@ -88,7 +89,8 @@ make_script(RelName, Output, Flags) when is_list(RelName), Path = make_set(Path1 ++ code:get_path()), ModTestP = {member(src_tests, Flags),xref_p(Flags)}, case get_release(RelName, Path, ModTestP, machine(Flags)) of - {ok, Release, Appls, Warnings} -> + {ok, Release, Appls, Warnings0} -> + Warnings = wsasl(Flags, Warnings0), case systools_lib:werror(Flags, Warnings) of true -> return(ok,Warnings,Flags); @@ -112,7 +114,13 @@ make_script(RelName, _Output, Flags) when is_list(Flags) -> make_script(RelName, _Output, Flags) -> badarg(Flags,[RelName, Flags]). -%% Inlined. + +wsasl(Options, Warnings) -> + case lists:member(no_warn_sasl,Options) of + true -> lists:delete({warning,missing_sasl},Warnings); + false -> Warnings + end. + badarg(BadArg, Args) -> erlang:error({badarg,BadArg}, Args). @@ -1970,90 +1978,67 @@ get_flag(_,_) -> false. %% Check Options for make_script check_args_script(Args) -> - cas(Args, - {undef, undef, undef, undef, undef, undef, undef, undef, - undef, []}). + cas(Args, []). -cas([], {_Path,_Sil,_Loc,_Test,_Var,_Mach,_Xref,_XrefApps,_Werror, X}) -> +cas([], X) -> X; %%% path --------------------------------------------------------------- -cas([{path, P} | Args], {Path, Sil, Loc, Test, Var, Mach, Xref, - XrefApps, Werror, X}) when is_list(P) -> +cas([{path, P} | Args], X) when is_list(P) -> case check_path(P) of ok -> - cas(Args, {P, Sil, Loc, Test, Var, Mach, Xref, XrefApps, - Werror, X}); + cas(Args, X); error -> - cas(Args, {Path, Sil, Loc, Test, Var, Mach, Xref, XrefApps, - Werror, X++[{path,P}]}) + cas(Args, X++[{path,P}]) end; %%% silent ------------------------------------------------------------- -cas([silent | Args], {Path, _Sil, Loc, Test, Var, Mach, Xref, - XrefApps, Werror, X}) -> - cas(Args, {Path, silent, Loc, Test, Var, Mach, Xref, XrefApps, - Werror, X}); +cas([silent | Args], X) -> + cas(Args, X); %%% local -------------------------------------------------------------- -cas([local | Args], {Path, Sil, _Loc, Test, Var, Mach, Xref, - XrefApps, Werror, X}) -> - cas(Args, {Path, Sil, local, Test, Var, Mach, Xref, XrefApps, - Werror, X}); +cas([local | Args], X) -> + cas(Args, X); %%% src_tests ------------------------------------------------------- -cas([src_tests | Args], {Path, Sil, Loc, _Test, Var, Mach, Xref, - XrefApps, Werror, X}) -> - cas(Args, - {Path, Sil, Loc, src_tests, Var, Mach, Xref, Werror, XrefApps,X}); +cas([src_tests | Args], X) -> + cas(Args, X); %%% variables ---------------------------------------------------------- -cas([{variables, V} | Args], {Path, Sil, Loc, Test, Var, Mach, Xref, - XrefApps, Werror, X}) when is_list(V) -> +cas([{variables, V} | Args], X) when is_list(V) -> case check_vars(V) of ok -> - cas(Args, - {Path, Sil, Loc, Test, V, Mach, Xref, XrefApps, Werror, X}); + cas(Args, X); error -> - cas(Args, {Path, Sil, Loc, Test, Var, Mach, Xref, XrefApps, - Werror, X++[{variables, V}]}) + cas(Args, X++[{variables, V}]) end; %%% machine ------------------------------------------------------------ -cas([{machine, M} | Args], {Path, Sil, Loc, Test, Var, Mach, Xref, - XrefApps, Werror, X}) when is_atom(M) -> - cas(Args, {Path, Sil, Loc, Test, Var, Mach, Xref, XrefApps, Werror, X}); +cas([{machine, M} | Args], X) when is_atom(M) -> + cas(Args, X); %%% exref -------------------------------------------------------------- -cas([exref | Args], {Path, Sil, Loc, Test, Var, Mach, _Xref, - XrefApps, Werror, X}) -> - cas(Args, {Path, Sil, Loc, Test, Var, Mach, exref, XrefApps, Werror, X}); +cas([exref | Args], X) -> + cas(Args, X); %%% exref Apps --------------------------------------------------------- -cas([{exref, Apps} | Args], {Path, Sil, Loc, Test, Var, Mach, Xref, - XrefApps, Werror, X}) when is_list(Apps) -> +cas([{exref, Apps} | Args], X) when is_list(Apps) -> case check_apps(Apps) of ok -> - cas(Args, {Path, Sil, Loc, Test, Var, Mach, - Xref, Apps, Werror, X}); + cas(Args, X); error -> - cas(Args, {Path, Sil, Loc, Test, Var, Mach, - Xref, XrefApps, Werror, X++[{exref, Apps}]}) + cas(Args, X++[{exref, Apps}]) end; %%% outdir Dir --------------------------------------------------------- -cas([{outdir, Dir} | Args], {Path, Sil, Loc, Test, Var, Mach, Xref, - XrefApps, Werror, X}) when is_list(Dir) -> - cas(Args, {Path, Sil, Loc, Test, Var, Mach, Xref, XrefApps, Werror, X}); +cas([{outdir, Dir} | Args], X) when is_list(Dir) -> + cas(Args, X); %%% otp_build (secret, not documented) --------------------------------- -cas([otp_build | Args], {Path, Sil, Loc, Test, Var, Mach, Xref, - XrefApps, Werror, X}) -> - cas(Args, {Path, Sil, Loc, Test, Var, Mach, Xref, XrefApps, Werror, X}); +cas([otp_build | Args], X) -> + cas(Args, X); +%%% warnings_as_errors ------------------------------------------------- +cas([warnings_as_errors | Args], X) -> + cas(Args, X); +%%% no_warn_sasl ------------------------------------------------------- +cas([no_warn_sasl | Args], X) -> + cas(Args, X); %%% no_module_tests (kept for backwards compatibility, but ignored) ---- -cas([no_module_tests | Args], {Path, Sil, Loc, Test, Var, Mach, Xref, - XrefApps, Werror, X}) -> - cas(Args, {Path, Sil, Loc, Test, Var, Mach, Xref, XrefApps, Werror, X}); -%%% warnings_as_errors (kept for backwards compatibility, but ignored) ---- -cas([warnings_as_errors | Args], {Path, Sil, Loc, Test, Var, Mach, Xref, - XrefApps, _Werror, X}) -> - cas(Args, {Path, Sil, Loc, Test, Var, Mach, Xref, XrefApps, - warnings_as_errors, X}); +cas([no_module_tests | Args], X) -> + cas(Args, X); %%% ERROR -------------------------------------------------------------- -cas([Y | Args], {Path, Sil, Loc, Test, Var, Mach, Xref, XrefApps, - Werror, X}) -> - cas(Args, {Path, Sil, Loc, Test, Var, Mach, Xref, XrefApps, Werror, - X++[Y]}). +cas([Y | Args], X) -> + cas(Args, X++[Y]). diff --git a/lib/sasl/test/release_handler_SUITE.erl b/lib/sasl/test/release_handler_SUITE.erl index 11f8bbe4fe..454fe26323 100644 --- a/lib/sasl/test/release_handler_SUITE.erl +++ b/lib/sasl/test/release_handler_SUITE.erl @@ -60,7 +60,8 @@ cases() -> [otp_2740, otp_2760, otp_5761, otp_9402, otp_9417, otp_9395_check_old_code, otp_9395_check_and_purge, otp_9395_update_many_mods, otp_9395_rm_many_mods, - instructions, eval_appup, supervisor_which_children_timeout, + instructions, eval_appup, eval_appup_with_restart, + supervisor_which_children_timeout, release_handler_which_releases, install_release_syntax_check]. groups() -> @@ -1226,6 +1227,12 @@ eval_appup(Conf) when is_list(Conf) -> App11Dir = code:lib_dir(app1), ok = gen_server:call(harry, error), + %% Read appup script + {ok,"2.0",UpScript} = release_handler:upgrade_script(app1,App12Dir), + [{load_object_code,_}, + point_of_no_return, + {load,_}] = UpScript, + %% Upgrade to app1-2.0 {ok, []} = release_handler:upgrade_app(app1, App12Dir), App12Dir = code:lib_dir(app1), @@ -1236,6 +1243,12 @@ eval_appup(Conf) when is_list(Conf) -> %% (see myrel/lib2/app1-2.0/ebin/app1.app) [{var,val2}] = ets:lookup(otp_6162, var), + %% Read appup script + {ok,DnScript} = release_handler:downgrade_script(app1,"1.0",App11Dir), + [{load_object_code,_}, + point_of_no_return, + {load,_}] = DnScript, + %% Downgrade to app1-1.0 {ok, []} = release_handler:downgrade_app(app1,"1.0",App11Dir), App11Dir = code:lib_dir(app1), @@ -1253,6 +1266,85 @@ eval_appup(Conf) when is_list(Conf) -> ok. +%% Test upgrade and downgrade of applications when appup contains +%% restart_emulator and restart_new_emulator instructions +eval_appup_with_restart(Conf) when is_list(Conf) -> + + %% Set some paths + RelDir = filename:join(?config(data_dir, Conf), "app1_app2"), + App11Dir = filename:join([RelDir, "lib1", "app1-1.0"]), + App13Dir = filename:join([RelDir, "lib3", "app1-3.0"]), %restart_emulator + App14Dir = filename:join([RelDir, "lib4", "app1-4.0"]), %restart_new_emulator + EbinDir1 = filename:join(App11Dir, "ebin"), + EbinDir3 = filename:join(App13Dir, "ebin"), + EbinDir4 = filename:join(App14Dir, "ebin"), + + %% Start app1-1.0 + code:add_patha(EbinDir1), + ok = application:start(app1), + App11Dir = code:lib_dir(app1), + + %% Read appup script + {ok,"3.0",UpScript3} = release_handler:upgrade_script(app1,App13Dir), + [{load_object_code,_}, + point_of_no_return, + {load,_}, + restart_emulator] = UpScript3, + + %% Upgrade to app1-3.0 - restart_emulator + restart_emulator = release_handler:upgrade_app(app1, App13Dir), + App13Dir = code:lib_dir(app1), + + %% Fake full upgrade to 3.0 + {ok,AppSpec} = file:consult(filename:join([App13Dir,"ebin","app1.app"])), + application_controller:change_application_data(AppSpec,[]), + + %% Read appup script + {ok,"4.0",UpScript4} = release_handler:upgrade_script(app1,App14Dir), + [restart_new_emulator,point_of_no_return] = UpScript4, + + %% Try pgrade to app1-4.0 - restart_new_emulator + {error,restart_new_emulator} = release_handler:upgrade_app(app1, App14Dir), + App13Dir = code:lib_dir(app1), + + %% Read appup script + {ok,DnScript1} = release_handler:downgrade_script(app1,"1.0",App11Dir), + [{load_object_code,_}, + point_of_no_return, + {load,_}, + restart_emulator] = DnScript1, + + %% Still running 3.0 - downgrade to app1-1.0 - restart_emulator + restart_emulator = release_handler:downgrade_app(app1,"1.0",App11Dir), + App11Dir = code:lib_dir(app1), + + ok = application:stop(app1), + ok = application:unload(app1), + true = code:del_path(EbinDir1), + + %% Start again as version 4.0 + code:add_patha(EbinDir4), + ok = application:start(app1), + App14Dir = code:lib_dir(app1), + + %% Read appup script + {ok,DnScript3} = release_handler:downgrade_script(app1,"3.0",App13Dir), + [point_of_no_return,restart_emulator] = DnScript3, + + %% Downgrade to app1-3.0 - restart_new_emulator + restart_emulator = release_handler:downgrade_app(app1,"3.0",App13Dir), + App13Dir = code:lib_dir(app1), + + ok = application:stop(app1), + ok = application:unload(app1), + + true = code:del_path(EbinDir3), + false = code:del_path(EbinDir1), + false = code:del_path(EbinDir4), + + ok. + + %% Test the example/target_system.erl module target_system(Conf) when is_list(Conf) -> PrivDir = priv_dir(Conf), diff --git a/lib/sasl/test/release_handler_SUITE_data/Makefile.src b/lib/sasl/test/release_handler_SUITE_data/Makefile.src index edb446413d..6f40088161 100644 --- a/lib/sasl/test/release_handler_SUITE_data/Makefile.src +++ b/lib/sasl/test/release_handler_SUITE_data/Makefile.src @@ -50,7 +50,13 @@ APP= \ app1_app2/lib2/app1-2.0/ebin/app1.@EMULATOR@ \ app1_app2/lib2/app2-1.0/ebin/app2_sup.@EMULATOR@ \ app1_app2/lib2/app2-1.0/ebin/app2_server.@EMULATOR@ \ - app1_app2/lib2/app2-1.0/ebin/app2.@EMULATOR@ + app1_app2/lib2/app2-1.0/ebin/app2.@EMULATOR@ \ + app1_app2/lib3/app1-3.0/ebin/app1_sup.@EMULATOR@ \ + app1_app2/lib3/app1-3.0/ebin/app1_server.@EMULATOR@ \ + app1_app2/lib3/app1-3.0/ebin/app1.@EMULATOR@ \ + app1_app2/lib4/app1-4.0/ebin/app1_sup.@EMULATOR@ \ + app1_app2/lib4/app1-4.0/ebin/app1_server.@EMULATOR@ \ + app1_app2/lib4/app1-4.0/ebin/app1.@EMULATOR@ OTP2740= \ otp_2740/vsn_atom.@EMULATOR@ \ @@ -183,6 +189,22 @@ app1_app2/lib2/app2-1.0/ebin/app2.@EMULATOR@: app1_app2/lib2/app2-1.0/src/app2.e erlc $(EFLAGS) -oapp1_app2/lib2/app2-1.0/ebin app1_app2/lib2/app2-1.0/src/app2.erl +app1_app2/lib3/app1-3.0/ebin/app1_sup.@EMULATOR@: app1_app2/lib3/app1-3.0/src/app1_sup.erl + erlc $(EFLAGS) -oapp1_app2/lib3/app1-3.0/ebin app1_app2/lib3/app1-3.0/src/app1_sup.erl +app1_app2/lib3/app1-3.0/ebin/app1_server.@EMULATOR@: app1_app2/lib3/app1-3.0/src/app1_server.erl + erlc $(EFLAGS) -oapp1_app2/lib3/app1-3.0/ebin app1_app2/lib3/app1-3.0/src/app1_server.erl +app1_app2/lib3/app1-3.0/ebin/app1.@EMULATOR@: app1_app2/lib3/app1-3.0/src/app1.erl + erlc $(EFLAGS) -oapp1_app2/lib3/app1-3.0/ebin app1_app2/lib3/app1-3.0/src/app1.erl + + +app1_app2/lib4/app1-4.0/ebin/app1_sup.@EMULATOR@: app1_app2/lib4/app1-4.0/src/app1_sup.erl + erlc $(EFLAGS) -oapp1_app2/lib4/app1-4.0/ebin app1_app2/lib4/app1-4.0/src/app1_sup.erl +app1_app2/lib4/app1-4.0/ebin/app1_server.@EMULATOR@: app1_app2/lib4/app1-4.0/src/app1_server.erl + erlc $(EFLAGS) -oapp1_app2/lib4/app1-4.0/ebin app1_app2/lib4/app1-4.0/src/app1_server.erl +app1_app2/lib4/app1-4.0/ebin/app1.@EMULATOR@: app1_app2/lib4/app1-4.0/src/app1.erl + erlc $(EFLAGS) -oapp1_app2/lib4/app1-4.0/ebin app1_app2/lib4/app1-4.0/src/app1.erl + + otp_2740/vsn_atom.@EMULATOR@: otp_2740/vsn_atom.erl erlc $(EFLAGS) -ootp_2740 otp_2740/vsn_atom.erl otp_2740/vsn_list.@EMULATOR@: otp_2740/vsn_list.erl diff --git a/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/ebin/app1.app b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/ebin/app1.app new file mode 100644 index 0000000000..4adc0540c4 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/ebin/app1.app @@ -0,0 +1,9 @@ +{application, app1, + [{description, "very simple example application"}, + {id, "app1"}, + {vsn, "3.0"}, + {modules, [app1, app1_sup, app1_server]}, + {registered, [harry]}, + {applications, [kernel, stdlib, sasl]}, + {env, [{var,val2}]}, + {mod, {app1, []}}]}. diff --git a/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/ebin/app1.appup b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/ebin/app1.appup new file mode 100644 index 0000000000..a5cdfe9fcc --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/ebin/app1.appup @@ -0,0 +1,4 @@ +{"3.0", + [{"1.0", [{load_module, app1_server},restart_emulator]}], + [{"1.0", [{load_module, app1_server},restart_emulator]}] +}. diff --git a/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/src/app1.erl b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/src/app1.erl new file mode 100644 index 0000000000..f123c8f470 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/src/app1.erl @@ -0,0 +1,22 @@ +-module(app1). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). +-export([config_change/3]). + +start(_Type, _StartArgs) -> + case app1_sup:start_link() of + {ok, Pid} -> + {ok, Pid}; + Error -> + Error + end. + +stop(_State) -> + ok. + +config_change(Changed, _New, _Removed) -> + catch ets:insert(otp_6162, hd(Changed)), + ok. diff --git a/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/src/app1_server.erl b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/src/app1_server.erl new file mode 100644 index 0000000000..660d095ebf --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/src/app1_server.erl @@ -0,0 +1,35 @@ +-module(app1_server). + +-behaviour(gen_server). + +%% API +-export([start_link/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +start_link() -> + gen_server:start_link({local, harry}, ?MODULE, [], []). + +init([]) -> + {ok, []}. + +handle_call(error, _From, State) -> + Reply = error, + {reply, Reply, State}; +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/src/app1_sup.erl b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/src/app1_sup.erl new file mode 100644 index 0000000000..e6ad9b6967 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib3/app1-3.0/src/app1_sup.erl @@ -0,0 +1,17 @@ +-module(app1_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +start_link() -> + supervisor:start_link(?MODULE, []). + +init([]) -> + AChild = {harry,{app1_server,start_link,[]}, + permanent,2000,worker,[app1_server]}, + {ok,{{one_for_all,0,1}, [AChild]}}. diff --git a/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/ebin/app1.app b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/ebin/app1.app new file mode 100644 index 0000000000..243bc21f02 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/ebin/app1.app @@ -0,0 +1,9 @@ +{application, app1, + [{description, "very simple example application"}, + {id, "app1"}, + {vsn, "4.0"}, + {modules, [app1, app1_sup, app1_server]}, + {registered, [harry]}, + {applications, [kernel, stdlib, sasl]}, + {env, [{var,val2}]}, + {mod, {app1, []}}]}. diff --git a/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/ebin/app1.appup b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/ebin/app1.appup new file mode 100644 index 0000000000..72535c8b34 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/ebin/app1.appup @@ -0,0 +1,4 @@ +{"4.0", + [{"3.0", [restart_new_emulator]}], + [{"3.0", [restart_new_emulator]}] +}. diff --git a/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/src/app1.erl b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/src/app1.erl new file mode 100644 index 0000000000..f123c8f470 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/src/app1.erl @@ -0,0 +1,22 @@ +-module(app1). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). +-export([config_change/3]). + +start(_Type, _StartArgs) -> + case app1_sup:start_link() of + {ok, Pid} -> + {ok, Pid}; + Error -> + Error + end. + +stop(_State) -> + ok. + +config_change(Changed, _New, _Removed) -> + catch ets:insert(otp_6162, hd(Changed)), + ok. diff --git a/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/src/app1_server.erl b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/src/app1_server.erl new file mode 100644 index 0000000000..660d095ebf --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/src/app1_server.erl @@ -0,0 +1,35 @@ +-module(app1_server). + +-behaviour(gen_server). + +%% API +-export([start_link/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +start_link() -> + gen_server:start_link({local, harry}, ?MODULE, [], []). + +init([]) -> + {ok, []}. + +handle_call(error, _From, State) -> + Reply = error, + {reply, Reply, State}; +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/src/app1_sup.erl b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/src/app1_sup.erl new file mode 100644 index 0000000000..e6ad9b6967 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/app1_app2/lib4/app1-4.0/src/app1_sup.erl @@ -0,0 +1,17 @@ +-module(app1_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +start_link() -> + supervisor:start_link(?MODULE, []). + +init([]) -> + AChild = {harry,{app1_server,start_link,[]}, + permanent,2000,worker,[app1_server]}, + {ok,{{one_for_all,0,1}, [AChild]}}. diff --git a/lib/sasl/test/systools_SUITE.erl b/lib/sasl/test/systools_SUITE.erl index 892c4994e8..beb1e48ca7 100644 --- a/lib/sasl/test/systools_SUITE.erl +++ b/lib/sasl/test/systools_SUITE.erl @@ -411,6 +411,9 @@ no_sasl_script(Config) when is_list(Config) -> ?line {ok, _ , [{warning,missing_sasl}]} = systools:make_script(LatestName,[{path, P},silent]), + ?line {ok, _ , []} = + systools:make_script(LatestName,[{path, P},silent, no_warn_sasl]), + ?line ok = file:set_cwd(OldDir), ok. diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 0c44d3ae90..d4807704b2 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -87,15 +87,17 @@ bytes_to_read, % integer(), # bytes to read in passive mode user_data_buffer, % binary() log_alert, % boolean() - renegotiation, % {boolean(), From | internal | peer} - recv_during_renegotiation, %boolean() - send_queue, % queue() - terminated = false % + renegotiation, % {boolean(), From | internal | peer} + recv_from, % + send_queue, % queue() + terminated = false, % + allow_renegotiate = true }). -define(DEFAULT_DIFFIE_HELLMAN_PARAMS, - #'DHParameter'{prime = ?DEFAULT_DIFFIE_HELLMAN_PRIME, + #'DHParameter'{prime = ?DEFAULT_DIFFIE_HELLMAN_PRIME, base = ?DEFAULT_DIFFIE_HELLMAN_GENERATOR}). +-define(WAIT_TO_ALLOW_RENEGOTIATION, 12000). -type state_name() :: hello | abbreviated | certify | cipher | connection. -type gen_fsm_state_return() :: {next_state, state_name(), #state{}} | @@ -355,15 +357,15 @@ hello(start, #state{host = Host, port = Port, role = client, Session0#session{session_id = Hello#client_hello.session_id}, tls_handshake_hashes = Hashes1}, {Record, State} = next_record(State1), - next_state(hello, Record, State); + next_state(hello, hello, Record, State); hello(start, #state{role = server} = State0) -> {Record, State} = next_record(State0), - next_state(hello, Record, State); + next_state(hello, hello, Record, State); hello(#hello_request{}, #state{role = client} = State0) -> {Record, State} = next_record(State0), - next_state(hello, Record, State); + next_state(hello, hello, Record, State); hello(#server_hello{cipher_suite = CipherSuite, compression_method = Compression} = Hello, @@ -426,7 +428,7 @@ hello(Msg, State) -> %%-------------------------------------------------------------------- abbreviated(#hello_request{}, State0) -> {Record, State} = next_record(State0), - next_state(hello, Record, State); + next_state(abbreviated, hello, Record, State); abbreviated(#finished{verify_data = Data} = Finished, #state{role = server, @@ -479,7 +481,7 @@ abbreviated(Msg, State) -> %%-------------------------------------------------------------------- certify(#hello_request{}, State0) -> {Record, State} = next_record(State0), - next_state(hello, Record, State); + next_state(certify, hello, Record, State); certify(#certificate{asn1_certificates = []}, #state{role = server, negotiated_version = Version, @@ -487,7 +489,7 @@ certify(#certificate{asn1_certificates = []}, fail_if_no_peer_cert = true}} = State) -> Alert = ?ALERT_REC(?FATAL,?HANDSHAKE_FAILURE), - handle_own_alert(Alert, Version, certify_certificate, State), + handle_own_alert(Alert, Version, certify, State), {stop, normal, State}; certify(#certificate{asn1_certificates = []}, @@ -496,7 +498,7 @@ certify(#certificate{asn1_certificates = []}, fail_if_no_peer_cert = false}} = State0) -> {Record, State} = next_record(State0#state{client_certificate_requested = false}), - next_state(certify, Record, State); + next_state(certify, certify, Record, State); certify(#certificate{} = Cert, #state{negotiated_version = Version, @@ -511,7 +513,7 @@ certify(#certificate{} = Cert, handle_peer_cert(PeerCert, PublicKeyInfo, State#state{client_certificate_requested = false}); #alert{} = Alert -> - handle_own_alert(Alert, Version, certify_certificate, State), + handle_own_alert(Alert, Version, certify, State), {stop, normal, State} end; @@ -522,10 +524,9 @@ certify(#server_key_exchange{} = KeyExchangeMsg, case handle_server_key(KeyExchangeMsg, State0) of #state{} = State1 -> {Record, State} = next_record(State1), - next_state(certify, Record, State); + next_state(certify, certify, Record, State); #alert{} = Alert -> - handle_own_alert(Alert, Version, certify_server_keyexchange, - State0), + handle_own_alert(Alert, Version, certify, State0), {stop, normal, State0} end; @@ -535,7 +536,7 @@ certify(#server_key_exchange{} = Msg, certify(#certificate_request{}, State0) -> {Record, State} = next_record(State0#state{client_certificate_requested = true}), - next_state(certify, Record, State); + next_state(certify, certify, Record, State); %% Master secret was determined with help of server-key exchange msg certify(#server_hello_done{}, @@ -550,8 +551,7 @@ certify(#server_hello_done{}, State = State0#state{connection_states = ConnectionStates1}, client_certify_and_key_exchange(State); #alert{} = Alert -> - handle_own_alert(Alert, Version, - certify_server_hello_done, State0), + handle_own_alert(Alert, Version, certify, State0), {stop, normal, State0} end; @@ -570,8 +570,7 @@ certify(#server_hello_done{}, session = Session}, client_certify_and_key_exchange(State); #alert{} = Alert -> - handle_own_alert(Alert, Version, - certify_server_hello_done, State0), + handle_own_alert(Alert, Version, certify, State0), {stop, normal, State0} end; @@ -588,7 +587,7 @@ certify(#client_key_exchange{exchange_keys = Keys}, certify_client_key_exchange(ssl_handshake:decode_client_key(Keys, KeyAlg, Version), State) catch #alert{} = Alert -> - handle_own_alert(Alert, Version, certify_client_key_exchange, State), + handle_own_alert(Alert, Version, certify, State), {stop, normal, State} end; @@ -611,10 +610,9 @@ certify_client_key_exchange(#encrypted_premaster_secret{premaster_secret= EncPMS State1 = State0#state{connection_states = ConnectionStates, session = Session}, {Record, State} = next_record(State1), - next_state(cipher, Record, State); + next_state(certify, cipher, Record, State); #alert{} = Alert -> - handle_own_alert(Alert, Version, - certify_client_key_exchange, State0), + handle_own_alert(Alert, Version, certify, State0), {stop, normal, State0} end; @@ -626,10 +624,9 @@ certify_client_key_exchange(#client_diffie_hellman_public{dh_public = ClientPubl case dh_master_secret(crypto:mpint(P), crypto:mpint(G), ClientPublicDhKey, ServerDhPrivateKey, State0) of #state{} = State1 -> {Record, State} = next_record(State1), - next_state(cipher, Record, State); + next_state(certify, cipher, Record, State); #alert{} = Alert -> - handle_own_alert(Alert, Version, - certify_client_key_exchange, State0), + handle_own_alert(Alert, Version, certify, State0), {stop, normal, State0} end. @@ -639,7 +636,7 @@ certify_client_key_exchange(#client_diffie_hellman_public{dh_public = ClientPubl %%-------------------------------------------------------------------- cipher(#hello_request{}, State0) -> {Record, State} = next_record(State0), - next_state(hello, Record, State); + next_state(cipher, hello, Record, State); cipher(#certificate_verify{signature = Signature}, #state{role = server, @@ -652,7 +649,7 @@ cipher(#certificate_verify{signature = Signature}, Version, MasterSecret, Hashes) of valid -> {Record, State} = next_record(State0), - next_state(cipher, Record, State); + next_state(cipher, cipher, Record, State); #alert{} = Alert -> handle_own_alert(Alert, Version, cipher, State0), {stop, normal, State0} @@ -705,10 +702,26 @@ connection(#hello_request{}, #state{host = Host, port = Port, {Record, State} = next_record(State0#state{connection_states = ConnectionStates1, tls_handshake_hashes = Hashes1}), - next_state(hello, Record, State); -connection(#client_hello{} = Hello, #state{role = server} = State) -> - hello(Hello, State); - + next_state(connection, hello, Record, State); +connection(#client_hello{} = Hello, #state{role = server, allow_renegotiate = true} = State) -> + %% Mitigate Computational DoS attack + %% http://www.educatedguesswork.org/2011/10/ssltls_and_computational_dos.html + %% http://www.thc.org/thc-ssl-dos/ Rather than disabling client + %% initiated renegotiation we will disallow many client initiated + %% renegotiations immediately after each other. + erlang:send_after(?WAIT_TO_ALLOW_RENEGOTIATION, self(), allow_renegotiate), + hello(Hello, State#state{allow_renegotiate = false}); + +connection(#client_hello{}, #state{role = server, allow_renegotiate = false, + connection_states = ConnectionStates0, + socket = Socket, transport_cb = Transport, + negotiated_version = Version} = State0) -> + Alert = ?ALERT_REC(?WARNING, ?NO_RENEGOTIATION), + {BinMsg, ConnectionStates} = + encode_alert(Alert, Version, ConnectionStates0), + Transport:send(Socket, BinMsg), + next_state_connection(connection, State0#state{connection_states = ConnectionStates}); + connection(timeout, State) -> {next_state, connection, State, hibernate}; @@ -737,37 +750,12 @@ handle_event(_Event, StateName, State) -> %% gen_fsm:sync_send_all_state_event/2,3, this function is called to handle %% the event. %%-------------------------------------------------------------------- -handle_sync_event({application_data, Data0}, From, connection, - #state{socket = Socket, - negotiated_version = Version, - transport_cb = Transport, - connection_states = ConnectionStates0, - send_queue = SendQueue, - socket_options = SockOpts, - ssl_options = #ssl_options{renegotiate_at = RenegotiateAt}} - = State) -> +handle_sync_event({application_data, Data}, From, connection, State) -> %% We should look into having a worker process to do this to %% parallize send and receive decoding and not block the receiver %% if sending is overloading the socket. try - Data = encode_packet(Data0, SockOpts), - case encode_data(Data, Version, ConnectionStates0, RenegotiateAt) of - {Msgs, [], ConnectionStates} -> - Result = Transport:send(Socket, Msgs), - {reply, Result, - connection, State#state{connection_states = ConnectionStates}, - get_timeout(State)}; - {Msgs, RestData, ConnectionStates} -> - if - Msgs =/= [] -> - Transport:send(Socket, Msgs); - true -> - ok - end, - renegotiate(State#state{connection_states = ConnectionStates, - send_queue = queue:in_r({From, RestData}, SendQueue), - renegotiation = {true, internal}}) - end + write_application_data(Data, From, State) catch throw:Error -> {reply, Error, connection, State, get_timeout(State)} end; @@ -824,14 +812,12 @@ handle_sync_event({shutdown, How0}, _, StateName, end; handle_sync_event({recv, N}, From, connection = StateName, State0) -> - passive_receive(State0#state{bytes_to_read = N, from = From}, StateName); + passive_receive(State0#state{bytes_to_read = N, recv_from = From}, StateName); %% Doing renegotiate wait with handling request until renegotiate is -%% finished. Will be handled by next_state_connection/2. +%% finished. Will be handled by next_state_is_connection/2. handle_sync_event({recv, N}, From, StateName, State) -> - {next_state, StateName, - State#state{bytes_to_read = N, from = From, - recv_during_renegotiation = true}, + {next_state, StateName, State#state{bytes_to_read = N, recv_from = From}, get_timeout(State)}; handle_sync_event({new_user, User}, _From, StateName, @@ -869,7 +855,7 @@ handle_sync_event({set_opts, Opts0}, _From, StateName, Buffer =:= <<>>, Opts1#socket_options.active =:= false -> %% Need data, set active once {Record, State2} = next_record_if_active(State1), - case next_state(StateName, Record, State2) of + case next_state(StateName, StateName, Record, State2) of {next_state, StateName, State, Timeout} -> {reply, Reply, StateName, State, Timeout}; {stop, Reason, State} -> @@ -879,11 +865,11 @@ handle_sync_event({set_opts, Opts0}, _From, StateName, %% Active once already set {reply, Reply, StateName, State1, get_timeout(State1)}; true -> - case application_data(<<>>, State1) of + case read_application_data(<<>>, State1) of Stop = {stop,_,_} -> Stop; {Record, State2} -> - case next_state(StateName, Record, State2) of + case next_state(StateName, StateName, Record, State2) of {next_state, StateName, State, Timeout} -> {reply, Reply, StateName, State, Timeout}; {stop, Reason, State} -> @@ -931,22 +917,18 @@ handle_sync_event(peer_certificate, _, StateName, %% raw data from TCP, unpack records handle_info({Protocol, _, Data}, StateName, - #state{data_tag = Protocol, - negotiated_version = Version} = State0) -> + #state{data_tag = Protocol} = State0) -> case next_tls_record(Data, State0) of {Record, State} -> - next_state(StateName, Record, State); + next_state(StateName, StateName, Record, State); #alert{} = Alert -> - handle_own_alert(Alert, Version, StateName, State0), + handle_normal_shutdown(Alert, StateName, State0), {stop, normal, State0} end; -handle_info({CloseTag, Socket}, _StateName, +handle_info({CloseTag, Socket}, StateName, #state{socket = Socket, close_tag = CloseTag, - negotiated_version = Version, - socket_options = Opts, - user_application = {_Mon,Pid}, from = From, - role = Role} = State) -> + negotiated_version = Version} = State) -> %% Note that as of TLS 1.1, %% failure to properly close a connection no longer requires that a %% session not be resumed. This is a change from TLS 1.0 to conform @@ -961,8 +943,7 @@ handle_info({CloseTag, Socket}, _StateName, %%invalidate_session(Role, Host, Port, Session) ok end, - alert_user(Opts#socket_options.active, Pid, From, - ?ALERT_REC(?WARNING, ?CLOSE_NOTIFY), Role), + handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), {stop, normal, State}; handle_info({ErrorTag, Socket, econnaborted}, StateName, @@ -971,18 +952,20 @@ handle_info({ErrorTag, Socket, econnaborted}, StateName, alert_user(User, ?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE), Role), {stop, normal, State}; -handle_info({ErrorTag, Socket, Reason}, _, - #state{socket = Socket, from = User, - role = Role, error_tag = ErrorTag} = State) -> +handle_info({ErrorTag, Socket, Reason}, StateName, #state{socket = Socket, + error_tag = ErrorTag} = State) -> Report = io_lib:format("SSL: Socket error: ~p ~n", [Reason]), error_logger:info_report(Report), - alert_user(User, ?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), Role), + handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State), {stop, normal, State}; handle_info({'DOWN', MonitorRef, _, _, _}, _, State = #state{user_application={MonitorRef,_Pid}}) -> {stop, normal, State}; +handle_info(allow_renegotiate, StateName, State) -> + {next_state, StateName, State#state{allow_renegotiate = true}, get_timeout(State)}; + handle_info(Msg, StateName, State) -> Report = io_lib:format("SSL: Got unexpected info: ~p ~n", [Msg]), error_logger:info_report(Report), @@ -1218,7 +1201,7 @@ handle_peer_cert(PeerCert, PublicKeyInfo, Session#session{peer_certificate = PeerCert}, public_key_info = PublicKeyInfo}, {Record, State} = next_record(State1), - next_state(certify, Record, State). + next_state(certify, certify, Record, State). certify_client(#state{client_certificate_requested = true, role = client, connection_states = ConnectionStates0, @@ -1260,8 +1243,7 @@ verify_client_cert(#state{client_certificate_requested = true, role = client, ignore -> State; #alert{} = Alert -> - handle_own_alert(Alert, Version, certify, State) - + throw(Alert) end; verify_client_cert(#state{client_certificate_requested = false} = State) -> State. @@ -1293,7 +1275,7 @@ do_server_hello(Type, #state{negotiated_version = Version, ConnectionStates, tls_handshake_hashes = Hashes}, {Record, State} = next_record(State3), - next_state(abbreviated, Record, State); + next_state(hello, abbreviated, Record, State); #alert{} = Alert -> handle_own_alert(Alert, Version, hello, State1), {stop, normal, State1} @@ -1313,7 +1295,7 @@ new_server_hello(#server_hello{cipher_suite = CipherSuite, cipher_suite = CipherSuite, compression_method = Compression}, {Record, State} = next_record(State2#state{session = Session}), - next_state(certify, Record, State) + next_state(hello, certify, Record, State) catch #alert{} = Alert -> handle_own_alert(Alert, Version, hello, State0), @@ -1325,7 +1307,7 @@ handle_new_session(NewId, CipherSuite, Compression, #state{session = Session0} = cipher_suite = CipherSuite, compression_method = Compression}, {Record, State} = next_record(State0#state{session = Session}), - next_state(certify, Record, State). + next_state(hello, certify, Record, State). handle_resumed_session(SessId, #state{connection_states = ConnectionStates0, negotiated_version = Version, @@ -1340,7 +1322,7 @@ handle_resumed_session(SessId, #state{connection_states = ConnectionStates0, next_record(State0#state{ connection_states = ConnectionStates1, session = Session}), - next_state(abbreviated, Record, State); + next_state(hello, abbreviated, Record, State); #alert{} = Alert -> handle_own_alert(Alert, Version, hello, State0), {stop, normal, State0} @@ -1357,10 +1339,10 @@ client_certify_and_key_exchange(#state{negotiated_version = Version} = client_certificate_requested = false, tls_handshake_hashes = Hashes}, {Record, State} = next_record(State2), - next_state(cipher, Record, State) + next_state(certify, cipher, Record, State) catch - #alert{} = Alert -> - handle_own_alert(Alert, Version, client_certify_and_key_exchange, State0), + throw:#alert{} = Alert -> + handle_own_alert(Alert, Version, certify, State0), {stop, normal, State0} end. @@ -1671,15 +1653,12 @@ encode_packet(Data, #socket_options{packet=Packet}) -> end. encode_size_packet(Bin, Size, Max) -> - Len = byte_size(Bin), + Len = erlang:byte_size(Bin), case Len > Max of true -> throw({error, {badarg, {packet_to_large, Len, Max}}}); false -> <<Len:Size, Bin/binary>> end. -encode_data(Data, Version, ConnectionStates, RenegotiateAt) -> - ssl_record:encode_data(Data, Version, ConnectionStates, RenegotiateAt). - decode_alerts(Bin) -> decode_alerts(Bin, []). @@ -1693,20 +1672,20 @@ passive_receive(State0 = #state{user_data_buffer = Buffer}, StateName) -> case Buffer of <<>> -> {Record, State} = next_record(State0), - next_state(StateName, Record, State); + next_state(StateName, StateName, Record, State); _ -> - case application_data(<<>>, State0) of + case read_application_data(<<>>, State0) of Stop = {stop, _, _} -> Stop; {Record, State} -> - next_state(StateName, Record, State) + next_state(StateName, StateName, Record, State) end end. -application_data(Data, #state{user_application = {_Mon, Pid}, +read_application_data(Data, #state{user_application = {_Mon, Pid}, socket_options = SOpts, bytes_to_read = BytesToRead, - from = From, + recv_from = From, user_data_buffer = Buffer0} = State0) -> Buffer1 = if Buffer0 =:= <<>> -> Data; @@ -1717,7 +1696,7 @@ application_data(Data, #state{user_application = {_Mon, Pid}, {ok, ClientData, Buffer} -> % Send data SocketOpt = deliver_app_data(SOpts, ClientData, Pid, From), State = State0#state{user_data_buffer = Buffer, - from = undefined, + recv_from = undefined, bytes_to_read = 0, socket_options = SocketOpt }, @@ -1727,7 +1706,7 @@ application_data(Data, #state{user_application = {_Mon, Pid}, %% Active and empty, get more data next_record_if_active(State); true -> %% We have more data - application_data(<<>>, State) + read_application_data(<<>>, State) end; {more, Buffer} -> % no reply, we need more data next_record(State0#state{user_data_buffer = Buffer}); @@ -1736,6 +1715,39 @@ application_data(Data, #state{user_application = {_Mon, Pid}, {stop, normal, State0} end. +write_application_data(Data0, From, #state{socket = Socket, + negotiated_version = Version, + transport_cb = Transport, + connection_states = ConnectionStates0, + send_queue = SendQueue, + socket_options = SockOpts, + ssl_options = #ssl_options{renegotiate_at = RenegotiateAt}} = State) -> + Data = encode_packet(Data0, SockOpts), + + case time_to_renegotiate(Data, ConnectionStates0, RenegotiateAt) of + true -> + renegotiate(State#state{send_queue = queue:in_r({From, Data}, SendQueue), + renegotiation = {true, internal}}); + false -> + {Msgs, ConnectionStates} = ssl_record:encode_data(Data, Version, ConnectionStates0), + Result = Transport:send(Socket, Msgs), + {reply, Result, + connection, State#state{connection_states = ConnectionStates}, get_timeout(State)} + end. + +time_to_renegotiate(_Data, #connection_states{current_write = + #connection_state{sequence_number = Num}}, RenegotiateAt) -> + + %% We could do test: + %% is_time_to_renegotiate((erlang:byte_size(_Data) div ?MAX_PLAIN_TEXT_LENGTH) + 1, RenegotiateAt), + %% but we chose to have a some what lower renegotiateAt and a much cheaper test + is_time_to_renegotiate(Num, RenegotiateAt). + +is_time_to_renegotiate(N, M) when N < M-> + false; +is_time_to_renegotiate(_,_) -> + true. + %% Picks ClientData get_data(_, _, <<>>) -> {more, <<>>}; @@ -1837,6 +1849,10 @@ header(N, Binary) -> send_or_reply(false, _Pid, From, Data) when From =/= undefined -> gen_fsm:reply(From, Data); +%% Can happen when handling own alert or tcp error/close and there is +%% no outstanding gen_fsm sync events +send_or_reply(false, no_pid, _, _) -> + ok; send_or_reply(_, Pid, _From, Data) -> send_user(Pid, Data). @@ -1861,18 +1877,18 @@ handle_tls_handshake(Handle, StateName, #state{tls_packets = [Packet | Packets]} Stop end. -next_state(_, #alert{} = Alert, #state{negotiated_version = Version} = State) -> - handle_own_alert(Alert, Version, decipher_error, State), +next_state(Current,_, #alert{} = Alert, #state{negotiated_version = Version} = State) -> + handle_own_alert(Alert, Version, Current, State), {stop, normal, State}; -next_state(Next, no_record, State) -> +next_state(_,Next, no_record, State) -> {next_state, Next, State, get_timeout(State)}; -next_state(Next, #ssl_tls{type = ?ALERT, fragment = EncAlerts}, State) -> +next_state(_,Next, #ssl_tls{type = ?ALERT, fragment = EncAlerts}, State) -> Alerts = decode_alerts(EncAlerts), handle_alerts(Alerts, {next_state, Next, State, get_timeout(State)}); -next_state(StateName, #ssl_tls{type = ?HANDSHAKE, fragment = Data}, +next_state(Current, Next, #ssl_tls{type = ?HANDSHAKE, fragment = Data}, State0 = #state{tls_handshake_buffer = Buf0, negotiated_version = Version}) -> Handle = fun({#hello_request{} = Packet, _}, {next_state, connection = SName, State}) -> @@ -1898,30 +1914,30 @@ next_state(StateName, #ssl_tls{type = ?HANDSHAKE, fragment = Data}, try {Packets, Buf} = ssl_handshake:get_tls_handshake(Data,Buf0), State = State0#state{tls_packets = Packets, tls_handshake_buffer = Buf}, - handle_tls_handshake(Handle, StateName, State) + handle_tls_handshake(Handle, Next, State) catch throw:#alert{} = Alert -> - handle_own_alert(Alert, Version, StateName, State0), + handle_own_alert(Alert, Version, Current, State0), {stop, normal, State0} end; -next_state(StateName, #ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, State0) -> - case application_data(Data, State0) of +next_state(_, StateName, #ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, State0) -> + case read_application_data(Data, State0) of Stop = {stop,_,_} -> Stop; {Record, State} -> - next_state(StateName, Record, State) + next_state(StateName, StateName, Record, State) end; -next_state(StateName, #ssl_tls{type = ?CHANGE_CIPHER_SPEC, fragment = <<1>>} = +next_state(Current, Next, #ssl_tls{type = ?CHANGE_CIPHER_SPEC, fragment = <<1>>} = _ChangeCipher, #state{connection_states = ConnectionStates0} = State0) -> ConnectionStates1 = ssl_record:activate_pending_connection_state(ConnectionStates0, read), {Record, State} = next_record(State0#state{connection_states = ConnectionStates1}), - next_state(StateName, Record, State); -next_state(StateName, #ssl_tls{type = _Unknown}, State0) -> + next_state(Current, Next, Record, State); +next_state(Current, Next, #ssl_tls{type = _Unknown}, State0) -> %% Ignore unknown type {Record, State} = next_record(State0), - next_state(StateName, Record, State). + next_state(Current, Next, Record, State). next_tls_record(Data, #state{tls_record_buffer = Buf0, tls_cipher_texts = CT0} = State0) -> @@ -1960,50 +1976,36 @@ next_state_connection(StateName, #state{send_queue = Queue0, negotiated_version = Version, socket = Socket, transport_cb = Transport, - connection_states = ConnectionStates0, - ssl_options = #ssl_options{renegotiate_at = RenegotiateAt} + connection_states = ConnectionStates0 } = State) -> - %% Send queued up data + %% Send queued up data that was queued while renegotiating case queue:out(Queue0) of {{value, {From, Data}}, Queue} -> - case encode_data(Data, Version, ConnectionStates0, RenegotiateAt) of - {Msgs, [], ConnectionStates} -> - Result = Transport:send(Socket, Msgs), - gen_fsm:reply(From, Result), - next_state_connection(StateName, - State#state{connection_states = ConnectionStates, - send_queue = Queue}); - %% This is unlikely to happen. User configuration of the - %% undocumented test option renegotiation_at can make it more likely. - {Msgs, RestData, ConnectionStates} -> - if - Msgs =/= [] -> - Transport:send(Socket, Msgs); - true -> - ok - end, - renegotiate(State#state{connection_states = ConnectionStates, - send_queue = queue:in_r({From, RestData}, Queue), - renegotiation = {true, internal}}) - end; + {Msgs, ConnectionStates} = + ssl_record:encode_data(Data, Version, ConnectionStates0), + Result = Transport:send(Socket, Msgs), + gen_fsm:reply(From, Result), + next_state_connection(StateName, + State#state{connection_states = ConnectionStates, + send_queue = Queue}); {empty, Queue0} -> - next_state_is_connection(State) + next_state_is_connection(StateName, State) end. %% In next_state_is_connection/1: clear tls_handshake_hashes, %% premaster_secret and public_key_info (only needed during handshake) %% to reduce memory foot print of a connection. -next_state_is_connection(State = - #state{recv_during_renegotiation = true, socket_options = - #socket_options{active = false}}) -> - passive_receive(State#state{recv_during_renegotiation = false, - premaster_secret = undefined, +next_state_is_connection(_, State = + #state{recv_from = From, + socket_options = + #socket_options{active = false}}) when From =/= undefined -> + passive_receive(State#state{premaster_secret = undefined, public_key_info = undefined, tls_handshake_hashes = {<<>>, <<>>}}, connection); -next_state_is_connection(State0) -> +next_state_is_connection(StateName, State0) -> {Record, State} = next_record_if_active(State0), - next_state(connection, Record, State#state{premaster_secret = undefined, + next_state(StateName, connection, Record, State#state{premaster_secret = undefined, public_key_info = undefined, tls_handshake_hashes = {<<>>, <<>>}}). @@ -2059,7 +2061,7 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions}, User, log_alert = true, session_cache_cb = SessionCacheCb, renegotiation = {false, first}, - recv_during_renegotiation = false, + recv_from = undefined, send_queue = queue:new() }. @@ -2172,16 +2174,14 @@ handle_alert(#alert{level = ?FATAL} = Alert, StateName, {stop, normal, State}; handle_alert(#alert{level = ?WARNING, description = ?CLOSE_NOTIFY} = Alert, - StateName, #state{from = From, role = Role, - user_application = {_Mon, Pid}, socket_options = Opts} = State) -> - alert_user(StateName, Opts, Pid, From, Alert, Role), + StateName, State) -> + handle_normal_shutdown(Alert, StateName, State), {stop, normal, State}; handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, StateName, - #state{log_alert = Log, renegotiation = {true, internal}, from = From, - role = Role} = State) -> + #state{log_alert = Log, renegotiation = {true, internal}} = State) -> log_alert(Log, StateName, Alert), - alert_user(From, Alert, Role), + handle_normal_shutdown(Alert, StateName, State), {stop, normal, State}; handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, StateName, @@ -2189,13 +2189,13 @@ handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, log_alert(Log, StateName, Alert), gen_fsm:reply(From, {error, renegotiation_rejected}), {Record, State} = next_record(State0), - next_state(connection, Record, State); + next_state(StateName, connection, Record, State); handle_alert(#alert{level = ?WARNING, description = ?USER_CANCELED} = Alert, StateName, #state{log_alert = Log} = State0) -> log_alert(Log, StateName, Alert), {Record, State} = next_record(State0), - next_state(StateName, Record, State). + next_state(StateName, StateName, Record, State). alert_user(connection, Opts, Pid, From, Alert, Role) -> alert_user(Opts#socket_options.active, Pid, From, Alert, Role); @@ -2227,13 +2227,11 @@ log_alert(true, Info, Alert) -> log_alert(false, _, _) -> ok. -handle_own_alert(Alert, Version, Info, +handle_own_alert(Alert, Version, StateName, #state{transport_cb = Transport, socket = Socket, - from = User, - role = Role, connection_states = ConnectionStates, - log_alert = Log}) -> + log_alert = Log} = State) -> try %% Try to tell the other side {BinMsg, _} = encode_alert(Alert, Version, ConnectionStates), @@ -2243,12 +2241,20 @@ handle_own_alert(Alert, Version, Info, ignore end, try %% Try to tell the local user - log_alert(Log, Info, Alert), - alert_user(User, Alert, Role) + log_alert(Log, StateName, Alert), + handle_normal_shutdown(Alert,StateName, State) catch _:_ -> ok end. +handle_normal_shutdown(Alert, _, #state{from = User, role = Role, renegotiation = {false, first}}) -> + alert_user(User, Alert, Role); + +handle_normal_shutdown(Alert, StateName, #state{socket_options = Opts, + user_application = {_Mon, Pid}, + from = User, role = Role}) -> + alert_user(StateName, Opts, Pid, User, Alert, Role). + handle_unexpected_message(Msg, Info, #state{negotiated_version = Version} = State) -> Alert = ?ALERT_REC(?FATAL,?UNEXPECTED_MESSAGE), handle_own_alert(Alert, Version, {Info, Msg}, State), @@ -2261,7 +2267,7 @@ make_premaster_secret(_, _) -> undefined. mpint_binary(Binary) -> - Size = byte_size(Binary), + Size = erlang:byte_size(Binary), <<?UINT32(Size), Binary/binary>>. @@ -2298,7 +2304,7 @@ renegotiate(#state{role = server, {Record, State} = next_record(State0#state{connection_states = ConnectionStates, tls_handshake_hashes = Hs0}), - next_state(hello, Record, State). + next_state(connection, hello, Record, State#state{allow_renegotiate = true}). notify_senders(SendQueue) -> lists:foreach(fun({From, _}) -> diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 7eb7f44df6..371f475c85 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -447,7 +447,7 @@ server_hello_done() -> -spec encode_handshake(tls_handshake(), tls_version()) -> iolist(). %% %% Description: Encode a handshake packet to binary -%%-------------------------------------------------------------------- +%%--------------------------------------------------------------------x encode_handshake(Package, Version) -> {MsgType, Bin} = enc_hs(Package, Version), Len = byte_size(Bin), diff --git a/lib/ssl/src/ssl_record.erl b/lib/ssl/src/ssl_record.erl index 72091fdd5f..f52d2f961c 100644 --- a/lib/ssl/src/ssl_record.erl +++ b/lib/ssl/src/ssl_record.erl @@ -48,7 +48,7 @@ %% Encoding records -export([encode_handshake/3, encode_alert_record/3, - encode_change_cipher_spec/2, encode_data/4]). + encode_change_cipher_spec/2, encode_data/3]). %% Decoding -export([decode_cipher_text/2]). @@ -503,36 +503,14 @@ decode_cipher_text(CipherText, ConnnectionStates0) -> Alert end. %%-------------------------------------------------------------------- --spec encode_data(iolist(), tls_version(), #connection_states{}, integer()) -> - {iolist(), iolist(), #connection_states{}}. +-spec encode_data(binary(), tls_version(), #connection_states{}) -> + {iolist(), #connection_states{}}. %% %% Description: Encodes data to send on the ssl-socket. %%-------------------------------------------------------------------- -encode_data(Frag, Version, ConnectionStates, RenegotiateAt) - when byte_size(Frag) < (?MAX_PLAIN_TEXT_LENGTH - 2048) -> - case encode_plain_text(?APPLICATION_DATA,Version,Frag,ConnectionStates, RenegotiateAt) of - {renegotiate, Data} -> - {[], Data, ConnectionStates}; - {Msg, CS} -> - {Msg, [], CS} - end; - -encode_data(Frag, Version, ConnectionStates, RenegotiateAt) when is_binary(Frag) -> - Data = split_bin(Frag, ?MAX_PLAIN_TEXT_LENGTH - 2048), - encode_data(Data, Version, ConnectionStates, RenegotiateAt); - -encode_data(Data, Version, ConnectionStates0, RenegotiateAt) when is_list(Data) -> - {ConnectionStates, EncodedMsg, NotEncdedData} = - lists:foldl(fun(B, {CS0, Encoded, Rest}) -> - case encode_plain_text(?APPLICATION_DATA, - Version, B, CS0, RenegotiateAt) of - {renegotiate, NotEnc} -> - {CS0, Encoded, [NotEnc | Rest]}; - {Enc, CS1} -> - {CS1, [Enc | Encoded], Rest} - end - end, {ConnectionStates0, [], []}, Data), - {lists:reverse(EncodedMsg), lists:reverse(NotEncdedData), ConnectionStates}. +encode_data(Frag, Version, ConnectionStates) -> + Data = split_bin(Frag, ?MAX_PLAIN_TEXT_LENGTH, Version), + encode_iolist(?APPLICATION_DATA, Data, Version, ConnectionStates). %%-------------------------------------------------------------------- -spec encode_handshake(iolist(), tls_version(), #connection_states{}) -> @@ -566,6 +544,14 @@ encode_change_cipher_spec(Version, ConnectionStates) -> %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- +encode_iolist(Type, Data, Version, ConnectionStates0) -> + {ConnectionStates, EncodedMsg} = + lists:foldl(fun(Text, {CS0, Encoded}) -> + {Enc, CS1} = encode_plain_text(Type, Version, Text, CS0), + {CS1, [Enc | Encoded]} + end, {ConnectionStates0, []}, Data), + {lists:reverse(EncodedMsg), ConnectionStates}. + highest_protocol_version() -> highest_protocol_version(supported_protocol_versions()). @@ -602,29 +588,23 @@ record_protocol_role(client) -> record_protocol_role(server) -> ?SERVER. -split_bin(Bin, ChunkSize) -> - split_bin(Bin, ChunkSize, []). +%% 1/n-1 splitting countermeasure Rizzo/Duong-Beast +split_bin(<<FirstByte:8, Rest/binary>>, ChunkSize, Version) when {3, 1} == Version orelse + {3, 0} == Version -> + do_split_bin(Rest, ChunkSize, [[FirstByte]]); +split_bin(Bin, ChunkSize, _) -> + do_split_bin(Bin, ChunkSize, []). -split_bin(<<>>, _, Acc) -> +do_split_bin(<<>>, _, Acc) -> lists:reverse(Acc); -split_bin(Bin, ChunkSize, Acc) -> +do_split_bin(Bin, ChunkSize, Acc) -> case Bin of <<Chunk:ChunkSize/binary, Rest/binary>> -> - split_bin(Rest, ChunkSize, [Chunk | Acc]); + do_split_bin(Rest, ChunkSize, [Chunk | Acc]); _ -> lists:reverse(Acc, [Bin]) end. -encode_plain_text(Type, Version, Data, ConnectionStates, RenegotiateAt) -> - #connection_states{current_write = - #connection_state{sequence_number = Num}} = ConnectionStates, - case renegotiate(Num, RenegotiateAt) of - false -> - encode_plain_text(Type, Version, Data, ConnectionStates); - true -> - {renegotiate, Data} - end. - encode_plain_text(Type, Version, Data, ConnectionStates) -> #connection_states{current_write=#connection_state{ compression_state=CompS0, @@ -637,11 +617,6 @@ encode_plain_text(Type, Version, Data, ConnectionStates) -> CTBin = encode_tls_cipher_text(Type, Version, CipherText), {CTBin, ConnectionStates#connection_states{current_write = CS2}}. -renegotiate(N, M) when N < M-> - false; -renegotiate(_,_) -> - true. - encode_tls_cipher_text(Type, {MajVer, MinVer}, Fragment) -> Length = erlang:iolist_size(Fragment), [<<?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer), ?UINT16(Length)>>, Fragment]. diff --git a/lib/ssl/src/ssl_record.hrl b/lib/ssl/src/ssl_record.hrl index 5fb0070b91..282d642138 100644 --- a/lib/ssl/src/ssl_record.hrl +++ b/lib/ssl/src/ssl_record.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2010. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -70,9 +70,10 @@ -define(MAX_SEQENCE_NUMBER, 18446744073709552000). %% math:pow(2, 64) - 1 = 1.8446744073709552e19 %% Sequence numbers can not wrap so when max is about to be reached we should renegotiate. %% We will renegotiate a little before so that there will be sequence numbers left -%% for the rehandshake and a little data. --define(MARGIN, 100). --define(DEFAULT_RENEGOTIATE_AT, ?MAX_SEQENCE_NUMBER - ?MARGIN). +%% for the rehandshake and a little data. Currently we decided to renegotiate a little more +%% often as we can have a cheaper test to check if it is time to renegotiate. It will still +%% be fairly seldom. +-define(DEFAULT_RENEGOTIATE_AT, 268435456). %% math:pow(2, 28) %% ConnectionEnd -define(SERVER, 0). diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index d9cb8002ed..228ec9e294 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -36,6 +36,7 @@ -define(LONG_TIMEOUT, 600000). -define(EXPIRE, 10). -define(SLEEP, 500). +-define(RENEGOTIATION_DISABLE_TIME, 12000). %% Test server callback functions %%-------------------------------------------------------------------- @@ -256,7 +257,9 @@ all() -> %%different_ca_peer_sign, no_reuses_session_server_restart_new_cert, no_reuses_session_server_restart_new_cert_file, reuseaddr, - hibernate, connect_twice + hibernate, connect_twice, renegotiate_dos_mitigate_active, + renegotiate_dos_mitigate_passive, + tcp_error_propagation_in_active_mode ]. groups() -> @@ -393,8 +396,8 @@ controlling_process(Config) when is_list(Config) -> ClientOpts = ?config(client_opts, Config), ServerOpts = ?config(server_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - ClientMsg = "Hello server", - ServerMsg = "Hello client", + ClientMsg = "Server hello", + ServerMsg = "Client hello", Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, @@ -415,11 +418,15 @@ controlling_process(Config) when is_list(Config) -> [self(), Client, Server]), receive + {ssl, _, "S"} -> + receive_s_rizzo_duong_beast(); {ssl, _, ServerMsg} -> receive {ssl, _, ClientMsg} -> ok end; + {ssl, _, "C"} -> + receive_c_rizzo_duong_beast(); {ssl, _, ClientMsg} -> receive {ssl, _, ServerMsg} -> @@ -440,6 +447,28 @@ controlling_process_result(Socket, Pid, Msg) -> ssl:send(Socket, Msg), no_result_msg. +receive_s_rizzo_duong_beast() -> + receive + {ssl, _, "erver hello"} -> + receive + {ssl, _, "C"} -> + receive + {ssl, _, "lient hello"} -> + ok + end + end + end. +receive_c_rizzo_duong_beast() -> + receive + {ssl, _, "lient hello"} -> + receive + {ssl, _, "S"} -> + receive + {ssl, _, "erver hello"} -> + ok + end + end + end. %%-------------------------------------------------------------------- controller_dies(doc) -> ["Test that the socket is closed after controlling process dies"]; @@ -1231,6 +1260,11 @@ upgrade_result(Socket) -> %% Make sure binary is inherited from tcp socket and that we do %% not get the list default! receive + {ssl, _, <<"H">>} -> + receive + {ssl, _, <<"ello world">>} -> + ok + end; {ssl, _, <<"Hello world">>} -> ok end. @@ -1532,14 +1566,14 @@ eoptions(Config) when is_list(Config) -> {cacertfile, ""}, {dhfile,'dh.pem' }, {ciphers, [{foo, bar, sha, ignore}]}, - {reuse_session, foo}, - {reuse_sessions, 0}, + {reuse_session, foo}, + {reuse_sessions, 0}, {renegotiate_at, "10"}, - {debug, 1}, + {debug, 1}, {mode, depech}, - {packet, 8.0}, - {packet_size, "2"}, - {header, a}, + {packet, 8.0}, + {packet_size, "2"}, + {header, a}, {active, trice}, {key, 'key.pem' }], @@ -2340,7 +2374,7 @@ server_verify_client_once_active(Config) when is_list(Config) -> Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {?MODULE, send_recv_result_active, []}}, - {options, [{active, once}, {verify, verify_peer}, + {options, [{active, true}, {verify, verify_peer}, {verify_client_once, true} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), @@ -2724,17 +2758,28 @@ client_no_wrap_sequence_number(Config) when is_list(Config) -> {options, ServerOpts}]), Port = ssl_test_lib:inet_port(Server), + Version = ssl_record:highest_protocol_version(ssl_record:supported_protocol_versions()), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, - trigger_renegotiate, [[ErlData, N+2]]}}, + trigger_renegotiate, [[ErlData, treashold(N, Version)]]}}, {options, [{reuse_sessions, false}, {renegotiate_at, N} | ClientOpts]}]), ssl_test_lib:check_result(Client, ok), ssl_test_lib:close(Server), ssl_test_lib:close(Client). + + %% First two clauses handles 1/n-1 splitting countermeasure Rizzo/Duong-Beast +treashold(N, {3,0}) -> + (N div 2) + 1; +treashold(N, {3,1}) -> + (N div 2) + 1; +treashold(N, _) -> + N + 1. + %%-------------------------------------------------------------------- server_no_wrap_sequence_number(doc) -> ["Test that erlang server will renegotiate session when", @@ -3647,8 +3692,102 @@ connect_twice(Config) when is_list(Config) -> ssl_test_lib:close(Client), ssl_test_lib:close(Client1). +%%-------------------------------------------------------------------- +renegotiate_dos_mitigate_active(doc) -> + ["Mitigate DOS computational attack by not allowing client to renegotiate many times in a row", + "immediately after each other"]; + +renegotiate_dos_mitigate_active(suite) -> + []; + +renegotiate_dos_mitigate_active(Config) when is_list(Config) -> + ServerOpts = ?config(server_opts, Config), + ClientOpts = ?config(client_opts, Config), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, send_recv_result_active, []}}, + {options, [ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, + renegotiate_immediately, []}}, + {options, ClientOpts}]), + + ssl_test_lib:check_result(Client, ok, Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). %%-------------------------------------------------------------------- +renegotiate_dos_mitigate_passive(doc) -> + ["Mitigate DOS computational attack by not allowing client to renegotiate many times in a row", + "immediately after each other"]; + +renegotiate_dos_mitigate_passive(suite) -> + []; + +renegotiate_dos_mitigate_passive(Config) when is_list(Config) -> + ServerOpts = ?config(server_opts, Config), + ClientOpts = ?config(client_opts, Config), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, send_recv_result, []}}, + {options, [{active, false} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, + renegotiate_immediately, []}}, + {options, ClientOpts}]), + + ssl_test_lib:check_result(Client, ok, Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- +tcp_error_propagation_in_active_mode(doc) -> + ["Test that process recives {ssl_error, Socket, closed} when tcp error ocurres"]; +tcp_error_propagation_in_active_mode(Config) when is_list(Config) -> + ClientOpts = ?config(client_opts, Config), + ServerOpts = ?config(server_opts, Config), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, no_result, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + {Client, #sslsocket{pid=Pid} = SslSocket} = ssl_test_lib:start_client([return_socket, + {node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, receive_msg, []}}, + {options, ClientOpts}]), + + {status, _, _, StatusInfo} = sys:get_status(Pid), + [_, _,_, _, Prop] = StatusInfo, + State = ssl_test_lib:state(Prop), + Socket = element(10, State), + + %% Fake tcp error + Pid ! {tcp_error, Socket, etimedout}, + + ssl_test_lib:check_result(Client, {ssl_closed, SslSocket}). + +%%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- send_recv_result(Socket) -> @@ -3659,6 +3798,11 @@ send_recv_result(Socket) -> send_recv_result_active(Socket) -> ssl:send(Socket, "Hello world"), receive + {ssl, Socket, "H"} -> + receive + {ssl, Socket, "ello world"} -> + ok + end; {ssl, Socket, "Hello world"} -> ok end. @@ -3666,6 +3810,12 @@ send_recv_result_active(Socket) -> send_recv_result_active_once(Socket) -> ssl:send(Socket, "Hello world"), receive + {ssl, Socket, "H"} -> + ssl:setopts(Socket, [{active, once}]), + receive + {ssl, Socket, "ello world"} -> + ok + end; {ssl, Socket, "Hello world"} -> ok end. @@ -3690,6 +3840,25 @@ renegotiate_reuse_session(Socket, Data) -> test_server:sleep(?SLEEP), renegotiate(Socket, Data). +renegotiate_immediately(Socket) -> + receive + {ssl, Socket, "Hello world"} -> + ok; + %% Handle 1/n-1 splitting countermeasure Rizzo/Duong-Beast + {ssl, Socket, "H"} -> + receive + {ssl, Socket, "ello world"} -> + ok + end + end, + ok = ssl:renegotiate(Socket), + {error, renegotiation_rejected} = ssl:renegotiate(Socket), + test_server:sleep(?RENEGOTIATION_DISABLE_TIME +1), + ok = ssl:renegotiate(Socket), + test_server:format("Renegotiated again"), + ssl:send(Socket, "Hello world"), + ok. + new_config(PrivDir, ServerOpts0) -> CaCertFile = proplists:get_value(cacertfile, ServerOpts0), CertFile = proplists:get_value(certfile, ServerOpts0), @@ -3863,8 +4032,17 @@ erlang_ssl_receive(Socket, Data) -> {ssl, Socket, Data} -> io:format("Received ~p~n",[Data]), ok; + {ssl, Socket, Byte} when length(Byte) == 1 -> %% Handle 1/n-1 splitting countermeasure Rizzo/Duong-Beast + io:format("Received ~p~n",[Byte]), + erlang_ssl_receive(Socket, tl(Data)); Other -> test_server:fail({unexpected_message, Other}) after ?SLEEP * 3 -> test_server:fail({did_not_get, Data}) end. + +receive_msg(_) -> + receive + Msg -> + Msg + end. diff --git a/lib/ssl/test/ssl_packet_SUITE.erl b/lib/ssl/test/ssl_packet_SUITE.erl index 9d2599b778..4b74f57a60 100644 --- a/lib/ssl/test/ssl_packet_SUITE.erl +++ b/lib/ssl/test/ssl_packet_SUITE.erl @@ -158,14 +158,24 @@ all() -> packet_asn1_decode, packet_asn1_decode_list, packet_tpkt_decode, packet_tpkt_decode_list, packet_sunrm_decode, packet_sunrm_decode_list, - header_decode_one_byte, header_decode_two_bytes, - header_decode_two_bytes_one_sent, - header_decode_two_bytes_two_sent]. + {group, header} + ]. groups() -> - []. + [{header, [], [ header_decode_one_byte, + header_decode_two_bytes, + header_decode_two_bytes_one_sent, + header_decode_two_bytes_two_sent]}]. + +init_per_group(header, Config) -> + case ssl_record:highest_protocol_version(ssl_record:supported_protocol_versions()) of + {3, N} when N < 2 -> + {skip, ""}; + _ -> + Config + end; -init_per_group(_GroupName, Config) -> +init_per_group(_, Config) -> Config. end_per_group(_GroupName, Config) -> @@ -2626,6 +2636,13 @@ active_once_raw(_, _, 0, _) -> ok; active_once_raw(Socket, Data, N, Acc) -> receive + {ssl, Socket, Byte} when length(Byte) == 1 -> + ssl:setopts(Socket, [{active, once}]), + receive + {ssl, Socket, _} -> + ssl:setopts(Socket, [{active, once}]), + active_once_raw(Socket, Data, N-1, []) + end; {ssl, Socket, Data} -> ssl:setopts(Socket, [{active, once}]), active_once_raw(Socket, Data, N-1, []); @@ -2648,7 +2665,14 @@ active_once_packet(Socket,_, 0) -> {other, Other, ssl:session_info(Socket), 0} end; active_once_packet(Socket, Data, N) -> - receive + receive + {ssl, Socket, Byte} when length(Byte) == 1 -> + ssl:setopts(Socket, [{active, once}]), + receive + {ssl, Socket, _} -> + ssl:setopts(Socket, [{active, once}]), + active_once_packet(Socket, Data, N-1) + end; {ssl, Socket, Data} -> ok end, @@ -2662,6 +2686,11 @@ active_raw(_Socket, _, 0, _) -> ok; active_raw(Socket, Data, N, Acc) -> receive + {ssl, Socket, Byte} when length(Byte) == 1 -> + receive + {ssl, Socket, _} -> + active_raw(Socket, Data, N -1) + end; {ssl, Socket, Data} -> active_raw(Socket, Data, N-1, []); {ssl, Socket, Other} -> @@ -2682,6 +2711,11 @@ active_packet(Socket, _, 0) -> end; active_packet(Socket, Data, N) -> receive + {ssl, Socket, Byte} when length(Byte) == 1 -> + receive + {ssl, Socket, _} -> + active_packet(Socket, Data, N -1) + end; {ssl, Socket, Data} -> active_packet(Socket, Data, N -1); Other -> diff --git a/lib/ssl/test/ssl_session_cache_SUITE.erl b/lib/ssl/test/ssl_session_cache_SUITE.erl index 8cdfdec2ce..9c4a5ce640 100644 --- a/lib/ssl/test/ssl_session_cache_SUITE.erl +++ b/lib/ssl/test/ssl_session_cache_SUITE.erl @@ -210,7 +210,7 @@ session_cleanup(Config)when is_list(Config) -> {status, _, _, StatusInfo} = sys:get_status(whereis(ssl_manager)), [_, _,_, _, Prop] = StatusInfo, - State = state(Prop), + State = ssl_test_lib:state(Prop), Cache = element(2, State), SessionTimer = element(6, State), @@ -238,11 +238,6 @@ session_cleanup(Config)when is_list(Config) -> ssl_test_lib:close(Server), ssl_test_lib:close(Client). -state([{data,[{"State", State}]} | _]) -> - State; -state([_ | Rest]) -> - state(Rest). - check_timer(Timer) -> case erlang:read_timer(Timer) of false -> @@ -256,7 +251,7 @@ check_timer(Timer) -> get_delay_timer() -> {status, _, _, StatusInfo} = sys:get_status(whereis(ssl_manager)), [_, _,_, _, Prop] = StatusInfo, - State = state(Prop), + State = ssl_test_lib:state(Prop), case element(7, State) of undefined -> test_server:sleep(?SLEEP), diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 46a8112a41..fa8a1826f2 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -662,6 +662,9 @@ cipher_result(Socket, Result) -> %% to properly test "cipher state" handling ssl:send(Socket, "Hello\n"), receive + {ssl, Socket, "H"} -> + ssl:send(Socket, " world\n"), + receive_rizzo_duong_beast(); {ssl, Socket, "Hello\n"} -> ssl:send(Socket, " world\n"), receive @@ -687,3 +690,21 @@ public_key(#'PrivateKeyInfo'{privateKeyAlgorithm = public_key:der_decode('DSAPrivateKey', iolist_to_binary(Key)); public_key(Key) -> Key. +receive_rizzo_duong_beast() -> + receive + {ssl, _, "ello\n"} -> + receive + {ssl, _, " "} -> + receive + {ssl, _, "world\n"} -> + ok + end + end + end. + +state([{data,[{"State", State}]} | _]) -> + State; +state([{data,[{"StateData", State}]} | _]) -> + State; +state([_ | Rest]) -> + state(Rest). diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index f37baeb9de..f04ab9af50 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -849,7 +849,9 @@ ssl3_erlang_server_erlang_client_client_cert(Config) when is_list(Config) -> Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {?MODULE, - erlang_ssl_receive, [Data]}}, + erlang_ssl_receive, + %% Due to 1/n-1 splitting countermeasure Rizzo/Duong-Beast + [Data]}}, {options, [{verify , verify_peer} | ServerOpts]}]), @@ -858,6 +860,7 @@ ssl3_erlang_server_erlang_client_client_cert(Config) when is_list(Config) -> Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, + %% Due to 1/n-1 splitting countermeasure Rizzo/Duong-Beast {mfa, {ssl, send, [Data]}}, {options, [{versions, [sslv3]} | ClientOpts]}]), @@ -869,6 +872,7 @@ ssl3_erlang_server_erlang_client_client_cert(Config) when is_list(Config) -> process_flag(trap_exit, false), ok. + %%-------------------------------------------------------------------- tls1_erlang_client_openssl_server(doc) -> @@ -1350,6 +1354,8 @@ erlang_ssl_receive(Socket, Data) -> %% open_ssl server sometimes hangs waiting in blocking read ssl:send(Socket, "Got it"), ok; + {ssl, Socket, Byte} when length(Byte) == 1 -> + erlang_ssl_receive(Socket, tl(Data)); {Port, {data,Debug}} when is_port(Port) -> io:format("openssl ~s~n",[Debug]), erlang_ssl_receive(Socket,Data); diff --git a/lib/stdlib/doc/src/Makefile b/lib/stdlib/doc/src/Makefile index 16e0a86e3b..6c92756ae7 100644 --- a/lib/stdlib/doc/src/Makefile +++ b/lib/stdlib/doc/src/Makefile @@ -83,7 +83,6 @@ XML_REF3_FILES = \ queue.xml \ random.xml \ re.xml \ - regexp.xml \ sets.xml \ shell.xml \ shell_default.xml \ diff --git a/lib/stdlib/doc/src/re.xml b/lib/stdlib/doc/src/re.xml index 18867cfb68..6d5336796c 100644 --- a/lib/stdlib/doc/src/re.xml +++ b/lib/stdlib/doc/src/re.xml @@ -41,8 +41,7 @@ strings and binaries.</p> <p>The regular expression syntax and semantics resemble that of - Perl. This library replaces the deprecated pure-Erlang regexp - library; it has a richer syntax, more options and is faster.</p> + Perl.</p> <p>The library's matching algorithms are currently based on the PCRE library, but not all of the PCRE library is interfaced and diff --git a/lib/stdlib/doc/src/ref_man.xml b/lib/stdlib/doc/src/ref_man.xml index 85aae6151d..6373922c92 100644 --- a/lib/stdlib/doc/src/ref_man.xml +++ b/lib/stdlib/doc/src/ref_man.xml @@ -80,7 +80,6 @@ <xi:include href="queue.xml"/> <xi:include href="random.xml"/> <xi:include href="re.xml"/> - <xi:include href="regexp.xml"/> <xi:include href="sets.xml"/> <xi:include href="shell.xml"/> <xi:include href="shell_default.xml"/> diff --git a/lib/stdlib/doc/src/regexp.xml b/lib/stdlib/doc/src/regexp.xml deleted file mode 100644 index 35d8e1c3f8..0000000000 --- a/lib/stdlib/doc/src/regexp.xml +++ /dev/null @@ -1,381 +0,0 @@ -<?xml version="1.0" encoding="latin1" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> - -<erlref> - <header> - <copyright> - <year>1996</year><year>2011</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - The contents of this file are subject to the Erlang Public License, - Version 1.1, (the "License"); you may not use this file except in - compliance with the License. You should have received a copy of the - Erlang Public License along with this software. If not, it can be - retrieved online at http://www.erlang.org/. - - Software distributed under the License is distributed on an "AS IS" - basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - the License for the specific language governing rights and limitations - under the License. - - </legalnotice> - - <title>regexp</title> - <prepared>Robert Virding</prepared> - <responsible>Bjarne Dacker</responsible> - <docno>1</docno> - <approved>Bjarne Däcker</approved> - <checked></checked> - <date>96-09-28</date> - <rev>A</rev> - <file>regexp.sgml</file> - </header> - <module>regexp</module> - <modulesummary>Regular Expression Functions for Strings</modulesummary> - <description> - <note><p>This module has been obsoleted by the - <seealso marker="re">re</seealso> module and will be removed in a future - release.</p></note> - <p>This module contains functions for regular expression - matching and substitution.</p> - </description> - <datatypes> - <datatype> - <name name="errordesc"></name> - </datatype> - <datatype> - <name name="regexp"></name> - <desc><p>Internal representation of a regular expression.</p></desc> - </datatype> - </datatypes> - <funcs> - <func> - <name name="match" arity="2"/> - <fsummary>Match a regular expression</fsummary> - <desc> - <p>Finds the first, longest match of the regular expression <c><anno>RegExp</anno></c> in <c><anno>String</anno></c>. This function searches for the longest possible match and returns the first one found if there are several expressions of the same length. It returns as follows:</p> - <taglist> - <tag><c>{match,<anno>Start</anno>,<anno>Length</anno>}</c></tag> - <item> - <p>if the match succeeded. <c><anno>Start</anno></c> is the starting - position of the match, and <c><anno>Length</anno></c> is the length of - the matching string.</p> - </item> - <tag><c>nomatch</c></tag> - <item> - <p>if there were no matching characters.</p> - </item> - <tag><c>{error,<anno>Error</anno>}</c></tag> - <item> - <p>if there was an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="first_match" arity="2"/> - <fsummary>Match a regular expression</fsummary> - <desc> - <p>Finds the first match of the regular expression <c><anno>RegExp</anno></c> in <c><anno>String</anno></c>. This call is - usually faster than <c>match</c> and it is also a useful way to ascertain that a match exists. It returns as follows:</p> - <taglist> - <tag><c>{match,<anno>Start</anno>,<anno>Length</anno>}</c></tag> - <item> - <p>if the match succeeded. <c><anno>Start</anno></c> is the starting - position of the match and <c><anno>Length</anno></c> is the length of - the matching string.</p> - </item> - <tag><c>nomatch</c></tag> - <item> - <p>if there were no matching characters.</p> - </item> - <tag><c>{error,<anno>Error</anno>}</c></tag> - <item> - <p>if there was an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="matches" arity="2"/> - <fsummary>Match a regular expression</fsummary> - <desc> - <p>Finds all non-overlapping matches of the - expression <c><anno>RegExp</anno></c> in <c><anno>String</anno></c>. - It returns as follows:</p> - <taglist> - <tag><c>{match, <anno>Matches</anno>}</c></tag> - <item> - <p>if the regular expression was correct. - The list will be empty if there was no match. Each element in the list looks like <c>{<anno>Start</anno>, <anno>Length</anno>}</c>, where <c><anno>Start</anno></c> is the starting position of the match, and <c><anno>Length</anno></c> is the length of the matching string.</p> - </item> - <tag><c>{error,<anno>Error</anno>}</c></tag> - <item> - <p>if there was an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="sub" arity="3"/> - <fsummary>Substitute the first occurrence of a regular expression</fsummary> - <desc> - <p>Substitutes the first occurrence of a substring matching <c><anno>RegExp</anno></c> in <c><anno>String</anno></c> with the string <c><anno>New</anno></c>. A <c><![CDATA[&]]></c> in the string <c><anno>New</anno></c> is replaced by the matched substring of <c><anno>String</anno></c>. <c><![CDATA[\&]]></c> puts a literal <c><![CDATA[&]]></c> into the replacement string. It returns as follows:</p> - <taglist> - <tag><c>{ok,<anno>NewString</anno>,<anno>RepCount</anno>}</c></tag> - <item> - <p>if <c><anno>RegExp</anno></c> is correct. <c><anno>RepCount</anno></c> is the number of replacements which have been made - (this will be either 0 or 1).</p> - </item> - <tag><c>{error, <anno>Error</anno>}</c></tag> - <item> - <p>if there is an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="gsub" arity="3"/> - <fsummary>Substitute all occurrences of a regular expression</fsummary> - <desc> - <p>The same as <c>sub</c>, except that all non-overlapping - occurrences of a substring matching - <c><anno>RegExp</anno></c> in <c><anno>String</anno></c> are replaced by the string <c><anno>New</anno></c>. It returns:</p> - <taglist> - <tag><c>{ok,<anno>NewString</anno>,<anno>RepCount</anno>}</c></tag> - <item> - <p>if <c><anno>RegExp</anno></c> is correct. <c><anno>RepCount</anno></c> is the number of replacements which have been made.</p> - </item> - <tag><c>{error, <anno>Error</anno>}</c></tag> - <item> - <p>if there is an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="split" arity="2"/> - <fsummary>Split a string into fields</fsummary> - <desc> - <p><c><anno>String</anno></c> is split into fields (sub-strings) by the - regular expression <c><anno>RegExp</anno></c>.</p> - <p>If the separator expression is <c>" "</c> (a single space), - then the fields are separated by blanks and/or tabs and - leading and trailing blanks and tabs are discarded. For all - other values of the separator, leading and trailing blanks - and tabs are not discarded. It returns:</p> - <taglist> - <tag><c>{ok, <anno>FieldList</anno>}</c></tag> - <item> - <p>to indicate that the string has been split up into the fields of - <c><anno>FieldList</anno></c>.</p> - </item> - <tag><c>{error, <anno>Error</anno>}</c></tag> - <item> - <p>if there is an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="sh_to_awk" arity="1"/> - <fsummary>Convert an <c>sh</c>regular expression into an <c>AWK</c>one</fsummary> - <desc> - <p>Converts the <c>sh</c> type regular expression - <c><anno>ShRegExp</anno></c> into a full <c>AWK</c> regular - expression. Returns the converted regular expression - string. <c>sh</c> expressions are used in the shell for - matching file names and have the following special - characters:</p> - <taglist> - <tag><c>*</c></tag> - <item> - <p>matches any string including the null string.</p> - </item> - <tag><c>?</c></tag> - <item> - <p>matches any single character.</p> - </item> - <tag><c>[...]</c></tag> - <item> - <p>matches any of the enclosed characters. Character - ranges are specified by a pair of characters separated - by a <c>-</c>. If the first character after <c>[</c> is a - <c>!</c>, then any character not enclosed is matched.</p> - </item> - </taglist> - <p>It may sometimes be more practical to use <c>sh</c> type - expansions as they are simpler and easier to use, even though they are not as powerful.</p> - </desc> - </func> - <func> - <name name="parse" arity="1"/> - <fsummary>Parse a regular expression</fsummary> - <desc> - <p>Parses the regular expression <c><anno>RegExp</anno></c> and builds the - internal representation used in the other regular expression - functions. Such representations can be used in all of the - other functions instead of a regular expression string. This - is more efficient when the same regular expression is used - in many strings. It returns:</p> - <taglist> - <tag><c>{ok, <anno>RE</anno>}</c></tag> - <item> - <p>if <c>RegExp</c> is correct and <c><anno>RE</anno></c> is the internal representation.</p> - </item> - <tag><c>{error, <anno>Error</anno>}</c></tag> - <item> - <p>if there is an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="format_error" arity="1"/> - <fsummary>Format an error descriptor</fsummary> - <desc> - <p>Returns a string which describes the error <c><anno>ErrorDescriptor</anno></c> - returned when there is an error in a regular expression.</p> - </desc> - </func> - </funcs> - - <section> - <title>Regular Expressions</title> - <p>The regular expressions allowed here is a subset of the set found - in <c>egrep</c> and in the <c>AWK</c> programming language, as - defined in the book, <c>The AWK Programming Language, by A. V. Aho, B. W. Kernighan, P. J. Weinberger</c>. They are - composed of the following characters:</p> - <taglist> - <tag>c</tag> - <item> - <p>matches the non-metacharacter <c>c</c>.</p> - </item> - <tag>\c</tag> - <item> - <p>matches the escape sequence or literal character <c>c</c>.</p> - </item> - <tag>.</tag> - <item> - <p>matches any character.</p> - </item> - <tag>^</tag> - <item> - <p>matches the beginning of a string.</p> - </item> - <tag>$</tag> - <item> - <p>matches the end of a string.</p> - </item> - <tag>[abc...]</tag> - <item> - <p>character class, which matches any of the characters - <c>abc...</c> Character ranges are specified by a pair of - characters separated by a <c>-</c>.</p> - </item> - <tag>[^abc...]</tag> - <item> - <p>negated character class, which matches any character except - <c>abc...</c>.</p> - </item> - <tag>r1 | r2</tag> - <item> - <p>alternation. It matches either <c>r1</c> or <c>r2</c>.</p> - </item> - <tag>r1r2</tag> - <item> - <p>concatenation. It matches <c>r1</c> and then <c>r2</c>.</p> - </item> - <tag>r+</tag> - <item> - <p>matches one or more <c>r</c>s.</p> - </item> - <tag>r*</tag> - <item> - <p>matches zero or more <c>r</c>s.</p> - </item> - <tag>r?</tag> - <item> - <p>matches zero or one <c>r</c>s.</p> - </item> - <tag>(r)</tag> - <item> - <p>grouping. It matches <c>r</c>.</p> - </item> - </taglist> - <p>The escape sequences allowed are the same as for Erlang - strings:</p> - <taglist> - <tag><c>\b</c></tag> - <item> - <p>backspace</p> - </item> - <tag><c>\f</c></tag> - <item> - <p>form feed </p> - </item> - <tag><c>\n</c></tag> - <item> - <p>newline (line feed) </p> - </item> - <tag><c>\r</c></tag> - <item> - <p>carriage return </p> - </item> - <tag><c>\t</c></tag> - <item> - <p>tab </p> - </item> - <tag><c>\e</c></tag> - <item> - <p>escape </p> - </item> - <tag><c>\v</c></tag> - <item> - <p>vertical tab </p> - </item> - <tag><c>\s</c></tag> - <item> - <p>space </p> - </item> - <tag><c>\d</c></tag> - <item> - <p>delete </p> - </item> - <tag><c>\ddd</c></tag> - <item> - <p>the octal value ddd </p> - </item> - <tag><c>\xhh</c></tag> - <item> - <p>The hexadecimal value <c>hh</c>.</p> - </item> - <tag><c>\x{h...}</c></tag> - <item> - <p>The hexadecimal value <c>h...</c>.</p> - </item> - <tag><c>\c</c></tag> - <item> - <p>any other character literally, for example <c>\\</c> for backslash, - <c>\"</c> for ")</p> - </item> - </taglist> - <p>To make these functions easier to use, in combination with the - function <c>io:get_line</c> which terminates the input line with - a new line, the <c>$</c> characters also matches a string ending - with <c>"...\n"</c>. The following examples - define Erlang data types:</p> - <pre> -Atoms [a-z][0-9a-zA-Z_]* - -Variables [A-Z_][0-9a-zA-Z_]* - -Floats (\+|-)?[0-9]+\.[0-9]+((E|e)(\+|-)?[0-9]+)?</pre> - <p>Regular expressions are written as Erlang strings when used with the functions in this module. This means that any <c>\</c> or <c>"</c> characters in a regular expression - string must be written with <c>\</c> as they are also escape characters for the string. For example, the regular expression string for Erlang floats is: - <c>"(\\+|-)?[0-9]+\\.[0-9]+((E|e)(\\+|-)?[0-9]+)?"</c>.</p> - <p>It is not really necessary to have the escape sequences as part of the regular expression syntax as they can always be generated directly in the string. They are included for completeness and can they can also be useful when generating regular expressions, or when they are entered other than with Erlang strings.</p> - </section> -</erlref> - diff --git a/lib/stdlib/doc/src/specs.xml b/lib/stdlib/doc/src/specs.xml index 98338b5ec2..49c60529d2 100644 --- a/lib/stdlib/doc/src/specs.xml +++ b/lib/stdlib/doc/src/specs.xml @@ -46,7 +46,6 @@ <xi:include href="../specs/specs_queue.xml"/> <xi:include href="../specs/specs_random.xml"/> <xi:include href="../specs/specs_re.xml"/> - <xi:include href="../specs/specs_regexp.xml"/> <xi:include href="../specs/specs_sets.xml"/> <xi:include href="../specs/specs_shell.xml"/> <xi:include href="../specs/specs_shell_default.xml"/> diff --git a/lib/stdlib/src/Makefile b/lib/stdlib/src/Makefile index 600303d7e1..9ce1f6f5c8 100644 --- a/lib/stdlib/src/Makefile +++ b/lib/stdlib/src/Makefile @@ -105,7 +105,6 @@ MODULES= \ qlc_pt \ queue \ random \ - regexp \ sets \ shell \ shell_default \ diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index 5d45260fe9..e5adb84932 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -2767,12 +2767,6 @@ default_types() -> {var, 1}], dict:from_list([{T, -1} || T <- DefTypes]). -%% R12B-5 -is_newly_introduced_builtin_type({module, 0}) -> true; -is_newly_introduced_builtin_type({node, 0}) -> true; -is_newly_introduced_builtin_type({nonempty_string, 0}) -> true; -is_newly_introduced_builtin_type({term, 0}) -> true; -is_newly_introduced_builtin_type({timeout, 0}) -> true; %% R13 is_newly_introduced_builtin_type({arity, 0}) -> true; is_newly_introduced_builtin_type({array, 0}) -> true; % opaque diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index c1285dab60..e88d0ff194 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -41,37 +41,12 @@ obsolete(Module, Name, Arity) -> no end. -obsolete_1(init, get_flag, 1) -> - {removed, {init, get_argument, 1}, "R12B"}; -obsolete_1(init, get_flags, 0) -> - {removed, {init, get_arguments, 0}, "R12B"}; -obsolete_1(init, get_args, 0) -> - {removed, {init, get_plain_arguments, 0}, "R12B"}; -obsolete_1(unix, cmd, 1) -> - {removed, {os,cmd,1}, "R9B"}; - obsolete_1(net, _, _) -> {deprecated, "module 'net' obsolete; use 'net_adm'"}; obsolete_1(erl_internal, builtins, 0) -> {deprecated, {erl_internal, bif, 2}}; -obsolete_1(string, re_sh_to_awk, 1) -> - {removed, {regexp, sh_to_awk, 1}, "R12B"}; -obsolete_1(string, re_parse, 1) -> - {removed, {regexp, parse, 1}, "R12B"}; -obsolete_1(string, re_match, 2) -> - {removed, {regexp, match, 2}, "R12B"}; -obsolete_1(string, re_sub, 3) -> - {removed, {regexp, sub, 3}, "R12B"}; -obsolete_1(string, re_gsub, 3) -> - {removed, {regexp, gsub, 3}, "R12B"}; -obsolete_1(string, re_split, 2) -> - {removed, {regexp, split, 2}, "R12B"}; - -obsolete_1(string, index, 2) -> - {removed, {string, str, 2}, "R12B"}; - obsolete_1(erl_eval, seq, 2) -> {deprecated, {erl_eval, exprs, 2}}; obsolete_1(erl_eval, seq, 3) -> @@ -81,99 +56,9 @@ obsolete_1(erl_eval, arg_list, 2) -> obsolete_1(erl_eval, arg_list, 3) -> {deprecated, {erl_eval, expr_list, 3}}; -obsolete_1(erl_pp, seq, 1) -> - {removed, {erl_pp, exprs, 1}, "R12B"}; -obsolete_1(erl_pp, seq, 2) -> - {removed, {erl_pp, exprs, 2}, "R12B"}; - -obsolete_1(io, scan_erl_seq, 1) -> - {removed, {io, scan_erl_exprs, 1}, "R12B"}; -obsolete_1(io, scan_erl_seq, 2) -> - {removed, {io, scan_erl_exprs, 2}, "R12B"}; -obsolete_1(io, scan_erl_seq, 3) -> - {removed, {io, scan_erl_exprs, 3}, "R12B"}; -obsolete_1(io, parse_erl_seq, 1) -> - {removed, {io, parse_erl_exprs, 1}, "R12B"}; -obsolete_1(io, parse_erl_seq, 2) -> - {removed, {io, parse_erl_exprs, 2}, "R12B"}; -obsolete_1(io, parse_erl_seq, 3) -> - {removed, {io, parse_erl_exprs, 3}, "R12B"}; -obsolete_1(io, parse_exprs, 2) -> - {removed, {io, parse_erl_exprs, 2}, "R12B"}; - -obsolete_1(io_lib, scan, 1) -> - {removed, {erl_scan, string, 1}, "R12B"}; -obsolete_1(io_lib, scan, 2) -> - {removed, {erl_scan, string, 2}, "R12B"}; -obsolete_1(io_lib, scan, 3) -> - {removed, {erl_scan, tokens, 3}, "R12B"}; -obsolete_1(io_lib, reserved_word, 1) -> - {removed, {erl_scan, reserved_word, 1}, "R12B"}; - -obsolete_1(lists, keymap, 4) -> - {removed, {lists, keymap, 3}, "R12B"}; -obsolete_1(lists, all, 3) -> - {removed, {lists, all, 2}, "R12B"}; -obsolete_1(lists, any, 3) -> - {removed, {lists, any, 2}, "R12B"}; -obsolete_1(lists, map, 3) -> - {removed, {lists, map, 2}, "R12B"}; -obsolete_1(lists, flatmap, 3) -> - {removed, {lists, flatmap, 2}, "R12B"}; -obsolete_1(lists, foldl, 4) -> - {removed, {lists, foldl, 3}, "R12B"}; -obsolete_1(lists, foldr, 4) -> - {removed, {lists, foldr, 3}, "R12B"}; -obsolete_1(lists, mapfoldl, 4) -> - {removed, {lists, mapfoldl, 3}, "R12B"}; -obsolete_1(lists, mapfoldr, 4) -> - {removed, {lists, mapfoldr, 3}, "R12B"}; -obsolete_1(lists, filter, 3) -> - {removed, {lists, filter, 2}, "R12B"}; -obsolete_1(lists, foreach, 3) -> - {removed, {lists, foreach, 2}, "R12B"}; -obsolete_1(lists, zf, 3) -> - {removed, {lists, zf, 2}, "R12B"}; - -obsolete_1(ets, fixtable, 2) -> - {removed, {ets, safe_fixtable, 2}, "R12B"}; - -obsolete_1(erlang, old_binary_to_term, 1) -> - {removed, {erlang, binary_to_term, 1}, "R12B"}; -obsolete_1(erlang, info, 1) -> - {removed, {erlang, system_info, 1}, "R12B"}; obsolete_1(erlang, hash, 2) -> {deprecated, {erlang, phash2, 2}}; -obsolete_1(file, file_info, 1) -> - {removed, {file, read_file_info, 1}, "R12B"}; - -obsolete_1(dict, dict_to_list, 1) -> - {removed, {dict,to_list,1}, "R12B"}; -obsolete_1(dict, list_to_dict, 1) -> - {removed, {dict,from_list,1}, "R12B"}; -obsolete_1(orddict, dict_to_list, 1) -> - {removed, {orddict,to_list,1}, "R12B"}; -obsolete_1(orddict, list_to_dict, 1) -> - {removed, {orddict,from_list,1}, "R12B"}; - -obsolete_1(sets, new_set, 0) -> - {removed, {sets, new, 0}, "R12B"}; -obsolete_1(sets, set_to_list, 1) -> - {removed, {sets, to_list, 1}, "R12B"}; -obsolete_1(sets, list_to_set, 1) -> - {removed, {sets, from_list, 1}, "R12B"}; -obsolete_1(sets, subset, 2) -> - {removed, {sets, is_subset, 2}, "R12B"}; -obsolete_1(ordsets, new_set, 0) -> - {removed, {ordsets, new, 0}, "R12B"}; -obsolete_1(ordsets, set_to_list, 1) -> - {removed, {ordsets, to_list, 1}, "R12B"}; -obsolete_1(ordsets, list_to_set, 1) -> - {removed, {ordsets, from_list, 1}, "R12B"}; -obsolete_1(ordsets, subset, 2) -> - {removed, {ordsets, is_subset, 2}, "R12B"}; - obsolete_1(calendar, local_time_to_universal_time, 1) -> {deprecated, {calendar, local_time_to_universal_time_dst, 1}}; @@ -302,17 +187,6 @@ obsolete_1(auth, node_cookie, 1) -> obsolete_1(auth, node_cookie, 2) -> {deprecated, "Deprecated; use erlang:set_cookie/2 and net_adm:ping/1 instead"}; -%% Added in R11B-5. -obsolete_1(http_base_64, _, _) -> - {removed, "The http_base_64 module was removed in R12B; use the base64 module instead"}; -obsolete_1(httpd_util, encode_base64, 1) -> - {removed, "Removed in R12B; use one of the encode functions in the base64 module instead"}; -obsolete_1(httpd_util, decode_base64, 1) -> - {removed, "Removed in R12B; use one of the decode functions in the base64 module instead"}; -obsolete_1(httpd_util, to_upper, 1) -> - {removed, {string, to_upper, 1}, "R12B"}; -obsolete_1(httpd_util, to_lower, 1) -> - {removed, {string, to_lower, 1}, "R12B"}; obsolete_1(erlang, is_constant, 1) -> {removed, "Removed in R13B"}; @@ -431,7 +305,7 @@ obsolete_1(ssh_sshd, stop, 1) -> %% Added in R13A. obsolete_1(regexp, _, _) -> - {deprecated, "the regexp module is deprecated (will be removed in R15A); use the re module instead"}; + {removed, "removed in R15; use the re module instead"}; obsolete_1(lists, flat_length, 1) -> {removed,{lists,flatlength,1},"R14"}; diff --git a/lib/stdlib/src/regexp.erl b/lib/stdlib/src/regexp.erl deleted file mode 100644 index 65f9ca247d..0000000000 --- a/lib/stdlib/src/regexp.erl +++ /dev/null @@ -1,557 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% --module(regexp). - -%% This entire module is deprecated and will be removed in a future -%% release. Use the 're' module instead. -%% -%% This module provides a basic set of regular expression functions -%% for strings. The functions provided are taken from AWK. -%% -%% Note that we interpret the syntax tree of a regular expression -%% directly instead of converting it to an NFA and then interpreting -%% that. This method seems to go significantly faster. - --export([sh_to_awk/1,parse/1,format_error/1,match/2,first_match/2,matches/2]). --export([sub/3,gsub/3,split/2]). - --deprecated([sh_to_awk/1,parse/1,format_error/1,match/2,first_match/2,matches/2]). --deprecated([sub/3,gsub/3,split/2]). - --import(string, [substr/2,substr/3]). --import(lists, [reverse/1]). - --type errordesc() :: term(). --opaque regexp() :: term(). - -%% -type matchres() = {match,Start,Length} | nomatch | {error,E}. -%% -type subres() = {ok,RepString,RepCount} | {error,E}. -%% -type splitres() = {ok,[SubString]} | {error,E}. - -%%-compile([export_all]). - -%% This is the regular expression grammar used. It is equivalent to the -%% one used in AWK, except that we allow ^ $ to be used anywhere and fail -%% in the matching. -%% -%% reg -> reg1 : '$1'. -%% reg1 -> reg1 "|" reg2 : {'or','$1','$2'}. -%% reg1 -> reg2 : '$1'. -%% reg2 -> reg2 reg3 : {concat,'$1','$2'}. -%% reg2 -> reg3 : '$1'. -%% reg3 -> reg3 "*" : {kclosure,'$1'}. -%% reg3 -> reg3 "+" : {pclosure,'$1'}. -%% reg3 -> reg3 "?" : {optional,'$1'}. -%% reg3 -> reg4 : '$1'. -%% reg4 -> "(" reg ")" : '$2'. -%% reg4 -> "\\" char : '$2'. -%% reg4 -> "^" : bos. -%% reg4 -> "$" : eos. -%% reg4 -> "." : char. -%% reg4 -> "[" class "]" : {char_class,char_class('$2')} -%% reg4 -> "[" "^" class "]" : {comp_class,char_class('$3')} -%% reg4 -> "\"" chars "\"" : char_string('$2') -%% reg4 -> char : '$1'. -%% reg4 -> empty : epsilon. -%% The grammar of the current regular expressions. The actual parser -%% is a recursive descent implementation of the grammar. - -reg(S) -> reg1(S). - -%% reg1 -> reg2 reg1' -%% reg1' -> "|" reg2 -%% reg1' -> empty - -reg1(S0) -> - {L,S1} = reg2(S0), - reg1p(S1, L). - -reg1p([$||S0], L) -> - {R,S1} = reg2(S0), - reg1p(S1, {'or',L,R}); -reg1p(S, L) -> {L,S}. - -%% reg2 -> reg3 reg2' -%% reg2' -> reg3 -%% reg2' -> empty - -reg2(S0) -> - {L,S1} = reg3(S0), - reg2p(S1, L). - -reg2p([C|S0], L) when C =/= $|, C =/= $) -> - {R,S1} = reg3([C|S0]), - reg2p(S1, {concat,L,R}); -reg2p(S, L) -> {L,S}. - -%% reg3 -> reg4 reg3' -%% reg3' -> "*" reg3' -%% reg3' -> "+" reg3' -%% reg3' -> "?" reg3' -%% reg3' -> empty - -reg3(S0) -> - {L,S1} = reg4(S0), - reg3p(S1, L). - -reg3p([$*|S], L) -> reg3p(S, {kclosure,L}); -reg3p([$+|S], L) -> reg3p(S, {pclosure,L}); -reg3p([$?|S], L) -> reg3p(S, {optional,L}); -reg3p(S, L) -> {L,S}. - --define(HEX(C), C >= $0 andalso C =< $9 orelse - C >= $A andalso C =< $F orelse - C >= $a andalso C =< $f). - -reg4([$(|S0]) -> - case reg(S0) of - {R,[$)|S1]} -> {R,S1}; - {_R,_S} -> throw({error,{unterminated,"("}}) - end; -reg4([$\\,O1,O2,O3|S]) when - O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> - {(O1*8 + O2)*8 + O3 - 73*$0,S}; -reg4([$\\,$x,H1,H2|S]) when ?HEX(H1), ?HEX(H2) -> - {erlang:list_to_integer([H1,H2], 16),S}; -reg4([$\\,$x,${|S]) -> - hex(S, []); -reg4([$\\,$x|_]) -> - throw({error,{illegal,[$x]}}); -reg4([$\\,C|S]) -> {escape_char(C),S}; -reg4([$\\]) -> throw({error,{unterminated,"\\"}}); -reg4([$^|S]) -> {bos,S}; -reg4([$$|S]) -> {eos,S}; -reg4([$.|S]) -> {{comp_class,"\n"},S}; -reg4("[^" ++ S0) -> - case char_class(S0) of - {Cc,[$]|S1]} -> {{comp_class,Cc},S1}; - {_Cc,_S} -> throw({error,{unterminated,"["}}) - end; -reg4([$[|S0]) -> - case char_class(S0) of - {Cc,[$]|S1]} -> {{char_class,Cc},S1}; - {_Cc,_S1} -> throw({error,{unterminated,"["}}) - end; -%reg4([$"|S0]) -> -% case char_string(S0) of -% {St,[$"|S1]} -> {St,S1}; -% {St,S1} -> throw({error,{unterminated,"\""}}) -% end; -reg4([C|S]) when C =/= $*, C =/= $+, C =/= $?, C =/= $] -> {C,S}; -reg4([C|_S]) -> throw({error,{illegal,[C]}}); -reg4([]) -> {epsilon,[]}. - -hex([C|Cs], L) when ?HEX(C) -> - hex(Cs, [C|L]); -hex([$}|S], L) -> - case catch erlang:list_to_integer(lists:reverse(L), 16) of - V when V =< 16#FF -> - {V,S}; - _ -> - throw({error,{illegal,[$}]}}) - end; -hex(_S, _) -> - throw({error,{unterminated,"\\x{"}}). - -escape_char($n) -> $\n; %\n = LF -escape_char($r) -> $\r; %\r = CR -escape_char($t) -> $\t; %\t = TAB -escape_char($v) -> $\v; %\v = VT -escape_char($b) -> $\b; %\b = BS -escape_char($f) -> $\f; %\f = FF -escape_char($e) -> $\e; %\e = ESC -escape_char($s) -> $\s; %\s = SPACE -escape_char($d) -> $\d; %\d = DEL -escape_char(C) -> C. - -char_class([$]|S]) -> char_class(S, [$]]); -char_class(S) -> char_class(S, []). - -char($\\, [O1,O2,O3|S]) when - O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> - {(O1*8 + O2)*8 + O3 - 73*$0,S}; -char($\\, [$x,H1,H2|S]) when ?HEX(H1), ?HEX(H2) -> - {erlang:list_to_integer([H1,H2], 16),S}; -char($\\,[$x,${|S]) -> - hex(S, []); -char($\\,[$x|_]) -> - throw({error,{illegal,[$x]}}); -char($\\, [C|S]) -> {escape_char(C),S}; -char(C, S) -> {C,S}. - -char_class([C1|S0], Cc) when C1 =/= $] -> - case char(C1, S0) of - {Cf,[$-,C2|S1]} when C2 =/= $] -> - case char(C2, S1) of - {Cl,S2} when Cf < Cl -> char_class(S2, [{Cf,Cl}|Cc]); - {Cl,_S2} -> throw({error,{char_class,[Cf,$-,Cl]}}) - end; - {C,S1} -> char_class(S1, [C|Cc]) - end; -char_class(S, Cc) -> {Cc,S}. - -%char_string([C|S]) when C =/= $" -> char_string(S, C); -%char_string(S) -> {epsilon,S}. - -%char_string([C|S0], L) when C =/= $" -> -% char_string(S0, {concat,L,C}); -%char_string(S, L) -> {L,S}. - -%% -deftype re_app_res() = {match,RestPos,Rest} | nomatch. - -%% re_apply(String, StartPos, RegExp) -> re_app_res(). -%% -%% Apply the (parse of the) regular expression RegExp to String. If -%% there is a match return the position of the remaining string and -%% the string if else return 'nomatch'. BestMatch specifies if we want -%% the longest match, or just a match. -%% -%% StartPos should be the real start position as it is used to decide -%% if we ae at the beginning of the string. -%% -%% Pass two functions to re_apply_or so it can decide, on the basis -%% of BestMatch, whether to just any take any match or try both to -%% find the longest. This is slower but saves duplicatng code. - -re_apply(S, St, RE) -> re_apply(RE, [], S, St). - -re_apply(epsilon, More, S, P) -> %This always matches - re_apply_more(More, S, P); -re_apply({'or',RE1,RE2}, More, S, P) -> - re_apply_or(re_apply(RE1, More, S, P), - re_apply(RE2, More, S, P)); -re_apply({concat,RE1,RE2}, More, S0, P) -> - re_apply(RE1, [RE2|More], S0, P); -re_apply({kclosure,CE}, More, S, P) -> - %% Be careful with the recursion, explicitly do one call before - %% looping. - re_apply_or(re_apply_more(More, S, P), - re_apply(CE, [{kclosure,CE}|More], S, P)); -re_apply({pclosure,CE}, More, S, P) -> - re_apply(CE, [{kclosure,CE}|More], S, P); -re_apply({optional,CE}, More, S, P) -> - re_apply_or(re_apply_more(More, S, P), - re_apply(CE, More, S, P)); -re_apply(bos, More, S, 1) -> re_apply_more(More, S, 1); -re_apply(eos, More, [$\n|S], P) -> re_apply_more(More, S, P); -re_apply(eos, More, [], P) -> re_apply_more(More, [], P); -re_apply({char_class,Cc}, More, [C|S], P) -> - case in_char_class(C, Cc) of - true -> re_apply_more(More, S, P+1); - false -> nomatch - end; -re_apply({comp_class,Cc}, More, [C|S], P) -> - case in_char_class(C, Cc) of - true -> nomatch; - false -> re_apply_more(More, S, P+1) - end; -re_apply(C, More, [C|S], P) when is_integer(C) -> - re_apply_more(More, S, P+1); -re_apply(_RE, _More, _S, _P) -> nomatch. - -%% re_apply_more([RegExp], String, Length) -> re_app_res(). - -re_apply_more([RE|More], S, P) -> re_apply(RE, More, S, P); -re_apply_more([], S, P) -> {match,P,S}. - -%% in_char_class(Char, Class) -> bool(). - -in_char_class(C, [{C1,C2}|_Cc]) when C >= C1, C =< C2 -> true; -in_char_class(C, [C|_Cc]) -> true; -in_char_class(C, [_|Cc]) -> in_char_class(C, Cc); -in_char_class(_C, []) -> false. - -%% re_apply_or(Match1, Match2) -> re_app_res(). -%% If we want the best match then choose the longest match, else just -%% choose one by trying sequentially. - -re_apply_or({match,P1,S1}, {match,P2,_S2}) when P1 >= P2 -> {match,P1,S1}; -re_apply_or({match,_P1,_S1}, {match,P2,S2}) -> {match,P2,S2}; -re_apply_or(nomatch, R2) -> R2; -re_apply_or(R1, nomatch) -> R1. - -%% sh_to_awk(ShellRegExp) -%% Convert a sh style regexp into a full AWK one. The main difficulty is -%% getting character sets right as the conventions are different. - --spec sh_to_awk(ShRegExp) -> AwkRegExp when - ShRegExp :: string(), - AwkRegExp :: string(). - -sh_to_awk(Sh) -> "^(" ++ sh_to_awk_1(Sh). %Fix the beginning - -sh_to_awk_1([$*|Sh]) -> %This matches any string - ".*" ++ sh_to_awk_1(Sh); -sh_to_awk_1([$?|Sh]) -> %This matches any character - [$.|sh_to_awk_1(Sh)]; -sh_to_awk_1([$[,$^,$]|Sh]) -> %This takes careful handling - "\\^" ++ sh_to_awk_1(Sh); -sh_to_awk_1("[^" ++ Sh) -> [$[|sh_to_awk_2(Sh, true)]; -sh_to_awk_1("[!" ++ Sh) -> "[^" ++ sh_to_awk_2(Sh, false); -sh_to_awk_1([$[|Sh]) -> [$[|sh_to_awk_2(Sh, false)]; -sh_to_awk_1([C|Sh]) -> - %% Unspecialise everything else which is not an escape character. - case special_char(C) of - true -> [$\\,C|sh_to_awk_1(Sh)]; - false -> [C|sh_to_awk_1(Sh)] - end; -sh_to_awk_1([]) -> ")$". %Fix the end - -sh_to_awk_2([$]|Sh], UpArrow) -> [$]|sh_to_awk_3(Sh, UpArrow)]; -sh_to_awk_2(Sh, UpArrow) -> sh_to_awk_3(Sh, UpArrow). - -sh_to_awk_3([$]|Sh], true) -> "^]" ++ sh_to_awk_1(Sh); -sh_to_awk_3([$]|Sh], false) -> [$]|sh_to_awk_1(Sh)]; -sh_to_awk_3([C|Sh], UpArrow) -> [C|sh_to_awk_3(Sh, UpArrow)]; -sh_to_awk_3([], true) -> [$^|sh_to_awk_1([])]; -sh_to_awk_3([], false) -> sh_to_awk_1([]). - -%% -type special_char(char()) -> bool(). -%% Test if a character is a special character. - -special_char($|) -> true; -special_char($*) -> true; -special_char($+) -> true; -special_char($?) -> true; -special_char($() -> true; -special_char($)) -> true; -special_char($\\) -> true; -special_char($^) -> true; -special_char($$) -> true; -special_char($.) -> true; -special_char($[) -> true; -special_char($]) -> true; -special_char($") -> true; -special_char(_C) -> false. - -%% parse(RegExp) -> {ok,RE} | {error,E}. -%% Parse the regexp described in the string RegExp. - --spec parse(RegExp) -> ParseRes when - RegExp :: string(), - ParseRes :: {ok, RE} | {error, Error}, - RE :: regexp(), - Error :: errordesc(). - -parse(S) -> - case catch reg(S) of - {R,[]} -> {ok,R}; - {_R,[C|_]} -> {error,{illegal,[C]}}; - {error,E} -> {error,E} - end. - -%% format_error(Error) -> String. - --spec format_error(ErrorDescriptor) -> Chars when - ErrorDescriptor :: errordesc(), - Chars :: io_lib:chars(). - -format_error({illegal,What}) -> ["illegal character `",What,"'"]; -format_error({unterminated,What}) -> ["unterminated `",What,"'"]; -format_error({char_class,What}) -> - ["illegal character class ",io_lib:write_string(What)]. - -%% -type match(String, RegExp) -> matchres(). -%% Find the longest match of RegExp in String. - --spec match(String, RegExp) -> MatchRes when - String :: string(), - RegExp :: string() | regexp(), - MatchRes :: {match, Start, Length} | nomatch | {error, Error}, - Start :: pos_integer(), - Length :: pos_integer(), - Error :: errordesc(). - -match(S, RegExp) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> match(S, RE); - {error,E} -> {error,E} - end; -match(S, RE) -> - case match(RE, S, 1, 0, -1) of - {Start,Len} when Len >= 0 -> - {match,Start,Len}; - {_Start,_Len} -> nomatch - end. - -match(RE, S, St, Pos, L) -> - case first_match(RE, S, St) of - {St1,L1} -> - Nst = St1 + 1, - if L1 > L -> match(RE, lists:nthtail(Nst-St, S), Nst, St1, L1); - true -> match(RE, lists:nthtail(Nst-St, S), Nst, Pos, L) - end; - nomatch -> {Pos,L} - end. - -%% -type first_match(String, RegExp) -> matchres(). -%% Find the first match of RegExp in String. - --spec first_match(String, RegExp) -> MatchRes when - String :: string(), - RegExp :: string() | regexp(), - MatchRes :: {match, Start, Length} | nomatch | {error, Error}, - Start :: pos_integer(), - Length :: pos_integer(), - Error :: errordesc(). - -first_match(S, RegExp) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> first_match(S, RE); - {error,E} -> {error,E} - end; -first_match(S, RE) -> - case first_match(RE, S, 1) of - {Start,Len} when Len >= 0 -> - {match,Start,Len}; - nomatch -> nomatch - end. - -first_match(RE, S, St) when S =/= [] -> - case re_apply(S, St, RE) of - {match,P,_Rest} -> {St,P-St}; - nomatch -> first_match(RE, tl(S), St+1) - end; -first_match(_RE, [], _St) -> nomatch. - -%% -type matches(String, RegExp) -> {match,[{Start,Length}]} | {error,E}. -%% Return the all the non-overlapping matches of RegExp in String. - --spec matches(String, RegExp) -> MatchRes when - String :: string(), - RegExp :: string() | regexp(), - MatchRes :: {match, Matches} | {error, Error}, - Matches :: [{Start, Length}], - Start :: pos_integer(), - Length :: pos_integer(), - Error :: errordesc(). - -matches(S, RegExp) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> matches(S, RE); - {error,E} -> {error,E} - end; -matches(S, RE) -> - {match,matches(S, RE, 1)}. - -matches(S, RE, St) -> - case first_match(RE, S, St) of - {St1,0} -> [{St1,0}|matches(substr(S, St1+2-St), RE, St1+1)]; - {St1,L1} -> [{St1,L1}|matches(substr(S, St1+L1+1-St), RE, St1+L1)]; - nomatch -> [] - end. - -%% -type sub(String, RegExp, Replace) -> subsres(). -%% Substitute the first match of the regular expression RegExp with -%% the string Replace in String. Accept pre-parsed regular -%% expressions. - --spec sub(String, RegExp, New) -> SubRes when - String :: string(), - RegExp :: string() | regexp(), - New :: string(), - NewString :: string(), - SubRes :: {ok, NewString, RepCount} | {error, Error}, - RepCount :: 0 | 1, - Error :: errordesc(). - -sub(String, RegExp, Rep) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> sub(String, RE, Rep); - {error,E} -> {error,E} - end; -sub(String, RE, Rep) -> - Ss = sub_match(String, RE, 1), - {ok,sub_repl(Ss, Rep, String, 1),length(Ss)}. - -sub_match(S, RE, St) -> - case first_match(RE, S, St) of - {St1,L1} -> [{St1,L1}]; - nomatch -> [] - end. - -sub_repl([{St,L}|Ss], Rep, S, Pos) -> - Rs = sub_repl(Ss, Rep, S, St+L), - substr(S, Pos, St-Pos) ++ sub_repl(Rep, substr(S, St, L), Rs); -sub_repl([], _Rep, S, Pos) -> substr(S, Pos). - -sub_repl([$&|Rep], M, Rest) -> M ++ sub_repl(Rep, M, Rest); -sub_repl("\\&" ++ Rep, M, Rest) -> [$&|sub_repl(Rep, M, Rest)]; -sub_repl([C|Rep], M, Rest) -> [C|sub_repl(Rep, M, Rest)]; -sub_repl([], _M, Rest) -> Rest. - -%% -type gsub(String, RegExp, Replace) -> subres(). -%% Substitute every match of the regular expression RegExp with the -%% string New in String. Accept pre-parsed regular expressions. - --spec gsub(String, RegExp, New) -> SubRes when - String :: string(), - RegExp :: string() | regexp(), - New :: string(), - NewString :: string(), - SubRes :: {ok, NewString, RepCount} | {error, Error}, - RepCount :: non_neg_integer(), - Error :: errordesc(). - -gsub(String, RegExp, Rep) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> gsub(String, RE, Rep); - {error,E} -> {error,E} - end; -gsub(String, RE, Rep) -> - Ss = matches(String, RE, 1), - {ok,sub_repl(Ss, Rep, String, 1),length(Ss)}. - -%% -type split(String, RegExp) -> splitres(). -%% Split a string into substrings where the RegExp describes the -%% field seperator. The RegExp " " is specially treated. - --spec split(String, RegExp) -> SplitRes when - String :: string(), - RegExp :: string() | regexp(), - SplitRes :: {ok, FieldList} | {error, Error}, - FieldList :: [string()], - Error :: errordesc(). - -split(String, " ") -> %This is really special - {ok,RE} = parse("[ \t]+"), - case split_apply(String, RE, true) of - [[]|Ss] -> {ok,Ss}; - Ss -> {ok,Ss} - end; -split(String, RegExp) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> {ok,split_apply(String, RE, false)}; - {error,E} -> {error,E} - end; -split(String, RE) -> {ok,split_apply(String, RE, false)}. - -split_apply(S, RE, Trim) -> split_apply(S, 1, RE, Trim, []). - -split_apply([], _P, _RE, true, []) -> []; -split_apply([], _P, _RE, _T, Sub) -> [reverse(Sub)]; -split_apply(S, P, RE, T, Sub) -> - case re_apply(S, P, RE) of - {match,P,_Rest} -> - split_apply(tl(S), P+1, RE, T, [hd(S)|Sub]); - {match,P1,Rest} -> - [reverse(Sub)|split_apply(Rest, P1, RE, T, [])]; - nomatch -> - split_apply(tl(S), P+1, RE, T, [hd(S)|Sub]) - end. diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index 9d15f01683..da65db4b9d 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -85,7 +85,6 @@ queue, random, re, - regexp, sets, shell, shell_default, diff --git a/system/doc/efficiency_guide/commoncaveats.xml b/system/doc/efficiency_guide/commoncaveats.xml index 61d13636c0..7f2d51e609 100644 --- a/system/doc/efficiency_guide/commoncaveats.xml +++ b/system/doc/efficiency_guide/commoncaveats.xml @@ -33,20 +33,6 @@ from a performance point of view.</p> <section> - <title>The regexp module</title> - - <p>The regular expression functions in the - <seealso marker="stdlib:regexp">regexp</seealso> - module are written in Erlang, not in C, and were - meant for occasional use on small amounts of data, - for instance for validation of configuration files - when starting an application.</p> - - <p>Use the <seealso marker="stdlib:re">re</seealso> module - (introduced in R13A) instead, especially in time-critical code.</p> - </section> - - <section> <title>The timer module</title> <p>Creating timers using <seealso |