diff options
35 files changed, 150 insertions, 1021 deletions
diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex 0b1fd8e039..d0ac42d595 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex 0b1fd8e039..d0ac42d595 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/bootstrap/lib/compiler/ebin/beam_type.beam b/bootstrap/lib/compiler/ebin/beam_type.beam Binary files differindex 67e6dbd8a1..6689135457 100644 --- a/bootstrap/lib/compiler/ebin/beam_type.beam +++ b/bootstrap/lib/compiler/ebin/beam_type.beam diff --git a/bootstrap/lib/kernel/ebin/application_controller.beam b/bootstrap/lib/kernel/ebin/application_controller.beam Binary files differindex 565691df01..38614f1285 100644 --- a/bootstrap/lib/kernel/ebin/application_controller.beam +++ b/bootstrap/lib/kernel/ebin/application_controller.beam diff --git a/bootstrap/lib/kernel/ebin/code_server.beam b/bootstrap/lib/kernel/ebin/code_server.beam Binary files differindex f0be53008b..567dcf93df 100644 --- a/bootstrap/lib/kernel/ebin/code_server.beam +++ b/bootstrap/lib/kernel/ebin/code_server.beam diff --git a/bootstrap/lib/kernel/ebin/global.beam b/bootstrap/lib/kernel/ebin/global.beam Binary files differindex 8049d8d729..b4a6a5dc3e 100644 --- a/bootstrap/lib/kernel/ebin/global.beam +++ b/bootstrap/lib/kernel/ebin/global.beam diff --git a/bootstrap/lib/kernel/ebin/inet_db.beam b/bootstrap/lib/kernel/ebin/inet_db.beam Binary files differindex d2c636d21f..555c8c307e 100644 --- a/bootstrap/lib/kernel/ebin/inet_db.beam +++ b/bootstrap/lib/kernel/ebin/inet_db.beam diff --git a/bootstrap/lib/kernel/ebin/inet_sctp.beam b/bootstrap/lib/kernel/ebin/inet_sctp.beam Binary files differindex 063e17f388..098a2c9277 100644 --- a/bootstrap/lib/kernel/ebin/inet_sctp.beam +++ b/bootstrap/lib/kernel/ebin/inet_sctp.beam diff --git a/bootstrap/lib/stdlib/ebin/beam_lib.beam b/bootstrap/lib/stdlib/ebin/beam_lib.beam Binary files differindex b1f5baf3b8..01fc86def0 100644 --- a/bootstrap/lib/stdlib/ebin/beam_lib.beam +++ b/bootstrap/lib/stdlib/ebin/beam_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/dets.beam b/bootstrap/lib/stdlib/ebin/dets.beam Binary files differindex 47cc5e70bb..73be548a89 100644 --- a/bootstrap/lib/stdlib/ebin/dets.beam +++ b/bootstrap/lib/stdlib/ebin/dets.beam diff --git a/bootstrap/lib/stdlib/ebin/filename.beam b/bootstrap/lib/stdlib/ebin/filename.beam Binary files differindex 35031334c4..ad7cf40df0 100644 --- a/bootstrap/lib/stdlib/ebin/filename.beam +++ b/bootstrap/lib/stdlib/ebin/filename.beam diff --git a/bootstrap/lib/stdlib/ebin/gb_sets.beam b/bootstrap/lib/stdlib/ebin/gb_sets.beam Binary files differindex d42fed4c30..dc4c752654 100644 --- a/bootstrap/lib/stdlib/ebin/gb_sets.beam +++ b/bootstrap/lib/stdlib/ebin/gb_sets.beam diff --git a/bootstrap/lib/stdlib/ebin/gen.beam b/bootstrap/lib/stdlib/ebin/gen.beam Binary files differindex 806515e2e0..db59451d78 100644 --- a/bootstrap/lib/stdlib/ebin/gen.beam +++ b/bootstrap/lib/stdlib/ebin/gen.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_fsm.beam b/bootstrap/lib/stdlib/ebin/gen_fsm.beam Binary files differindex 5ee3a82cb6..03f40579cb 100644 --- a/bootstrap/lib/stdlib/ebin/gen_fsm.beam +++ b/bootstrap/lib/stdlib/ebin/gen_fsm.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_server.beam b/bootstrap/lib/stdlib/ebin/gen_server.beam Binary files differindex e37fa78649..2391903567 100644 --- a/bootstrap/lib/stdlib/ebin/gen_server.beam +++ b/bootstrap/lib/stdlib/ebin/gen_server.beam diff --git a/bootstrap/lib/stdlib/ebin/otp_internal.beam b/bootstrap/lib/stdlib/ebin/otp_internal.beam Binary files differindex f22f936f42..07a627b2fd 100644 --- a/bootstrap/lib/stdlib/ebin/otp_internal.beam +++ b/bootstrap/lib/stdlib/ebin/otp_internal.beam diff --git a/bootstrap/lib/stdlib/ebin/random.beam b/bootstrap/lib/stdlib/ebin/random.beam Binary files differindex 9df28216b0..bc8c5ef31f 100644 --- a/bootstrap/lib/stdlib/ebin/random.beam +++ b/bootstrap/lib/stdlib/ebin/random.beam diff --git a/bootstrap/lib/stdlib/ebin/regexp.beam b/bootstrap/lib/stdlib/ebin/regexp.beam Binary files differdeleted file mode 100644 index 023f2fb9b2..0000000000 --- a/bootstrap/lib/stdlib/ebin/regexp.beam +++ /dev/null diff --git a/bootstrap/lib/stdlib/ebin/supervisor.beam b/bootstrap/lib/stdlib/ebin/supervisor.beam Binary files differindex d1fe9e0cf5..9d3b9ddd87 100644 --- a/bootstrap/lib/stdlib/ebin/supervisor.beam +++ b/bootstrap/lib/stdlib/ebin/supervisor.beam diff --git a/bootstrap/lib/stdlib/ebin/zip.beam b/bootstrap/lib/stdlib/ebin/zip.beam Binary files differindex 051c688e73..ca76a6fa3b 100644 --- a/bootstrap/lib/stdlib/ebin/zip.beam +++ b/bootstrap/lib/stdlib/ebin/zip.beam diff --git a/erts/vsn.mk b/erts/vsn.mk index d56d03146d..1b3cd67947 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # VSN = 5.9 -SYSTEM_VSN = R15A +SYSTEM_VSN = R15B # Port number 4365 in 4.2 # Port number 4366 in 4.3 diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index e3d22e7999..32a12e2b52 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -32,6 +32,8 @@ -import(lists, [foreach/2]). +-define(ANY_NATIVE_CODE_LOADED, any_native_code_loaded). + -record(state, {supervisor, root, path, @@ -97,6 +99,8 @@ init(Ref, Parent, [Root,Mode0]) -> State0 end, + put(?ANY_NATIVE_CODE_LOADED, false), + Parent ! {Ref,{ok,self()}}, loop(State#state{supervisor = Parent}). @@ -1278,20 +1282,35 @@ load_native_code(Mod, Bin) -> %% Therefore we must test for that the loader modules are available %% before trying to to load native code. case erlang:module_loaded(hipe_unified_loader) of - false -> no_native; - true -> hipe_unified_loader:load_native_code(Mod, Bin) + false -> + no_native; + true -> + Result = hipe_unified_loader:load_native_code(Mod, Bin), + case Result of + {module,_} -> + put(?ANY_NATIVE_CODE_LOADED, true); + _ -> + ok + end, + Result end. hipe_result_to_status(Result) -> case Result of - {module,_} -> Result; - _ -> {error,Result} + {module,_} -> + put(?ANY_NATIVE_CODE_LOADED, true), + Result; + _ -> + {error,Result} end. post_beam_load(Mod) -> - case erlang:module_loaded(hipe_unified_loader) of - false -> ok; - true -> hipe_unified_loader:post_beam_load(Mod) + %% post_beam_load/1 can potentially be very expensive because it + %% blocks multi-scheduling; thus we want to avoid the call if we + %% know that it is not needed. + case get(?ANY_NATIVE_CODE_LOADED) of + true -> hipe_unified_loader:post_beam_load(Mod); + false -> ok end. int_list([H|T]) when is_integer(H) -> int_list(T); diff --git a/lib/sasl/src/release_handler.erl b/lib/sasl/src/release_handler.erl index 4e8cb4628c..b6ef8582c4 100644 --- a/lib/sasl/src/release_handler.erl +++ b/lib/sasl/src/release_handler.erl @@ -1050,38 +1050,50 @@ new_emulator_make_tmp_release(CurrentRelease,ToRelease,RelDir,Opts,Masters) -> CurrentVsn = CurrentRelease#release.vsn, ToVsn = ToRelease#release.vsn, TmpVsn = ?tmp_vsn(CurrentVsn), - BaseApps = [kernel,stdlib,sasl], - BaseLibs = [{App,Vsn,Lib} || {App,Vsn,Lib} <- ToRelease#release.libs, - lists:member(App,BaseApps)], - check_base_libs(BaseLibs,ToVsn), - OldBaseLibs = [{App,Vsn,Lib} || {App,Vsn,Lib} <- CurrentRelease#release.libs, - lists:member(App,BaseApps)], - check_base_libs(OldBaseLibs,CurrentVsn), - RestLibs = [{App,Vsn,Lib} || {App,Vsn,Lib} <- CurrentRelease#release.libs, - not lists:member(App,BaseApps)], - TmpRelease = CurrentRelease#release{vsn=TmpVsn, - erts_vsn=ToRelease#release.erts_vsn, - libs = BaseLibs ++ RestLibs, - status = unpacked}, - new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn,BaseLibs, - RelDir,Opts,Masters), - new_emulator_make_hybrid_config(CurrentVsn,ToVsn,TmpVsn,RelDir,Masters), - {TmpVsn,TmpRelease}. - -check_base_libs([_,_,_]=BaseLibs,_Vsn) -> - [Kernel,Sasl,Stdlib] = lists:keysort(1,BaseLibs), - [Kernel,Stdlib,Sasl]; -check_base_libs(SomeMissing,Vsn) -> - find_missing(SomeMissing,[kernel,stdlib,sasl],Vsn). - -find_missing(SomeMissing,[H|T],Vsn) -> - case lists:keymember(H,1,SomeMissing) of - true -> - find_missing(SomeMissing,T,Vsn); - false -> - throw({error,{missing_base_app,Vsn,H}}) + case get_base_libs(ToRelease#release.libs) of + {ok,{Kernel,Stdlib,Sasl}=BaseLibs,_} -> + case get_base_libs(ToRelease#release.libs) of + {ok,_,RestLibs} -> + TmpErtsVsn = ToRelease#release.erts_vsn, + TmpLibs = [Kernel,Stdlib,Sasl|RestLibs], + TmpRelease = CurrentRelease#release{vsn=TmpVsn, + erts_vsn=TmpErtsVsn, + libs = TmpLibs, + status = unpacked}, + new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn, + BaseLibs,RelDir,Opts,Masters), + new_emulator_make_hybrid_config(CurrentVsn,ToVsn,TmpVsn, + RelDir,Masters), + {TmpVsn,TmpRelease}; + {error,{missing,Missing}} -> + throw({error,{missing_base_app,CurrentVsn,Missing}}) + end; + {error,{missing,Missing}} -> + throw({error,{missing_base_app,ToVsn,Missing}}) end. +%% Get kernel, stdlib and sasl libs, +%% and also return the rest of the libs as a list. +%% Return error if any of kernel, stdlib or sasl does not exist. +get_base_libs(Libs) -> + get_base_libs(Libs,undefined,undefined,undefined,[]). +get_base_libs([{kernel,_,_}=Kernel|Libs],undefined,Stdlib,Sasl,Rest) -> + get_base_libs(Libs,Kernel,Stdlib,Sasl,Rest); +get_base_libs([{stdlib,_,_}=Stdlib|Libs],Kernel,undefined,Sasl,Rest) -> + get_base_libs(Libs,Kernel,Stdlib,Sasl,Rest); +get_base_libs([{sasl,_,_}=Sasl|Libs],Kernel,Stdlib,undefined,Rest) -> + get_base_libs(Libs,Kernel,Stdlib,Sasl,Rest); +get_base_libs([Lib|Libs],Kernel,Stdlib,Sasl,Rest) -> + get_base_libs(Libs,Kernel,Stdlib,Sasl,[Lib|Rest]); +get_base_libs([],undefined,_Stdlib,_Sasl,_Rest) -> + {error,{missing,kernel}}; +get_base_libs([],_Kernel,undefined,_Sasl,_Rest) -> + {error,{missing,stdlib}}; +get_base_libs([],_Kernel,_Stdlib,undefined,_Rest) -> + {error,{missing,sasl}}; +get_base_libs([],Kernel,Stdlib,Sasl,Rest) -> + {ok,{Kernel,Stdlib,Sasl},lists:reverse(Rest)}. + new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn,BaseLibs,RelDir,Opts,Masters) -> FromBootFile = filename:join([RelDir,CurrentVsn,"start.boot"]), ToBootFile = filename:join([RelDir,ToVsn,"start.boot"]), @@ -1090,9 +1102,10 @@ new_emulator_make_hybrid_boot(CurrentVsn,ToVsn,TmpVsn,BaseLibs,RelDir,Opts,Maste Args = [ToVsn,Opts], {ok,FromBoot} = read_file(FromBootFile,Masters), {ok,ToBoot} = read_file(ToBootFile,Masters), - [KernelPath,SaslPath,StdlibPath] = - [filename:join(Path,ebin) || {_,_,Path} <- lists:keysort(1,BaseLibs)], - Paths = {KernelPath,StdlibPath,SaslPath}, + {{_,_,KernelPath},{_,_,SaslPath},{_,_,StdlibPath}} = BaseLibs, + Paths = {filename:join(KernelPath,"ebin"), + filename:join(StdlibPath,"ebin"), + filename:join(SaslPath,"ebin")}, case systools_make:make_hybrid_boot(TmpVsn,FromBoot,ToBoot,Paths,Args) of {ok,TmpBoot} -> write_file(TmpBootFile,TmpBoot,Masters); diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 0c44d3ae90..dda0c27d0c 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -90,12 +90,14 @@ renegotiation, % {boolean(), From | internal | peer} recv_during_renegotiation, %boolean() send_queue, % queue() - terminated = false % + terminated = false, % + allow_renegotiate = true }). -define(DEFAULT_DIFFIE_HELLMAN_PARAMS, - #'DHParameter'{prime = ?DEFAULT_DIFFIE_HELLMAN_PRIME, + #'DHParameter'{prime = ?DEFAULT_DIFFIE_HELLMAN_PRIME, base = ?DEFAULT_DIFFIE_HELLMAN_GENERATOR}). +-define(WAIT_TO_ALLOW_RENEGOTIATION, 12000). -type state_name() :: hello | abbreviated | certify | cipher | connection. -type gen_fsm_state_return() :: {next_state, state_name(), #state{}} | @@ -706,9 +708,25 @@ connection(#hello_request{}, #state{host = Host, port = Port, ConnectionStates1, tls_handshake_hashes = Hashes1}), next_state(hello, Record, State); -connection(#client_hello{} = Hello, #state{role = server} = State) -> - hello(Hello, State); - +connection(#client_hello{} = Hello, #state{role = server, allow_renegotiate = true} = State) -> + %% Mitigate Computational DoS attack http://www.educatedguesswork.org/2011/10/ssltls_and_computational_dos.html + %% http://www.thc.org/thc-ssl-dos/ Rather than disabling client initiated renegotiation + %% we will disallow many client initiated renegotiations immediately after each other. + erlang:send_after(?WAIT_TO_ALLOW_RENEGOTIATION, self(), allow_renegotiate), + hello(Hello, State#state{allow_renegotiate = false}); + +connection(#client_hello{}, #state{role = server, allow_renegotiate = false, + connection_states = ConnectionStates0, + socket = Socket, transport_cb = Transport, + negotiated_version = Version} = State0) -> + Alert = ?ALERT_REC(?WARNING, ?NO_RENEGOTIATION), + {BinMsg, ConnectionStates} = + encode_alert(Alert, Version, ConnectionStates0), + Transport:send(Socket, BinMsg), + {Record, State} = next_record(State0#state{connection_states = + ConnectionStates}), + next_state(connection, Record, State); + connection(timeout, State) -> {next_state, connection, State, hibernate}; @@ -983,6 +1001,9 @@ handle_info({'DOWN', MonitorRef, _, _, _}, _, State = #state{user_application={MonitorRef,_Pid}}) -> {stop, normal, State}; +handle_info(allow_renegotiate, StateName, State) -> + {next_state, StateName, State#state{allow_renegotiate = true}, get_timeout(State)}; + handle_info(Msg, StateName, State) -> Report = io_lib:format("SSL: Got unexpected info: ~p ~n", [Msg]), error_logger:info_report(Report), @@ -2298,7 +2319,7 @@ renegotiate(#state{role = server, {Record, State} = next_record(State0#state{connection_states = ConnectionStates, tls_handshake_hashes = Hs0}), - next_state(hello, Record, State). + next_state(hello, Record, State#state{allow_renegotiate = true}). notify_senders(SendQueue) -> lists:foreach(fun({From, _}) -> diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index d9cb8002ed..fc8aafa426 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -36,6 +36,7 @@ -define(LONG_TIMEOUT, 600000). -define(EXPIRE, 10). -define(SLEEP, 500). +-define(RENEGOTIATION_DISABLE_TIME, 12000). %% Test server callback functions %%-------------------------------------------------------------------- @@ -256,7 +257,7 @@ all() -> %%different_ca_peer_sign, no_reuses_session_server_restart_new_cert, no_reuses_session_server_restart_new_cert_file, reuseaddr, - hibernate, connect_twice + hibernate, connect_twice, renegotiate_dos_mitigate ]. groups() -> @@ -3647,7 +3648,40 @@ connect_twice(Config) when is_list(Config) -> ssl_test_lib:close(Client), ssl_test_lib:close(Client1). +%%-------------------------------------------------------------------- +renegotiate_dos_mitigate(doc) -> + ["Mitigate DOS computational attack by not allowing client to renegotiate many times in a row", + "immediately after each other"]; + +renegotiate_dos_mitigate(suite) -> + []; + +renegotiate_dos_mitigate(Config) when is_list(Config) -> + ServerOpts = ?config(server_opts, Config), + ClientOpts = ?config(client_opts, Config), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, send_recv_result_active, []}}, + {options, [ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, + renegotiate_immediately, []}}, + {options, ClientOpts}]), + + ssl_test_lib:check_result(Client, ok, Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- @@ -3690,6 +3724,19 @@ renegotiate_reuse_session(Socket, Data) -> test_server:sleep(?SLEEP), renegotiate(Socket, Data). +renegotiate_immediately(Socket) -> + receive + {ssl, Socket, "Hello world"} -> + ok + end, + ok = ssl:renegotiate(Socket), + {error, renegotiation_rejected} = ssl:renegotiate(Socket), + test_server:sleep(?RENEGOTIATION_DISABLE_TIME +1), + ok = ssl:renegotiate(Socket), + test_server:format("Renegotiated again"), + ssl:send(Socket, "Hello world"), + ok. + new_config(PrivDir, ServerOpts0) -> CaCertFile = proplists:get_value(cacertfile, ServerOpts0), CertFile = proplists:get_value(certfile, ServerOpts0), diff --git a/lib/stdlib/doc/src/Makefile b/lib/stdlib/doc/src/Makefile index 16e0a86e3b..6c92756ae7 100644 --- a/lib/stdlib/doc/src/Makefile +++ b/lib/stdlib/doc/src/Makefile @@ -83,7 +83,6 @@ XML_REF3_FILES = \ queue.xml \ random.xml \ re.xml \ - regexp.xml \ sets.xml \ shell.xml \ shell_default.xml \ diff --git a/lib/stdlib/doc/src/re.xml b/lib/stdlib/doc/src/re.xml index 18867cfb68..6d5336796c 100644 --- a/lib/stdlib/doc/src/re.xml +++ b/lib/stdlib/doc/src/re.xml @@ -41,8 +41,7 @@ strings and binaries.</p> <p>The regular expression syntax and semantics resemble that of - Perl. This library replaces the deprecated pure-Erlang regexp - library; it has a richer syntax, more options and is faster.</p> + Perl.</p> <p>The library's matching algorithms are currently based on the PCRE library, but not all of the PCRE library is interfaced and diff --git a/lib/stdlib/doc/src/ref_man.xml b/lib/stdlib/doc/src/ref_man.xml index 85aae6151d..6373922c92 100644 --- a/lib/stdlib/doc/src/ref_man.xml +++ b/lib/stdlib/doc/src/ref_man.xml @@ -80,7 +80,6 @@ <xi:include href="queue.xml"/> <xi:include href="random.xml"/> <xi:include href="re.xml"/> - <xi:include href="regexp.xml"/> <xi:include href="sets.xml"/> <xi:include href="shell.xml"/> <xi:include href="shell_default.xml"/> diff --git a/lib/stdlib/doc/src/regexp.xml b/lib/stdlib/doc/src/regexp.xml deleted file mode 100644 index 35d8e1c3f8..0000000000 --- a/lib/stdlib/doc/src/regexp.xml +++ /dev/null @@ -1,381 +0,0 @@ -<?xml version="1.0" encoding="latin1" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> - -<erlref> - <header> - <copyright> - <year>1996</year><year>2011</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - The contents of this file are subject to the Erlang Public License, - Version 1.1, (the "License"); you may not use this file except in - compliance with the License. You should have received a copy of the - Erlang Public License along with this software. If not, it can be - retrieved online at http://www.erlang.org/. - - Software distributed under the License is distributed on an "AS IS" - basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - the License for the specific language governing rights and limitations - under the License. - - </legalnotice> - - <title>regexp</title> - <prepared>Robert Virding</prepared> - <responsible>Bjarne Dacker</responsible> - <docno>1</docno> - <approved>Bjarne Däcker</approved> - <checked></checked> - <date>96-09-28</date> - <rev>A</rev> - <file>regexp.sgml</file> - </header> - <module>regexp</module> - <modulesummary>Regular Expression Functions for Strings</modulesummary> - <description> - <note><p>This module has been obsoleted by the - <seealso marker="re">re</seealso> module and will be removed in a future - release.</p></note> - <p>This module contains functions for regular expression - matching and substitution.</p> - </description> - <datatypes> - <datatype> - <name name="errordesc"></name> - </datatype> - <datatype> - <name name="regexp"></name> - <desc><p>Internal representation of a regular expression.</p></desc> - </datatype> - </datatypes> - <funcs> - <func> - <name name="match" arity="2"/> - <fsummary>Match a regular expression</fsummary> - <desc> - <p>Finds the first, longest match of the regular expression <c><anno>RegExp</anno></c> in <c><anno>String</anno></c>. This function searches for the longest possible match and returns the first one found if there are several expressions of the same length. It returns as follows:</p> - <taglist> - <tag><c>{match,<anno>Start</anno>,<anno>Length</anno>}</c></tag> - <item> - <p>if the match succeeded. <c><anno>Start</anno></c> is the starting - position of the match, and <c><anno>Length</anno></c> is the length of - the matching string.</p> - </item> - <tag><c>nomatch</c></tag> - <item> - <p>if there were no matching characters.</p> - </item> - <tag><c>{error,<anno>Error</anno>}</c></tag> - <item> - <p>if there was an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="first_match" arity="2"/> - <fsummary>Match a regular expression</fsummary> - <desc> - <p>Finds the first match of the regular expression <c><anno>RegExp</anno></c> in <c><anno>String</anno></c>. This call is - usually faster than <c>match</c> and it is also a useful way to ascertain that a match exists. It returns as follows:</p> - <taglist> - <tag><c>{match,<anno>Start</anno>,<anno>Length</anno>}</c></tag> - <item> - <p>if the match succeeded. <c><anno>Start</anno></c> is the starting - position of the match and <c><anno>Length</anno></c> is the length of - the matching string.</p> - </item> - <tag><c>nomatch</c></tag> - <item> - <p>if there were no matching characters.</p> - </item> - <tag><c>{error,<anno>Error</anno>}</c></tag> - <item> - <p>if there was an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="matches" arity="2"/> - <fsummary>Match a regular expression</fsummary> - <desc> - <p>Finds all non-overlapping matches of the - expression <c><anno>RegExp</anno></c> in <c><anno>String</anno></c>. - It returns as follows:</p> - <taglist> - <tag><c>{match, <anno>Matches</anno>}</c></tag> - <item> - <p>if the regular expression was correct. - The list will be empty if there was no match. Each element in the list looks like <c>{<anno>Start</anno>, <anno>Length</anno>}</c>, where <c><anno>Start</anno></c> is the starting position of the match, and <c><anno>Length</anno></c> is the length of the matching string.</p> - </item> - <tag><c>{error,<anno>Error</anno>}</c></tag> - <item> - <p>if there was an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="sub" arity="3"/> - <fsummary>Substitute the first occurrence of a regular expression</fsummary> - <desc> - <p>Substitutes the first occurrence of a substring matching <c><anno>RegExp</anno></c> in <c><anno>String</anno></c> with the string <c><anno>New</anno></c>. A <c><![CDATA[&]]></c> in the string <c><anno>New</anno></c> is replaced by the matched substring of <c><anno>String</anno></c>. <c><![CDATA[\&]]></c> puts a literal <c><![CDATA[&]]></c> into the replacement string. It returns as follows:</p> - <taglist> - <tag><c>{ok,<anno>NewString</anno>,<anno>RepCount</anno>}</c></tag> - <item> - <p>if <c><anno>RegExp</anno></c> is correct. <c><anno>RepCount</anno></c> is the number of replacements which have been made - (this will be either 0 or 1).</p> - </item> - <tag><c>{error, <anno>Error</anno>}</c></tag> - <item> - <p>if there is an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="gsub" arity="3"/> - <fsummary>Substitute all occurrences of a regular expression</fsummary> - <desc> - <p>The same as <c>sub</c>, except that all non-overlapping - occurrences of a substring matching - <c><anno>RegExp</anno></c> in <c><anno>String</anno></c> are replaced by the string <c><anno>New</anno></c>. It returns:</p> - <taglist> - <tag><c>{ok,<anno>NewString</anno>,<anno>RepCount</anno>}</c></tag> - <item> - <p>if <c><anno>RegExp</anno></c> is correct. <c><anno>RepCount</anno></c> is the number of replacements which have been made.</p> - </item> - <tag><c>{error, <anno>Error</anno>}</c></tag> - <item> - <p>if there is an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="split" arity="2"/> - <fsummary>Split a string into fields</fsummary> - <desc> - <p><c><anno>String</anno></c> is split into fields (sub-strings) by the - regular expression <c><anno>RegExp</anno></c>.</p> - <p>If the separator expression is <c>" "</c> (a single space), - then the fields are separated by blanks and/or tabs and - leading and trailing blanks and tabs are discarded. For all - other values of the separator, leading and trailing blanks - and tabs are not discarded. It returns:</p> - <taglist> - <tag><c>{ok, <anno>FieldList</anno>}</c></tag> - <item> - <p>to indicate that the string has been split up into the fields of - <c><anno>FieldList</anno></c>.</p> - </item> - <tag><c>{error, <anno>Error</anno>}</c></tag> - <item> - <p>if there is an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="sh_to_awk" arity="1"/> - <fsummary>Convert an <c>sh</c>regular expression into an <c>AWK</c>one</fsummary> - <desc> - <p>Converts the <c>sh</c> type regular expression - <c><anno>ShRegExp</anno></c> into a full <c>AWK</c> regular - expression. Returns the converted regular expression - string. <c>sh</c> expressions are used in the shell for - matching file names and have the following special - characters:</p> - <taglist> - <tag><c>*</c></tag> - <item> - <p>matches any string including the null string.</p> - </item> - <tag><c>?</c></tag> - <item> - <p>matches any single character.</p> - </item> - <tag><c>[...]</c></tag> - <item> - <p>matches any of the enclosed characters. Character - ranges are specified by a pair of characters separated - by a <c>-</c>. If the first character after <c>[</c> is a - <c>!</c>, then any character not enclosed is matched.</p> - </item> - </taglist> - <p>It may sometimes be more practical to use <c>sh</c> type - expansions as they are simpler and easier to use, even though they are not as powerful.</p> - </desc> - </func> - <func> - <name name="parse" arity="1"/> - <fsummary>Parse a regular expression</fsummary> - <desc> - <p>Parses the regular expression <c><anno>RegExp</anno></c> and builds the - internal representation used in the other regular expression - functions. Such representations can be used in all of the - other functions instead of a regular expression string. This - is more efficient when the same regular expression is used - in many strings. It returns:</p> - <taglist> - <tag><c>{ok, <anno>RE</anno>}</c></tag> - <item> - <p>if <c>RegExp</c> is correct and <c><anno>RE</anno></c> is the internal representation.</p> - </item> - <tag><c>{error, <anno>Error</anno>}</c></tag> - <item> - <p>if there is an error in <c><anno>RegExp</anno></c>.</p> - </item> - </taglist> - </desc> - </func> - <func> - <name name="format_error" arity="1"/> - <fsummary>Format an error descriptor</fsummary> - <desc> - <p>Returns a string which describes the error <c><anno>ErrorDescriptor</anno></c> - returned when there is an error in a regular expression.</p> - </desc> - </func> - </funcs> - - <section> - <title>Regular Expressions</title> - <p>The regular expressions allowed here is a subset of the set found - in <c>egrep</c> and in the <c>AWK</c> programming language, as - defined in the book, <c>The AWK Programming Language, by A. V. Aho, B. W. Kernighan, P. J. Weinberger</c>. They are - composed of the following characters:</p> - <taglist> - <tag>c</tag> - <item> - <p>matches the non-metacharacter <c>c</c>.</p> - </item> - <tag>\c</tag> - <item> - <p>matches the escape sequence or literal character <c>c</c>.</p> - </item> - <tag>.</tag> - <item> - <p>matches any character.</p> - </item> - <tag>^</tag> - <item> - <p>matches the beginning of a string.</p> - </item> - <tag>$</tag> - <item> - <p>matches the end of a string.</p> - </item> - <tag>[abc...]</tag> - <item> - <p>character class, which matches any of the characters - <c>abc...</c> Character ranges are specified by a pair of - characters separated by a <c>-</c>.</p> - </item> - <tag>[^abc...]</tag> - <item> - <p>negated character class, which matches any character except - <c>abc...</c>.</p> - </item> - <tag>r1 | r2</tag> - <item> - <p>alternation. It matches either <c>r1</c> or <c>r2</c>.</p> - </item> - <tag>r1r2</tag> - <item> - <p>concatenation. It matches <c>r1</c> and then <c>r2</c>.</p> - </item> - <tag>r+</tag> - <item> - <p>matches one or more <c>r</c>s.</p> - </item> - <tag>r*</tag> - <item> - <p>matches zero or more <c>r</c>s.</p> - </item> - <tag>r?</tag> - <item> - <p>matches zero or one <c>r</c>s.</p> - </item> - <tag>(r)</tag> - <item> - <p>grouping. It matches <c>r</c>.</p> - </item> - </taglist> - <p>The escape sequences allowed are the same as for Erlang - strings:</p> - <taglist> - <tag><c>\b</c></tag> - <item> - <p>backspace</p> - </item> - <tag><c>\f</c></tag> - <item> - <p>form feed </p> - </item> - <tag><c>\n</c></tag> - <item> - <p>newline (line feed) </p> - </item> - <tag><c>\r</c></tag> - <item> - <p>carriage return </p> - </item> - <tag><c>\t</c></tag> - <item> - <p>tab </p> - </item> - <tag><c>\e</c></tag> - <item> - <p>escape </p> - </item> - <tag><c>\v</c></tag> - <item> - <p>vertical tab </p> - </item> - <tag><c>\s</c></tag> - <item> - <p>space </p> - </item> - <tag><c>\d</c></tag> - <item> - <p>delete </p> - </item> - <tag><c>\ddd</c></tag> - <item> - <p>the octal value ddd </p> - </item> - <tag><c>\xhh</c></tag> - <item> - <p>The hexadecimal value <c>hh</c>.</p> - </item> - <tag><c>\x{h...}</c></tag> - <item> - <p>The hexadecimal value <c>h...</c>.</p> - </item> - <tag><c>\c</c></tag> - <item> - <p>any other character literally, for example <c>\\</c> for backslash, - <c>\"</c> for ")</p> - </item> - </taglist> - <p>To make these functions easier to use, in combination with the - function <c>io:get_line</c> which terminates the input line with - a new line, the <c>$</c> characters also matches a string ending - with <c>"...\n"</c>. The following examples - define Erlang data types:</p> - <pre> -Atoms [a-z][0-9a-zA-Z_]* - -Variables [A-Z_][0-9a-zA-Z_]* - -Floats (\+|-)?[0-9]+\.[0-9]+((E|e)(\+|-)?[0-9]+)?</pre> - <p>Regular expressions are written as Erlang strings when used with the functions in this module. This means that any <c>\</c> or <c>"</c> characters in a regular expression - string must be written with <c>\</c> as they are also escape characters for the string. For example, the regular expression string for Erlang floats is: - <c>"(\\+|-)?[0-9]+\\.[0-9]+((E|e)(\\+|-)?[0-9]+)?"</c>.</p> - <p>It is not really necessary to have the escape sequences as part of the regular expression syntax as they can always be generated directly in the string. They are included for completeness and can they can also be useful when generating regular expressions, or when they are entered other than with Erlang strings.</p> - </section> -</erlref> - diff --git a/lib/stdlib/doc/src/specs.xml b/lib/stdlib/doc/src/specs.xml index 98338b5ec2..49c60529d2 100644 --- a/lib/stdlib/doc/src/specs.xml +++ b/lib/stdlib/doc/src/specs.xml @@ -46,7 +46,6 @@ <xi:include href="../specs/specs_queue.xml"/> <xi:include href="../specs/specs_random.xml"/> <xi:include href="../specs/specs_re.xml"/> - <xi:include href="../specs/specs_regexp.xml"/> <xi:include href="../specs/specs_sets.xml"/> <xi:include href="../specs/specs_shell.xml"/> <xi:include href="../specs/specs_shell_default.xml"/> diff --git a/lib/stdlib/src/Makefile b/lib/stdlib/src/Makefile index 600303d7e1..9ce1f6f5c8 100644 --- a/lib/stdlib/src/Makefile +++ b/lib/stdlib/src/Makefile @@ -105,7 +105,6 @@ MODULES= \ qlc_pt \ queue \ random \ - regexp \ sets \ shell \ shell_default \ diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index c1285dab60..37246f82aa 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -56,19 +56,6 @@ obsolete_1(net, _, _) -> obsolete_1(erl_internal, builtins, 0) -> {deprecated, {erl_internal, bif, 2}}; -obsolete_1(string, re_sh_to_awk, 1) -> - {removed, {regexp, sh_to_awk, 1}, "R12B"}; -obsolete_1(string, re_parse, 1) -> - {removed, {regexp, parse, 1}, "R12B"}; -obsolete_1(string, re_match, 2) -> - {removed, {regexp, match, 2}, "R12B"}; -obsolete_1(string, re_sub, 3) -> - {removed, {regexp, sub, 3}, "R12B"}; -obsolete_1(string, re_gsub, 3) -> - {removed, {regexp, gsub, 3}, "R12B"}; -obsolete_1(string, re_split, 2) -> - {removed, {regexp, split, 2}, "R12B"}; - obsolete_1(string, index, 2) -> {removed, {string, str, 2}, "R12B"}; @@ -431,7 +418,7 @@ obsolete_1(ssh_sshd, stop, 1) -> %% Added in R13A. obsolete_1(regexp, _, _) -> - {deprecated, "the regexp module is deprecated (will be removed in R15A); use the re module instead"}; + {removed, "removed in R15; use the re module instead"}; obsolete_1(lists, flat_length, 1) -> {removed,{lists,flatlength,1},"R14"}; diff --git a/lib/stdlib/src/regexp.erl b/lib/stdlib/src/regexp.erl deleted file mode 100644 index 65f9ca247d..0000000000 --- a/lib/stdlib/src/regexp.erl +++ /dev/null @@ -1,557 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% --module(regexp). - -%% This entire module is deprecated and will be removed in a future -%% release. Use the 're' module instead. -%% -%% This module provides a basic set of regular expression functions -%% for strings. The functions provided are taken from AWK. -%% -%% Note that we interpret the syntax tree of a regular expression -%% directly instead of converting it to an NFA and then interpreting -%% that. This method seems to go significantly faster. - --export([sh_to_awk/1,parse/1,format_error/1,match/2,first_match/2,matches/2]). --export([sub/3,gsub/3,split/2]). - --deprecated([sh_to_awk/1,parse/1,format_error/1,match/2,first_match/2,matches/2]). --deprecated([sub/3,gsub/3,split/2]). - --import(string, [substr/2,substr/3]). --import(lists, [reverse/1]). - --type errordesc() :: term(). --opaque regexp() :: term(). - -%% -type matchres() = {match,Start,Length} | nomatch | {error,E}. -%% -type subres() = {ok,RepString,RepCount} | {error,E}. -%% -type splitres() = {ok,[SubString]} | {error,E}. - -%%-compile([export_all]). - -%% This is the regular expression grammar used. It is equivalent to the -%% one used in AWK, except that we allow ^ $ to be used anywhere and fail -%% in the matching. -%% -%% reg -> reg1 : '$1'. -%% reg1 -> reg1 "|" reg2 : {'or','$1','$2'}. -%% reg1 -> reg2 : '$1'. -%% reg2 -> reg2 reg3 : {concat,'$1','$2'}. -%% reg2 -> reg3 : '$1'. -%% reg3 -> reg3 "*" : {kclosure,'$1'}. -%% reg3 -> reg3 "+" : {pclosure,'$1'}. -%% reg3 -> reg3 "?" : {optional,'$1'}. -%% reg3 -> reg4 : '$1'. -%% reg4 -> "(" reg ")" : '$2'. -%% reg4 -> "\\" char : '$2'. -%% reg4 -> "^" : bos. -%% reg4 -> "$" : eos. -%% reg4 -> "." : char. -%% reg4 -> "[" class "]" : {char_class,char_class('$2')} -%% reg4 -> "[" "^" class "]" : {comp_class,char_class('$3')} -%% reg4 -> "\"" chars "\"" : char_string('$2') -%% reg4 -> char : '$1'. -%% reg4 -> empty : epsilon. -%% The grammar of the current regular expressions. The actual parser -%% is a recursive descent implementation of the grammar. - -reg(S) -> reg1(S). - -%% reg1 -> reg2 reg1' -%% reg1' -> "|" reg2 -%% reg1' -> empty - -reg1(S0) -> - {L,S1} = reg2(S0), - reg1p(S1, L). - -reg1p([$||S0], L) -> - {R,S1} = reg2(S0), - reg1p(S1, {'or',L,R}); -reg1p(S, L) -> {L,S}. - -%% reg2 -> reg3 reg2' -%% reg2' -> reg3 -%% reg2' -> empty - -reg2(S0) -> - {L,S1} = reg3(S0), - reg2p(S1, L). - -reg2p([C|S0], L) when C =/= $|, C =/= $) -> - {R,S1} = reg3([C|S0]), - reg2p(S1, {concat,L,R}); -reg2p(S, L) -> {L,S}. - -%% reg3 -> reg4 reg3' -%% reg3' -> "*" reg3' -%% reg3' -> "+" reg3' -%% reg3' -> "?" reg3' -%% reg3' -> empty - -reg3(S0) -> - {L,S1} = reg4(S0), - reg3p(S1, L). - -reg3p([$*|S], L) -> reg3p(S, {kclosure,L}); -reg3p([$+|S], L) -> reg3p(S, {pclosure,L}); -reg3p([$?|S], L) -> reg3p(S, {optional,L}); -reg3p(S, L) -> {L,S}. - --define(HEX(C), C >= $0 andalso C =< $9 orelse - C >= $A andalso C =< $F orelse - C >= $a andalso C =< $f). - -reg4([$(|S0]) -> - case reg(S0) of - {R,[$)|S1]} -> {R,S1}; - {_R,_S} -> throw({error,{unterminated,"("}}) - end; -reg4([$\\,O1,O2,O3|S]) when - O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> - {(O1*8 + O2)*8 + O3 - 73*$0,S}; -reg4([$\\,$x,H1,H2|S]) when ?HEX(H1), ?HEX(H2) -> - {erlang:list_to_integer([H1,H2], 16),S}; -reg4([$\\,$x,${|S]) -> - hex(S, []); -reg4([$\\,$x|_]) -> - throw({error,{illegal,[$x]}}); -reg4([$\\,C|S]) -> {escape_char(C),S}; -reg4([$\\]) -> throw({error,{unterminated,"\\"}}); -reg4([$^|S]) -> {bos,S}; -reg4([$$|S]) -> {eos,S}; -reg4([$.|S]) -> {{comp_class,"\n"},S}; -reg4("[^" ++ S0) -> - case char_class(S0) of - {Cc,[$]|S1]} -> {{comp_class,Cc},S1}; - {_Cc,_S} -> throw({error,{unterminated,"["}}) - end; -reg4([$[|S0]) -> - case char_class(S0) of - {Cc,[$]|S1]} -> {{char_class,Cc},S1}; - {_Cc,_S1} -> throw({error,{unterminated,"["}}) - end; -%reg4([$"|S0]) -> -% case char_string(S0) of -% {St,[$"|S1]} -> {St,S1}; -% {St,S1} -> throw({error,{unterminated,"\""}}) -% end; -reg4([C|S]) when C =/= $*, C =/= $+, C =/= $?, C =/= $] -> {C,S}; -reg4([C|_S]) -> throw({error,{illegal,[C]}}); -reg4([]) -> {epsilon,[]}. - -hex([C|Cs], L) when ?HEX(C) -> - hex(Cs, [C|L]); -hex([$}|S], L) -> - case catch erlang:list_to_integer(lists:reverse(L), 16) of - V when V =< 16#FF -> - {V,S}; - _ -> - throw({error,{illegal,[$}]}}) - end; -hex(_S, _) -> - throw({error,{unterminated,"\\x{"}}). - -escape_char($n) -> $\n; %\n = LF -escape_char($r) -> $\r; %\r = CR -escape_char($t) -> $\t; %\t = TAB -escape_char($v) -> $\v; %\v = VT -escape_char($b) -> $\b; %\b = BS -escape_char($f) -> $\f; %\f = FF -escape_char($e) -> $\e; %\e = ESC -escape_char($s) -> $\s; %\s = SPACE -escape_char($d) -> $\d; %\d = DEL -escape_char(C) -> C. - -char_class([$]|S]) -> char_class(S, [$]]); -char_class(S) -> char_class(S, []). - -char($\\, [O1,O2,O3|S]) when - O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> - {(O1*8 + O2)*8 + O3 - 73*$0,S}; -char($\\, [$x,H1,H2|S]) when ?HEX(H1), ?HEX(H2) -> - {erlang:list_to_integer([H1,H2], 16),S}; -char($\\,[$x,${|S]) -> - hex(S, []); -char($\\,[$x|_]) -> - throw({error,{illegal,[$x]}}); -char($\\, [C|S]) -> {escape_char(C),S}; -char(C, S) -> {C,S}. - -char_class([C1|S0], Cc) when C1 =/= $] -> - case char(C1, S0) of - {Cf,[$-,C2|S1]} when C2 =/= $] -> - case char(C2, S1) of - {Cl,S2} when Cf < Cl -> char_class(S2, [{Cf,Cl}|Cc]); - {Cl,_S2} -> throw({error,{char_class,[Cf,$-,Cl]}}) - end; - {C,S1} -> char_class(S1, [C|Cc]) - end; -char_class(S, Cc) -> {Cc,S}. - -%char_string([C|S]) when C =/= $" -> char_string(S, C); -%char_string(S) -> {epsilon,S}. - -%char_string([C|S0], L) when C =/= $" -> -% char_string(S0, {concat,L,C}); -%char_string(S, L) -> {L,S}. - -%% -deftype re_app_res() = {match,RestPos,Rest} | nomatch. - -%% re_apply(String, StartPos, RegExp) -> re_app_res(). -%% -%% Apply the (parse of the) regular expression RegExp to String. If -%% there is a match return the position of the remaining string and -%% the string if else return 'nomatch'. BestMatch specifies if we want -%% the longest match, or just a match. -%% -%% StartPos should be the real start position as it is used to decide -%% if we ae at the beginning of the string. -%% -%% Pass two functions to re_apply_or so it can decide, on the basis -%% of BestMatch, whether to just any take any match or try both to -%% find the longest. This is slower but saves duplicatng code. - -re_apply(S, St, RE) -> re_apply(RE, [], S, St). - -re_apply(epsilon, More, S, P) -> %This always matches - re_apply_more(More, S, P); -re_apply({'or',RE1,RE2}, More, S, P) -> - re_apply_or(re_apply(RE1, More, S, P), - re_apply(RE2, More, S, P)); -re_apply({concat,RE1,RE2}, More, S0, P) -> - re_apply(RE1, [RE2|More], S0, P); -re_apply({kclosure,CE}, More, S, P) -> - %% Be careful with the recursion, explicitly do one call before - %% looping. - re_apply_or(re_apply_more(More, S, P), - re_apply(CE, [{kclosure,CE}|More], S, P)); -re_apply({pclosure,CE}, More, S, P) -> - re_apply(CE, [{kclosure,CE}|More], S, P); -re_apply({optional,CE}, More, S, P) -> - re_apply_or(re_apply_more(More, S, P), - re_apply(CE, More, S, P)); -re_apply(bos, More, S, 1) -> re_apply_more(More, S, 1); -re_apply(eos, More, [$\n|S], P) -> re_apply_more(More, S, P); -re_apply(eos, More, [], P) -> re_apply_more(More, [], P); -re_apply({char_class,Cc}, More, [C|S], P) -> - case in_char_class(C, Cc) of - true -> re_apply_more(More, S, P+1); - false -> nomatch - end; -re_apply({comp_class,Cc}, More, [C|S], P) -> - case in_char_class(C, Cc) of - true -> nomatch; - false -> re_apply_more(More, S, P+1) - end; -re_apply(C, More, [C|S], P) when is_integer(C) -> - re_apply_more(More, S, P+1); -re_apply(_RE, _More, _S, _P) -> nomatch. - -%% re_apply_more([RegExp], String, Length) -> re_app_res(). - -re_apply_more([RE|More], S, P) -> re_apply(RE, More, S, P); -re_apply_more([], S, P) -> {match,P,S}. - -%% in_char_class(Char, Class) -> bool(). - -in_char_class(C, [{C1,C2}|_Cc]) when C >= C1, C =< C2 -> true; -in_char_class(C, [C|_Cc]) -> true; -in_char_class(C, [_|Cc]) -> in_char_class(C, Cc); -in_char_class(_C, []) -> false. - -%% re_apply_or(Match1, Match2) -> re_app_res(). -%% If we want the best match then choose the longest match, else just -%% choose one by trying sequentially. - -re_apply_or({match,P1,S1}, {match,P2,_S2}) when P1 >= P2 -> {match,P1,S1}; -re_apply_or({match,_P1,_S1}, {match,P2,S2}) -> {match,P2,S2}; -re_apply_or(nomatch, R2) -> R2; -re_apply_or(R1, nomatch) -> R1. - -%% sh_to_awk(ShellRegExp) -%% Convert a sh style regexp into a full AWK one. The main difficulty is -%% getting character sets right as the conventions are different. - --spec sh_to_awk(ShRegExp) -> AwkRegExp when - ShRegExp :: string(), - AwkRegExp :: string(). - -sh_to_awk(Sh) -> "^(" ++ sh_to_awk_1(Sh). %Fix the beginning - -sh_to_awk_1([$*|Sh]) -> %This matches any string - ".*" ++ sh_to_awk_1(Sh); -sh_to_awk_1([$?|Sh]) -> %This matches any character - [$.|sh_to_awk_1(Sh)]; -sh_to_awk_1([$[,$^,$]|Sh]) -> %This takes careful handling - "\\^" ++ sh_to_awk_1(Sh); -sh_to_awk_1("[^" ++ Sh) -> [$[|sh_to_awk_2(Sh, true)]; -sh_to_awk_1("[!" ++ Sh) -> "[^" ++ sh_to_awk_2(Sh, false); -sh_to_awk_1([$[|Sh]) -> [$[|sh_to_awk_2(Sh, false)]; -sh_to_awk_1([C|Sh]) -> - %% Unspecialise everything else which is not an escape character. - case special_char(C) of - true -> [$\\,C|sh_to_awk_1(Sh)]; - false -> [C|sh_to_awk_1(Sh)] - end; -sh_to_awk_1([]) -> ")$". %Fix the end - -sh_to_awk_2([$]|Sh], UpArrow) -> [$]|sh_to_awk_3(Sh, UpArrow)]; -sh_to_awk_2(Sh, UpArrow) -> sh_to_awk_3(Sh, UpArrow). - -sh_to_awk_3([$]|Sh], true) -> "^]" ++ sh_to_awk_1(Sh); -sh_to_awk_3([$]|Sh], false) -> [$]|sh_to_awk_1(Sh)]; -sh_to_awk_3([C|Sh], UpArrow) -> [C|sh_to_awk_3(Sh, UpArrow)]; -sh_to_awk_3([], true) -> [$^|sh_to_awk_1([])]; -sh_to_awk_3([], false) -> sh_to_awk_1([]). - -%% -type special_char(char()) -> bool(). -%% Test if a character is a special character. - -special_char($|) -> true; -special_char($*) -> true; -special_char($+) -> true; -special_char($?) -> true; -special_char($() -> true; -special_char($)) -> true; -special_char($\\) -> true; -special_char($^) -> true; -special_char($$) -> true; -special_char($.) -> true; -special_char($[) -> true; -special_char($]) -> true; -special_char($") -> true; -special_char(_C) -> false. - -%% parse(RegExp) -> {ok,RE} | {error,E}. -%% Parse the regexp described in the string RegExp. - --spec parse(RegExp) -> ParseRes when - RegExp :: string(), - ParseRes :: {ok, RE} | {error, Error}, - RE :: regexp(), - Error :: errordesc(). - -parse(S) -> - case catch reg(S) of - {R,[]} -> {ok,R}; - {_R,[C|_]} -> {error,{illegal,[C]}}; - {error,E} -> {error,E} - end. - -%% format_error(Error) -> String. - --spec format_error(ErrorDescriptor) -> Chars when - ErrorDescriptor :: errordesc(), - Chars :: io_lib:chars(). - -format_error({illegal,What}) -> ["illegal character `",What,"'"]; -format_error({unterminated,What}) -> ["unterminated `",What,"'"]; -format_error({char_class,What}) -> - ["illegal character class ",io_lib:write_string(What)]. - -%% -type match(String, RegExp) -> matchres(). -%% Find the longest match of RegExp in String. - --spec match(String, RegExp) -> MatchRes when - String :: string(), - RegExp :: string() | regexp(), - MatchRes :: {match, Start, Length} | nomatch | {error, Error}, - Start :: pos_integer(), - Length :: pos_integer(), - Error :: errordesc(). - -match(S, RegExp) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> match(S, RE); - {error,E} -> {error,E} - end; -match(S, RE) -> - case match(RE, S, 1, 0, -1) of - {Start,Len} when Len >= 0 -> - {match,Start,Len}; - {_Start,_Len} -> nomatch - end. - -match(RE, S, St, Pos, L) -> - case first_match(RE, S, St) of - {St1,L1} -> - Nst = St1 + 1, - if L1 > L -> match(RE, lists:nthtail(Nst-St, S), Nst, St1, L1); - true -> match(RE, lists:nthtail(Nst-St, S), Nst, Pos, L) - end; - nomatch -> {Pos,L} - end. - -%% -type first_match(String, RegExp) -> matchres(). -%% Find the first match of RegExp in String. - --spec first_match(String, RegExp) -> MatchRes when - String :: string(), - RegExp :: string() | regexp(), - MatchRes :: {match, Start, Length} | nomatch | {error, Error}, - Start :: pos_integer(), - Length :: pos_integer(), - Error :: errordesc(). - -first_match(S, RegExp) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> first_match(S, RE); - {error,E} -> {error,E} - end; -first_match(S, RE) -> - case first_match(RE, S, 1) of - {Start,Len} when Len >= 0 -> - {match,Start,Len}; - nomatch -> nomatch - end. - -first_match(RE, S, St) when S =/= [] -> - case re_apply(S, St, RE) of - {match,P,_Rest} -> {St,P-St}; - nomatch -> first_match(RE, tl(S), St+1) - end; -first_match(_RE, [], _St) -> nomatch. - -%% -type matches(String, RegExp) -> {match,[{Start,Length}]} | {error,E}. -%% Return the all the non-overlapping matches of RegExp in String. - --spec matches(String, RegExp) -> MatchRes when - String :: string(), - RegExp :: string() | regexp(), - MatchRes :: {match, Matches} | {error, Error}, - Matches :: [{Start, Length}], - Start :: pos_integer(), - Length :: pos_integer(), - Error :: errordesc(). - -matches(S, RegExp) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> matches(S, RE); - {error,E} -> {error,E} - end; -matches(S, RE) -> - {match,matches(S, RE, 1)}. - -matches(S, RE, St) -> - case first_match(RE, S, St) of - {St1,0} -> [{St1,0}|matches(substr(S, St1+2-St), RE, St1+1)]; - {St1,L1} -> [{St1,L1}|matches(substr(S, St1+L1+1-St), RE, St1+L1)]; - nomatch -> [] - end. - -%% -type sub(String, RegExp, Replace) -> subsres(). -%% Substitute the first match of the regular expression RegExp with -%% the string Replace in String. Accept pre-parsed regular -%% expressions. - --spec sub(String, RegExp, New) -> SubRes when - String :: string(), - RegExp :: string() | regexp(), - New :: string(), - NewString :: string(), - SubRes :: {ok, NewString, RepCount} | {error, Error}, - RepCount :: 0 | 1, - Error :: errordesc(). - -sub(String, RegExp, Rep) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> sub(String, RE, Rep); - {error,E} -> {error,E} - end; -sub(String, RE, Rep) -> - Ss = sub_match(String, RE, 1), - {ok,sub_repl(Ss, Rep, String, 1),length(Ss)}. - -sub_match(S, RE, St) -> - case first_match(RE, S, St) of - {St1,L1} -> [{St1,L1}]; - nomatch -> [] - end. - -sub_repl([{St,L}|Ss], Rep, S, Pos) -> - Rs = sub_repl(Ss, Rep, S, St+L), - substr(S, Pos, St-Pos) ++ sub_repl(Rep, substr(S, St, L), Rs); -sub_repl([], _Rep, S, Pos) -> substr(S, Pos). - -sub_repl([$&|Rep], M, Rest) -> M ++ sub_repl(Rep, M, Rest); -sub_repl("\\&" ++ Rep, M, Rest) -> [$&|sub_repl(Rep, M, Rest)]; -sub_repl([C|Rep], M, Rest) -> [C|sub_repl(Rep, M, Rest)]; -sub_repl([], _M, Rest) -> Rest. - -%% -type gsub(String, RegExp, Replace) -> subres(). -%% Substitute every match of the regular expression RegExp with the -%% string New in String. Accept pre-parsed regular expressions. - --spec gsub(String, RegExp, New) -> SubRes when - String :: string(), - RegExp :: string() | regexp(), - New :: string(), - NewString :: string(), - SubRes :: {ok, NewString, RepCount} | {error, Error}, - RepCount :: non_neg_integer(), - Error :: errordesc(). - -gsub(String, RegExp, Rep) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> gsub(String, RE, Rep); - {error,E} -> {error,E} - end; -gsub(String, RE, Rep) -> - Ss = matches(String, RE, 1), - {ok,sub_repl(Ss, Rep, String, 1),length(Ss)}. - -%% -type split(String, RegExp) -> splitres(). -%% Split a string into substrings where the RegExp describes the -%% field seperator. The RegExp " " is specially treated. - --spec split(String, RegExp) -> SplitRes when - String :: string(), - RegExp :: string() | regexp(), - SplitRes :: {ok, FieldList} | {error, Error}, - FieldList :: [string()], - Error :: errordesc(). - -split(String, " ") -> %This is really special - {ok,RE} = parse("[ \t]+"), - case split_apply(String, RE, true) of - [[]|Ss] -> {ok,Ss}; - Ss -> {ok,Ss} - end; -split(String, RegExp) when is_list(RegExp) -> - case parse(RegExp) of - {ok,RE} -> {ok,split_apply(String, RE, false)}; - {error,E} -> {error,E} - end; -split(String, RE) -> {ok,split_apply(String, RE, false)}. - -split_apply(S, RE, Trim) -> split_apply(S, 1, RE, Trim, []). - -split_apply([], _P, _RE, true, []) -> []; -split_apply([], _P, _RE, _T, Sub) -> [reverse(Sub)]; -split_apply(S, P, RE, T, Sub) -> - case re_apply(S, P, RE) of - {match,P,_Rest} -> - split_apply(tl(S), P+1, RE, T, [hd(S)|Sub]); - {match,P1,Rest} -> - [reverse(Sub)|split_apply(Rest, P1, RE, T, [])]; - nomatch -> - split_apply(tl(S), P+1, RE, T, [hd(S)|Sub]) - end. diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index 9d15f01683..da65db4b9d 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -85,7 +85,6 @@ queue, random, re, - regexp, sets, shell, shell_default, diff --git a/system/doc/efficiency_guide/commoncaveats.xml b/system/doc/efficiency_guide/commoncaveats.xml index 61d13636c0..7f2d51e609 100644 --- a/system/doc/efficiency_guide/commoncaveats.xml +++ b/system/doc/efficiency_guide/commoncaveats.xml @@ -33,20 +33,6 @@ from a performance point of view.</p> <section> - <title>The regexp module</title> - - <p>The regular expression functions in the - <seealso marker="stdlib:regexp">regexp</seealso> - module are written in Erlang, not in C, and were - meant for occasional use on small amounts of data, - for instance for validation of configuration files - when starting an application.</p> - - <p>Use the <seealso marker="stdlib:re">re</seealso> module - (introduced in R13A) instead, especially in time-critical code.</p> - </section> - - <section> <title>The timer module</title> <p>Creating timers using <seealso |