diff options
Diffstat (limited to 'lib/stdlib')
25 files changed, 603 insertions, 216 deletions
diff --git a/lib/stdlib/doc/src/erl_tar.xml b/lib/stdlib/doc/src/erl_tar.xml index 7f25f5b7bc..95eefb8f9b 100644 --- a/lib/stdlib/doc/src/erl_tar.xml +++ b/lib/stdlib/doc/src/erl_tar.xml @@ -80,6 +80,12 @@ </section> <section> + <title>OTHER STORAGE MEDIA</title> + <p>The <c>erl_ftp</c> module normally accesses the tar-file on disk using the <seealso marker="kernel:file">file module</seealso>. When other needs arise, there is a way to define your own low-level Erlang functions to perform the writing and reading on the storage media. See <seealso marker="#init/3">init/3</seealso> for usage.</p> + <p>An example of this is the sftp support in <seealso marker="ssh:ssh_sftp#open_tar/3">ssh_sftp:open_tar/3</seealso>. That function opens a tar file on a remote machine using an sftp channel.</p> + </section> + + <section> <title>LIMITATIONS</title> <p>For maximum compatibility, it is safe to archive files with names up to 100 characters in length. Such tar files can generally be @@ -99,7 +105,8 @@ <v>TarDescriptor = term()</v> <v>Filename = filename()</v> <v>Options = [Option]</v> - <v>Option = dereference|verbose</v> + <v>Option = dereference|verbose|{chunks,ChunkSize}</v> + <v>ChunkSize = positive_integer()</v> <v>RetValue = ok|{error,{Filename,Reason}}</v> <v>Reason = term()</v> </type> @@ -119,6 +126,12 @@ <item> <p>Print an informational message about the file being added.</p> </item> + <tag><c>{chunks,ChunkSize}</c></tag> + <item> + <p>Read data in parts from the file. This is intended for memory-limited + machines that for example builds a tar file on a remote machine over + <seealso marker="ssh:ssh_sftp#open_tar/3">sftp</seealso>.</p> + </item> </taglist> </desc> </func> @@ -389,6 +402,101 @@ </warning> </desc> </func> + + <func> + <name>init(UserPrivate, AccessMode, Fun) -> {ok,TarDescriptor} | {error,Reason} +</name> + <fsummary>Creates a TarDescriptor used in subsequent tar operations when + defining own low-level storage access functions + </fsummary> + <type> + <v>UserPrivate = term()</v> + <v>AccessMode = [write] | [read]</v> + <v>Fun when AccessMode is [write] = fun(write, {UserPrivate,DataToWrite})->...; + (position,{UserPrivate,Position})->...; + (close, UserPrivate)->... + end + </v> + <v>Fun when AccessMode is [read] = fun(read2, {UserPrivate,Size})->...; + (position,{UserPrivate,Position})->...; + (close, UserPrivate)->... + end + </v> + <v>TarDescriptor = term()</v> + <v>Reason = term()</v> + </type> + <desc> + <p>The <c>Fun</c> is the definition of what to do when the different + storage operations functions are to be called from the higher tar + handling functions (<c>add/3</c>, <c>add/4</c>, <c>close/1</c>...). + </p> + <p>The <c>Fun</c> will be called when the tar function wants to do + a low-level operation, like writing a block to a file. The Fun is called + as <c>Fun(Op,{UserPrivate,Parameters...})</c> where <c>Op</c> is the operation name, + <c>UserPrivate</c> is the term passed as the first argument to <c>init/1</c> and + <c>Parameters...</c> are the data added by the tar function to be passed down to + the storage handling function. + </p> + <p>The parameter <c>UserPrivate</c> is typically the result of opening a low level + structure like a file descriptor, a sftp channel id or such. The different <c>Fun</c> + clauses operates on that very term. + </p> + <p>The fun clauses parameter lists are: + <taglist> + <tag><c>(write, {UserPrivate,DataToWrite})</c></tag> + <item>Write the term <c>DataToWrite</c> using <c>UserPrivate</c></item> + <tag><c>(close, UserPrivate)</c></tag> + <item>Close the access.</item> + <tag><c>(read2, {UserPrivate,Size})</c></tag> + <item>Read using <c>UserPrivate</c> but only <c>Size</c> bytes. Note that there is + only an arity-2 read function, not an arity-1 + </item> + <tag><c> (position,{UserPrivate,Position})</c></tag> + <item>Sets the position of <c>UserPrivate</c> as defined for files in <seealso marker="kernel:file#position-2">file:position/2</seealso></item> + <tag><c></c></tag> + <item></item> + </taglist> + </p> + <p>A complete <c>Fun</c> parameter for reading and writing on files using the + <seealso marker="kernel:file">file module</seealso> could be: + </p> + <code type="none"> + ExampleFun = + fun(write, {Fd,Data}) -> file:write(Fd, Data); + (position, {Fd,Pos}) -> file:position(Fd, Pos); + (read2, {Fd,Size}) -> file:read(Fd,Size); + (close, Fd) -> file:close(Fd) + end + </code> + <p>where <c>Fd</c> was given to the <c>init/3</c> function as:</p> + <code> + {ok,Fd} = file:open(Name,...). + {ok,TarDesc} = erl_tar:init(Fd, [write], ExampleFun), + </code> + <p>The <c>TarDesc</c> is then used:</p> + <code> + erl_tar:add(TarDesc, SomeValueIwantToAdd, FileNameInTarFile), + ...., + erl_tar:close(TarDesc) + </code> + <p>When the erl_tar core wants to e.g. write a piece of Data, it would call + <c>ExampleFun(write,{UserPrivate,Data})</c>. + </p> + <note> + <p>The example above with <c>file</c> module operations is not necessary to + use directly since that is what the <seealso marker="#open">open</seealso> function + in principle does. + </p> + </note> + <warning> + <p>The <c>TarDescriptor</c> term is not a file descriptor. + You should not rely on the specific contents of the <c>TarDescriptor</c> + term, as it may change in future versions as more features are added + to the <c>erl_tar</c> module.</p> + </warning> + </desc> + </func> + <func> <name>table(Name) -> RetValue</name> <fsummary>Retrieve the name of all files in a tar file</fsummary> diff --git a/lib/stdlib/doc/src/io_protocol.xml b/lib/stdlib/doc/src/io_protocol.xml index 9328704e11..21da404c35 100644 --- a/lib/stdlib/doc/src/io_protocol.xml +++ b/lib/stdlib/doc/src/io_protocol.xml @@ -49,7 +49,7 @@ current I/O-protocol.</p> <p>The original I/O-protocol was simple and flexible. Demands for spacial and execution time efficiency has triggered extensions to the protocol over the years, making the protocol larger and somewhat less easy to -implement than the original. It can certainly be argumented that the +implement than the original. It can certainly be argued that the current protocol is too complex, but this text describes how it looks today, not how it should have looked.</p> @@ -76,10 +76,11 @@ the server eventually sends a corresponding <c>io_reply</c> tuple.</p> the I/O server sends the IO reply to.</item> <item><c>ReplyAs</c> can be any datum and is returned in the corresponding -<c>io_reply</c>. The <seealso marker="stdlib:io">io</seealso> module simply uses the pid() -of the I/O server as the <c>ReplyAs</c> datum, but a more complicated client +<c>io_reply</c>. The <seealso marker="stdlib:io">io</seealso> module monitors +the I/O server, and uses the monitor reference as the <c>ReplyAs</c> datum. +A more complicated client could have several outstanding I/O requests to the same I/O server and -would then use i.e. a <c>reference()</c> or something else to differentiate among +would then use different references (or something else) to differentiate among the incoming IO replies. The <c>ReplyAs</c> element should be considered opaque by the I/O server. Note that the <c>pid()</c> of the I/O server is not explicitly present in the <c>io_reply</c> tuple. The reply can be sent from any diff --git a/lib/stdlib/doc/src/maps.xml b/lib/stdlib/doc/src/maps.xml index 64229fa8d3..f766c843be 100644 --- a/lib/stdlib/doc/src/maps.xml +++ b/lib/stdlib/doc/src/maps.xml @@ -330,7 +330,7 @@ false</code> <code type="none"> > Map = #{42 => value_three,1337 => "value two","a" => 1}, Ks = ["a",42,"other key"], - maps:without(Ks,Map). + maps:with(Ks,Map). #{42 => value_three,"a" => 1}</code> </desc> </func> diff --git a/lib/stdlib/doc/src/unicode_usage.xml b/lib/stdlib/doc/src/unicode_usage.xml index bebfbd4514..29b8940c62 100644 --- a/lib/stdlib/doc/src/unicode_usage.xml +++ b/lib/stdlib/doc/src/unicode_usage.xml @@ -50,12 +50,8 @@ encoded files in several circumstances. Most notable is the support for UTF-8 in files read by <c>file:consult/1</c>, release handler support for UTF-8 and more support for Unicode character sets in the - I/O-system.</p> - - <p>In Erlang/OTP 17.0, the encoding default for Erlang source files was - switched to UTF-8 and in Erlang/OTP 18.0 Erlang will support atoms in the full - Unicode range, meaning full Unicode function and module - names</p> + I/O-system. In Erlang/OTP 17.0, the encoding default for Erlang source files was + switched to UTF-8.</p> <p>This guide outlines the current Unicode support and gives a couple of recipes for working with Unicode data.</p> @@ -289,8 +285,8 @@ <tag>The language</tag> <item>Having the source code in UTF-8 also allows you to write string literals containing Unicode characters with code points > - 255, although atoms, module names and function names will be - restricted to the ISO-Latin-1 range until the Erlang/OTP 18.0 release. Binary + 255, although atoms, module names and function names are + restricted to the ISO-Latin-1 range. Binary literals where you use the <c>/utf8</c> type, can also be expressed using Unicode characters > 255. Having module names using characters other than 7-bit ASCII can cause trouble on @@ -385,8 +381,7 @@ external_charlist() = maybe_improper_list(char() | using characters from the ISO-latin-1 character set and atoms are restricted to the same ISO-latin-1 range. These restrictions in the language are of course independent of the encoding of the source - file. Erlang/OTP 18.0 is expected to handle functions named in - Unicode as well as Unicode atoms.</p> + file.</p> <section> <title>Bit-syntax</title> <p>The bit-syntax contains types for coping with binary data in the diff --git a/lib/stdlib/src/dets.erl b/lib/stdlib/src/dets.erl index 76e03bbfaa..a4bd45ea19 100644 --- a/lib/stdlib/src/dets.erl +++ b/lib/stdlib/src/dets.erl @@ -2839,17 +2839,22 @@ fsck_try(Fd, Tab, FH, Fname, SlotNumbers, Version) -> tempfile(Fname) -> Tmp = lists:concat([Fname, ".TMP"]), - tempfile(Tmp, 10). - -tempfile(Tmp, 0) -> - Tmp; -tempfile(Tmp, N) -> case file:delete(Tmp) of - {error, eacces} -> % 'dets_process_died' happened anyway... (W-nd-ws) - timer:sleep(1000), - tempfile(Tmp, N-1); - _ -> - Tmp + {error, _Reason} -> % typically enoent + ok; + ok -> + assure_no_file(Tmp) + end, + Tmp. + +assure_no_file(File) -> + case file:read_file_info(File) of + {ok, _FileInfo} -> + %% Wait for some other process to close the file: + timer:sleep(100), + assure_no_file(File); + {error, _} -> + ok end. %% -> {ok, NewHead} | {try_again, integer()} | Error diff --git a/lib/stdlib/src/dets_server.erl b/lib/stdlib/src/dets_server.erl index 268c201047..3164d40f35 100644 --- a/lib/stdlib/src/dets_server.erl +++ b/lib/stdlib/src/dets_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2013. All Rights Reserved. +%% Copyright Ericsson AB 2001-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -171,9 +171,15 @@ handle_info({pending_reply, {Ref, Result0}}, State) -> link(Pid), do_link(Store, FromPid), true = ets:insert(Store, {FromPid, Tab}), - true = ets:insert(?REGISTRY, {Tab, 1, Pid}), - true = ets:insert(?OWNERS, {Pid, Tab}), + %% do_internal_open() has already done the following: + %% true = ets:insert(?REGISTRY, {Tab, 1, Pid}), + %% true = ets:insert(?OWNERS, {Pid, Tab}), {ok, Tab}; + {Reply, internal_open} -> + %% Clean up what do_internal_open() did: + true = ets:delete(?REGISTRY, Tab), + true = ets:delete(?OWNERS, Pid), + Reply; {Reply, _} -> % ok or Error Reply end, @@ -309,6 +315,12 @@ do_internal_open(State, From, Args) -> [T, _, _] -> T; [_, _] -> Ref end, + %% Pretend the table is open. If someone else tries to + %% open the file it will always become a pending + %% 'add_user' request. If someone tries to use the table + %% there will be a delay, but that is OK. + true = ets:insert(?REGISTRY, {Tab, 1, Pid}), + true = ets:insert(?OWNERS, {Pid, Tab}), pending_call(Tab, Pid, Ref, From, Args, internal_open, State); Error -> {Error, State} diff --git a/lib/stdlib/src/edlin.erl b/lib/stdlib/src/edlin.erl index be9a4f5107..b3bc5f6d92 100644 --- a/lib/stdlib/src/edlin.erl +++ b/lib/stdlib/src/edlin.erl @@ -390,7 +390,7 @@ do_op(end_of_line, Bef, [C|Aft], Rs) -> do_op(end_of_line, Bef, [], Rs) -> {{Bef,[]},Rs}; do_op(ctlu, Bef, Aft, Rs) -> - put(kill_buffer, Bef), + put(kill_buffer, reverse(Bef)), {{[], Aft}, [{delete_chars, -length(Bef)} | Rs]}; do_op(beep, Bef, Aft, Rs) -> {{Bef,Aft},[beep|Rs]}; diff --git a/lib/stdlib/src/erl_expand_records.erl b/lib/stdlib/src/erl_expand_records.erl index c74f68647f..64a00acd88 100644 --- a/lib/stdlib/src/erl_expand_records.erl +++ b/lib/stdlib/src/erl_expand_records.erl @@ -702,9 +702,10 @@ record_update(R, Name, Fs, Us0, St0) -> record_match(R, Name, Lr, Fs, Us, St0) -> {Ps,News,St1} = record_upd_fs(Fs, Us, St0), NLr = neg_line(Lr), + RLine = record_offset(Lr, St1), {{'case',Lr,R, - [{clause,Lr,[{tuple,Lr,[{atom,Lr,Name} | Ps]}],[], - [{tuple,Lr,[{atom,Lr,Name} | News]}]}, + [{clause,Lr,[{tuple,RLine,[{atom,Lr,Name} | Ps]}],[], + [{tuple,RLine,[{atom,Lr,Name} | News]}]}, {clause,NLr,[{var,NLr,'_'}],[], [call_error(NLr, {tuple,NLr,[{atom,NLr,badrecord},{atom,NLr,Name}]})]} ]}, @@ -733,6 +734,10 @@ record_setel(R, Name, Fs, Us0) -> Lr = element(2, hd(Us)), Wildcards = duplicate(length(Fs), {var,Lr,'_'}), NLr = neg_line(Lr), + %% Note: calling record_offset() here is not necessary since it is + %% targeted at Dialyzer which always calls the compiler with + %% 'strict_record_updates' meaning that record_setel() will never + %% be called. {'case',Lr,R, [{clause,Lr,[{tuple,Lr,[{atom,Lr,Name} | Wildcards]}],[], [foldr(fun ({I,Lf,Val}, Acc) -> diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index e1ae3b7aea..1d4a2a1fef 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -753,6 +753,9 @@ attribute_farity({cons,L,H,T}) -> attribute_farity({tuple,L,Args0}) -> Args = attribute_farity_list(Args0), {tuple,L,Args}; +attribute_farity({map,L,Args0}) -> + Args = attribute_farity_map(Args0), + {map,L,Args}; attribute_farity({op,L,'/',{atom,_,_}=Name,{integer,_,_}=Arity}) -> {tuple,L,[Name,Arity]}; attribute_farity(Other) -> Other. @@ -760,6 +763,10 @@ attribute_farity(Other) -> Other. attribute_farity_list(Args) -> [attribute_farity(A) || A <- Args]. +%% It is not meaningful to have farity keys. +attribute_farity_map(Args) -> + [{Op,L,K,attribute_farity(V)} || {Op,L,K,V} <- Args]. + -spec error_bad_decl(integer(), attributes()) -> no_return(). error_bad_decl(L, S) -> @@ -954,7 +961,9 @@ abstract([H|T], L, none=E) -> abstract(List, L, E) when is_list(List) -> abstract_list(List, [], L, E); abstract(Tuple, L, E) when is_tuple(Tuple) -> - {tuple,L,abstract_tuple_list(tuple_to_list(Tuple), L, E)}. + {tuple,L,abstract_tuple_list(tuple_to_list(Tuple), L, E)}; +abstract(Map, L, E) when is_map(Map) -> + {map,L,abstract_map_fields(maps:to_list(Map),L,E)}. abstract_list([H|T], String, L, E) -> case is_integer(H) andalso H >= 0 andalso E(H) of @@ -979,6 +988,9 @@ abstract_tuple_list([H|T], L, E) -> abstract_tuple_list([], _L, _E) -> []. +abstract_map_fields(Fs,L,E) -> + [{map_field_assoc,L,abstract(K,L,E),abstract(V,L,E)}||{K,V}<-Fs]. + abstract_byte(Byte, L) when is_integer(Byte) -> {bin_element, L, {integer, L, Byte}, default, default}; abstract_byte(Bits, L) -> diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl index 3dbb5ab64c..1fd6d2a8df 100644 --- a/lib/stdlib/src/erl_pp.erl +++ b/lib/stdlib/src/erl_pp.erl @@ -300,7 +300,7 @@ map_pair_types(Fs) -> tuple_type(Fs, fun map_pair_type/1). map_pair_type({type,_Line,map_field_assoc,Ktype,Vtype}) -> - map_assoc_typed(lexpr(Ktype, options(none)), Vtype). + map_assoc_typed(ltype(Ktype), Vtype). map_assoc_typed(B, {type,_,union,Ts}) -> {first,[B,$\s],{seq,[],[],[],map_assoc_union_type(Ts)}}; diff --git a/lib/stdlib/src/erl_tar.erl b/lib/stdlib/src/erl_tar.erl index acf7a5cd40..ab6223c0fe 100644 --- a/lib/stdlib/src/erl_tar.erl +++ b/lib/stdlib/src/erl_tar.erl @@ -22,7 +22,7 @@ %% Purpose: Unix tar (tape archive) utility. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --export([create/2, create/3, extract/1, extract/2, table/1, table/2, +-export([init/3, create/2, create/3, extract/1, extract/2, table/1, table/2, open/2, close/1, add/3, add/4, t/1, tt/1, format_error/1]). @@ -30,10 +30,16 @@ -record(add_opts, {read_info, % Fun to use for read file/link info. + chunk_size = 0, % For file reading when sending to sftp. 0=do not chunk verbose = false :: boolean()}). % Verbose on/off. %% Opens a tar archive. +init(UsrHandle, AccessMode, Fun) when is_function(Fun,2) -> + {ok, {AccessMode,{UsrHandle,Fun}}}. + +%%%================================================================ +%%% The open function with friends is to keep the file and binary api of this module open(Name, Mode) -> case open_mode(Mode) of {ok, Access, Raw, Opts} -> @@ -46,27 +52,37 @@ open1({binary,Bin}, read, _Raw, Opts) -> case file:open(Bin, [ram,binary,read]) of {ok,File} -> _ = [ram_file:uncompress(File) || Opts =:= [compressed]], - {ok,{read,File}}; + init(File,read,file_fun()); Error -> Error end; open1({file, Fd}, read, _Raw, _Opts) -> - {ok, {read, Fd}}; + init(Fd, read, file_fun()); open1(Name, Access, Raw, Opts) -> case file:open(Name, Raw ++ [binary, Access|Opts]) of {ok, File} -> - {ok, {Access, File}}; + init(File, Access, file_fun()); {error, Reason} -> {error, {Name, Reason}} end. +file_fun() -> + fun(write, {Fd,Data}) -> file:write(Fd, Data); + (position, {Fd,Pos}) -> file:position(Fd, Pos); + (read2, {Fd,Size}) -> file:read(Fd,Size); + (close, Fd) -> file:close(Fd) + end. + +%%% End of file and binary api (except for open_mode/1 downwards +%%%================================================================ + %% Closes a tar archive. close({read, File}) -> - ok = file:close(File); + ok = do_close(File); close({write, File}) -> PadResult = pad_file(File), - ok = file:close(File), + ok = do_close(File), PadResult; close(_) -> {error, einval}. @@ -75,7 +91,6 @@ close(_) -> add(File, Name, Options) -> add(File, Name, Name, Options). - add({write, File}, Name, NameInArchive, Options) -> Opts = #add_opts{read_info=fun(F) -> file:read_link_info(F) end}, add1(File, Name, NameInArchive, add_opts(Options, Opts)); @@ -88,6 +103,8 @@ add_opts([dereference|T], Opts) -> add_opts(T, Opts#add_opts{read_info=fun(F) -> file:read_file_info(F) end}); add_opts([verbose|T], Opts) -> add_opts(T, Opts#add_opts{verbose=true}); +add_opts([{chunks,N}|T], Opts) -> + add_opts(T, Opts#add_opts{chunk_size=N}); add_opts([_|T], Opts) -> add_opts(T, Opts); add_opts([], Opts) -> @@ -321,16 +338,46 @@ add1(TarFile, Name, NameInArchive, Opts) -> {error, {Name, Reason}} end. +add1(Tar, Name, Header, chunked, Options) -> + add_verbose(Options, "a ~ts [chunked ", [Name]), + try + ok = do_write(Tar, Header), + {ok,D} = file:open(Name, [read,binary]), + {ok,NumBytes} = add_read_write_chunks(D, Tar, Options#add_opts.chunk_size, 0, Options), + _ = file:close(D), + ok = do_write(Tar, padding(NumBytes,?record_size)) + of + ok -> + add_verbose(Options, "~n", []), + ok + catch + error:{badmatch,{error,Error}} -> + add_verbose(Options, "~n", []), + {error,{Name,Error}} + end; add1(Tar, Name, Header, Bin, Options) -> add_verbose(Options, "a ~ts~n", [Name]), - file:write(Tar, [Header, Bin, padding(byte_size(Bin), ?record_size)]). + do_write(Tar, [Header, Bin, padding(byte_size(Bin), ?record_size)]). + +add_read_write_chunks(D, Tar, ChunkSize, SumNumBytes, Options) -> + case file:read(D, ChunkSize) of + {ok,Bin} -> + ok = do_write(Tar, Bin), + add_verbose(Options, ".", []), + add_read_write_chunks(D, Tar, ChunkSize, SumNumBytes+byte_size(Bin), Options); + eof -> + add_verbose(Options, "]", []), + {ok,SumNumBytes}; + Other -> + Other + end. add_directory(TarFile, DirName, NameInArchive, Info, Options) -> case file:list_dir(DirName) of {ok, []} -> add_verbose(Options, "a ~ts~n", [DirName]), Header = create_header(NameInArchive, Info), - file:write(TarFile, Header); + do_write(TarFile, Header); {ok, Files} -> Add = fun (File) -> add1(TarFile, @@ -396,7 +443,7 @@ to_string(Str0, Count) -> %% Pads out end of file. pad_file(File) -> - {ok,Position} = file:position(File, {cur,0}), + {ok,Position} = do_position(File, {cur,0}), %% There must be at least two zero records at the end. Fill = case ?block_size - (Position rem ?block_size) of Fill0 when Fill0 < 2*?record_size -> @@ -407,7 +454,7 @@ pad_file(File) -> %% Large enough. Fill0 end, - file:write(File, zeroes(Fill)). + do_write(File, zeroes(Fill)). split_filename(Name) when length(Name) =< ?th_name_len -> {"", Name}; @@ -500,7 +547,7 @@ foldl_read(TarName, Fun, Accu, Opts) -> Ok -> Ok end, - ok = file:close(File), + ok = do_close(File), Result; Error -> Error @@ -559,7 +606,7 @@ check_extract(Name, #read_opts{files=Files}) -> ordsets:is_element(Name, Files). get_header(File) -> - case file:read(File, ?record_size) of + case do_read(File, ?record_size) of eof -> throw({error,eof}); {ok, Bin} when is_binary(Bin) -> @@ -690,7 +737,7 @@ get_element(File, #tar_header{size = 0}) -> skip_to_next(File), {ok,<<>>}; get_element(File, #tar_header{size = Size}) -> - case file:read(File, Size) of + case do_read(File, Size) of {ok,Bin}=Res when byte_size(Bin) =:= Size -> skip_to_next(File), Res; @@ -880,7 +927,7 @@ skip(File, Size) -> %% Note: There is no point in handling failure to get the current position %% in the file. If it doesn't work, something serious is wrong. Amount = ((Size + ?record_size - 1) div ?record_size) * ?record_size, - {ok,_} = file:position(File, {cur, Amount}), + {ok,_} = do_position(File, {cur, Amount}), ok. %% Skips to the next record in the file. @@ -888,9 +935,9 @@ skip(File, Size) -> skip_to_next(File) -> %% Note: There is no point in handling failure to get the current position %% in the file. If it doesn't work, something serious is wrong. - {ok, Position} = file:position(File, {cur, 0}), + {ok, Position} = do_position(File, {cur, 0}), NewPosition = ((Position + ?record_size - 1) div ?record_size) * ?record_size, - {ok,NewPosition} = file:position(File, NewPosition), + {ok,NewPosition} = do_position(File, NewPosition), ok. %% Prints the message on if the verbose option is given. @@ -916,6 +963,9 @@ posix_to_erlang_time(Sec) -> read_file_and_info(Name, Opts) -> ReadInfo = Opts#add_opts.read_info, case ReadInfo(Name) of + {ok,Info} when Info#file_info.type =:= regular, + Opts#add_opts.chunk_size>0 -> + {ok,chunked,Info}; {ok,Info} when Info#file_info.type =:= regular -> case file:read_file(Name) of {ok,Bin} -> @@ -962,3 +1012,12 @@ open_mode([], Access, Raw, Opts) -> {ok, Access, Raw, Opts}; open_mode(_, _, _, _) -> {error, einval}. + +%%%================================================================ +do_write({UsrHandle,Fun}, Data) -> Fun(write,{UsrHandle,Data}). + +do_position({UsrHandle,Fun}, Pos) -> Fun(position,{UsrHandle,Pos}). + +do_read({UsrHandle,Fun}, Len) -> Fun(read2,{UsrHandle,Len}). + +do_close({UsrHandle,Fun}) -> Fun(close,UsrHandle). diff --git a/lib/stdlib/src/filelib.erl b/lib/stdlib/src/filelib.erl index 9efbe8da20..daae1fd2d2 100644 --- a/lib/stdlib/src/filelib.erl +++ b/lib/stdlib/src/filelib.erl @@ -371,7 +371,7 @@ compile_wildcard(Pattern, Cwd0) -> [Root|Rest] = filename:split(Pattern), case filename:pathtype(Root) of relative -> - Cwd = filename:join([Cwd0]), + Cwd = prepare_base(Cwd0), compile_wildcard_2([Root|Rest], {cwd,Cwd}); _ -> compile_wildcard_2(Rest, {root,0,Root}) diff --git a/lib/stdlib/src/gen_event.erl b/lib/stdlib/src/gen_event.erl index d39dd89d3a..469acdc37c 100644 --- a/lib/stdlib/src/gen_event.erl +++ b/lib/stdlib/src/gen_event.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -49,8 +49,6 @@ -import(error_logger, [error_msg/2]). --define(reply(X), From ! {element(2,Tag), X}). - -record(handler, {module :: atom(), id = false, state, @@ -249,49 +247,49 @@ handle_msg(Msg, Parent, ServerName, MSL, Debug) -> {notify, Event} -> {Hib,MSL1} = server_notify(Event, handle_event, MSL, ServerName), loop(Parent, ServerName, MSL1, Debug, Hib); - {From, Tag, {sync_notify, Event}} -> + {_From, Tag, {sync_notify, Event}} -> {Hib, MSL1} = server_notify(Event, handle_event, MSL, ServerName), - ?reply(ok), + reply(Tag, ok), loop(Parent, ServerName, MSL1, Debug, Hib); {'EXIT', From, Reason} -> MSL1 = handle_exit(From, Reason, MSL, ServerName), loop(Parent, ServerName, MSL1, Debug, false); - {From, Tag, {call, Handler, Query}} -> + {_From, Tag, {call, Handler, Query}} -> {Hib, Reply, MSL1} = server_call(Handler, Query, MSL, ServerName), - ?reply(Reply), + reply(Tag, Reply), loop(Parent, ServerName, MSL1, Debug, Hib); - {From, Tag, {add_handler, Handler, Args}} -> + {_From, Tag, {add_handler, Handler, Args}} -> {Hib, Reply, MSL1} = server_add_handler(Handler, Args, MSL), - ?reply(Reply), + reply(Tag, Reply), loop(Parent, ServerName, MSL1, Debug, Hib); - {From, Tag, {add_sup_handler, Handler, Args, SupP}} -> + {_From, Tag, {add_sup_handler, Handler, Args, SupP}} -> {Hib, Reply, MSL1} = server_add_sup_handler(Handler, Args, MSL, SupP), - ?reply(Reply), + reply(Tag, Reply), loop(Parent, ServerName, MSL1, Debug, Hib); - {From, Tag, {delete_handler, Handler, Args}} -> + {_From, Tag, {delete_handler, Handler, Args}} -> {Reply, MSL1} = server_delete_handler(Handler, Args, MSL, ServerName), - ?reply(Reply), + reply(Tag, Reply), loop(Parent, ServerName, MSL1, Debug, false); - {From, Tag, {swap_handler, Handler1, Args1, Handler2, Args2}} -> + {_From, Tag, {swap_handler, Handler1, Args1, Handler2, Args2}} -> {Hib, Reply, MSL1} = server_swap_handler(Handler1, Args1, Handler2, Args2, MSL, ServerName), - ?reply(Reply), + reply(Tag, Reply), loop(Parent, ServerName, MSL1, Debug, Hib); - {From, Tag, {swap_sup_handler, Handler1, Args1, Handler2, Args2, + {_From, Tag, {swap_sup_handler, Handler1, Args1, Handler2, Args2, Sup}} -> {Hib, Reply, MSL1} = server_swap_handler(Handler1, Args1, Handler2, Args2, MSL, Sup, ServerName), - ?reply(Reply), + reply(Tag, Reply), loop(Parent, ServerName, MSL1, Debug, Hib); - {From, Tag, stop} -> + {_From, Tag, stop} -> catch terminate_server(normal, Parent, MSL, ServerName), - ?reply(ok); - {From, Tag, which_handlers} -> - ?reply(the_handlers(MSL)), + reply(Tag, ok); + {_From, Tag, which_handlers} -> + reply(Tag, the_handlers(MSL)), loop(Parent, ServerName, MSL, Debug, false); - {From, Tag, get_modules} -> - ?reply(get_modules(MSL)), + {_From, Tag, get_modules} -> + reply(Tag, get_modules(MSL)), loop(Parent, ServerName, MSL, Debug, false); Other -> {Hib, MSL1} = server_notify(Other, handle_info, MSL, ServerName), @@ -303,6 +301,10 @@ terminate_server(Reason, Parent, MSL, ServerName) -> do_unlink(Parent, MSL), exit(Reason). +reply({From, Ref}, Msg) -> + From ! {Ref, Msg}, + ok. + %% unlink the supervisor process of all supervised handlers. %% We do not want a handler supervisor to EXIT due to the %% termination of the event manager (server). diff --git a/lib/stdlib/src/gen_server.erl b/lib/stdlib/src/gen_server.erl index dadfe56b3d..528dd23e1c 100644 --- a/lib/stdlib/src/gen_server.erl +++ b/lib/stdlib/src/gen_server.erl @@ -567,28 +567,88 @@ start_monitor(Node, Name) when is_atom(Node), is_atom(Name) -> end end. +%% --------------------------------------------------- +%% Helper functions for try-catch of callbacks. +%% Returns the return value of the callback, or +%% {'EXIT', ExitReason, ReportReason} (if an exception occurs) +%% +%% ExitReason is the reason that shall be used when the process +%% terminates. +%% +%% ReportReason is the reason that shall be printed in the error +%% report. +%% +%% These functions are introduced in order to add the stack trace in +%% the error report produced when a callback is terminated with +%% erlang:exit/1 (OTP-12263). +%% --------------------------------------------------- + +try_dispatch({'$gen_cast', Msg}, Mod, State) -> + try_dispatch(Mod, handle_cast, Msg, State); +try_dispatch(Info, Mod, State) -> + try_dispatch(Mod, handle_info, Info, State). + +try_dispatch(Mod, Func, Msg, State) -> + try + {ok, Mod:Func(Msg, State)} + catch + throw:R -> + {ok, R}; + error:R -> + Stacktrace = erlang:get_stacktrace(), + {'EXIT', {R, Stacktrace}, {R, Stacktrace}}; + exit:R -> + Stacktrace = erlang:get_stacktrace(), + {'EXIT', R, {R, Stacktrace}} + end. + +try_handle_call(Mod, Msg, From, State) -> + try + {ok, Mod:handle_call(Msg, From, State)} + catch + throw:R -> + {ok, R}; + error:R -> + Stacktrace = erlang:get_stacktrace(), + {'EXIT', {R, Stacktrace}, {R, Stacktrace}}; + exit:R -> + Stacktrace = erlang:get_stacktrace(), + {'EXIT', R, {R, Stacktrace}} + end. + +try_terminate(Mod, Reason, State) -> + try + {ok, Mod:terminate(Reason, State)} + catch + throw:R -> + {ok, R}; + error:R -> + Stacktrace = erlang:get_stacktrace(), + {'EXIT', {R, Stacktrace}, {R, Stacktrace}}; + exit:R -> + Stacktrace = erlang:get_stacktrace(), + {'EXIT', R, {R, Stacktrace}} + end. + + %%% --------------------------------------------------- %%% Message handling functions %%% --------------------------------------------------- -dispatch({'$gen_cast', Msg}, Mod, State) -> - Mod:handle_cast(Msg, State); -dispatch(Info, Mod, State) -> - Mod:handle_info(Info, State). - handle_msg({'$gen_call', From, Msg}, Parent, Name, State, Mod) -> - case catch Mod:handle_call(Msg, From, State) of - {reply, Reply, NState} -> + Result = try_handle_call(Mod, Msg, From, State), + case Result of + {ok, {reply, Reply, NState}} -> reply(From, Reply), loop(Parent, Name, NState, Mod, infinity, []); - {reply, Reply, NState, Time1} -> + {ok, {reply, Reply, NState, Time1}} -> reply(From, Reply), loop(Parent, Name, NState, Mod, Time1, []); - {noreply, NState} -> + {ok, {noreply, NState}} -> loop(Parent, Name, NState, Mod, infinity, []); - {noreply, NState, Time1} -> + {ok, {noreply, NState, Time1}} -> loop(Parent, Name, NState, Mod, Time1, []); - {stop, Reason, Reply, NState} -> + {ok, {stop, Reason, Reply, NState}} -> {'EXIT', R} = (catch terminate(Reason, Name, Msg, Mod, NState, [])), reply(From, Reply), @@ -596,26 +656,27 @@ handle_msg({'$gen_call', From, Msg}, Parent, Name, State, Mod) -> Other -> handle_common_reply(Other, Parent, Name, Msg, Mod, State) end; handle_msg(Msg, Parent, Name, State, Mod) -> - Reply = (catch dispatch(Msg, Mod, State)), + Reply = try_dispatch(Msg, Mod, State), handle_common_reply(Reply, Parent, Name, Msg, Mod, State). handle_msg({'$gen_call', From, Msg}, Parent, Name, State, Mod, Debug) -> - case catch Mod:handle_call(Msg, From, State) of - {reply, Reply, NState} -> + Result = try_handle_call(Mod, Msg, From, State), + case Result of + {ok, {reply, Reply, NState}} -> Debug1 = reply(Name, From, Reply, NState, Debug), loop(Parent, Name, NState, Mod, infinity, Debug1); - {reply, Reply, NState, Time1} -> + {ok, {reply, Reply, NState, Time1}} -> Debug1 = reply(Name, From, Reply, NState, Debug), loop(Parent, Name, NState, Mod, Time1, Debug1); - {noreply, NState} -> + {ok, {noreply, NState}} -> Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, {noreply, NState}), loop(Parent, Name, NState, Mod, infinity, Debug1); - {noreply, NState, Time1} -> + {ok, {noreply, NState, Time1}} -> Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, {noreply, NState}), loop(Parent, Name, NState, Mod, Time1, Debug1); - {stop, Reason, Reply, NState} -> + {ok, {stop, Reason, Reply, NState}} -> {'EXIT', R} = (catch terminate(Reason, Name, Msg, Mod, NState, Debug)), _ = reply(Name, From, Reply, NState, Debug), @@ -624,39 +685,39 @@ handle_msg({'$gen_call', From, Msg}, Parent, Name, State, Mod, Debug) -> handle_common_reply(Other, Parent, Name, Msg, Mod, State, Debug) end; handle_msg(Msg, Parent, Name, State, Mod, Debug) -> - Reply = (catch dispatch(Msg, Mod, State)), + Reply = try_dispatch(Msg, Mod, State), handle_common_reply(Reply, Parent, Name, Msg, Mod, State, Debug). handle_common_reply(Reply, Parent, Name, Msg, Mod, State) -> case Reply of - {noreply, NState} -> + {ok, {noreply, NState}} -> loop(Parent, Name, NState, Mod, infinity, []); - {noreply, NState, Time1} -> + {ok, {noreply, NState, Time1}} -> loop(Parent, Name, NState, Mod, Time1, []); - {stop, Reason, NState} -> + {ok, {stop, Reason, NState}} -> terminate(Reason, Name, Msg, Mod, NState, []); - {'EXIT', What} -> - terminate(What, Name, Msg, Mod, State, []); - _ -> - terminate({bad_return_value, Reply}, Name, Msg, Mod, State, []) + {'EXIT', ExitReason, ReportReason} -> + terminate(ExitReason, ReportReason, Name, Msg, Mod, State, []); + {ok, BadReply} -> + terminate({bad_return_value, BadReply}, Name, Msg, Mod, State, []) end. handle_common_reply(Reply, Parent, Name, Msg, Mod, State, Debug) -> case Reply of - {noreply, NState} -> + {ok, {noreply, NState}} -> Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, {noreply, NState}), loop(Parent, Name, NState, Mod, infinity, Debug1); - {noreply, NState, Time1} -> + {ok, {noreply, NState, Time1}} -> Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, {noreply, NState}), loop(Parent, Name, NState, Mod, Time1, Debug1); - {stop, Reason, NState} -> + {ok, {stop, Reason, NState}} -> terminate(Reason, Name, Msg, Mod, NState, Debug); - {'EXIT', What} -> - terminate(What, Name, Msg, Mod, State, Debug); - _ -> - terminate({bad_return_value, Reply}, Name, Msg, Mod, State, Debug) + {'EXIT', ExitReason, ReportReason} -> + terminate(ExitReason, ReportReason, Name, Msg, Mod, State, Debug); + {ok, BadReply} -> + terminate({bad_return_value, BadReply}, Name, Msg, Mod, State, Debug) end. reply(Name, {To, Tag}, Reply, State, Debug) -> @@ -718,13 +779,16 @@ print_event(Dev, Event, Name) -> %%% --------------------------------------------------- terminate(Reason, Name, Msg, Mod, State, Debug) -> - case catch Mod:terminate(Reason, State) of - {'EXIT', R} -> + terminate(Reason, Reason, Name, Msg, Mod, State, Debug). +terminate(ExitReason, ReportReason, Name, Msg, Mod, State, Debug) -> + Reply = try_terminate(Mod, ExitReason, State), + case Reply of + {'EXIT', ExitReason1, ReportReason1} -> FmtState = format_status(terminate, Mod, get(), State), - error_info(R, Name, Msg, FmtState, Debug), - exit(R); + error_info(ReportReason1, Name, Msg, FmtState, Debug), + exit(ExitReason1); _ -> - case Reason of + case ExitReason of normal -> exit(normal); shutdown -> @@ -733,8 +797,8 @@ terminate(Reason, Name, Msg, Mod, State, Debug) -> exit(Shutdown); _ -> FmtState = format_status(terminate, Mod, get(), State), - error_info(Reason, Name, Msg, FmtState, Debug), - exit(Reason) + error_info(ReportReason, Name, Msg, FmtState, Debug), + exit(ExitReason) end end. diff --git a/lib/stdlib/src/io.erl b/lib/stdlib/src/io.erl index 27e2a82b41..b9ace2f442 100644 --- a/lib/stdlib/src/io.erl +++ b/lib/stdlib/src/io.erl @@ -566,12 +566,23 @@ request(Name, Request) when is_atom(Name) -> execute_request(Pid, {Convert,Converted}) -> Mref = erlang:monitor(process, Pid), - Pid ! {io_request,self(),Pid,Converted}, - if - Convert -> - convert_binaries(wait_io_mon_reply(Pid, Mref)); - true -> - wait_io_mon_reply(Pid, Mref) + Pid ! {io_request,self(),Mref,Converted}, + + receive + {io_reply, Mref, Reply} -> + erlang:demonitor(Mref, [flush]), + if + Convert -> + convert_binaries(Reply); + true -> + Reply + end; + {'DOWN', Mref, _, _, _} -> + receive + {'EXIT', Pid, _What} -> true + after 0 -> true + end, + {error,terminated} end. requests(Requests) -> %Requests as atomic action @@ -597,26 +608,6 @@ default_input() -> default_output() -> group_leader(). -wait_io_mon_reply(From, Mref) -> - receive - {io_reply, From, Reply} -> - erlang:demonitor(Mref, [flush]), - Reply; - {'EXIT', From, _What} -> - receive - {'DOWN', Mref, _, _, _} -> true - after 0 -> true - end, - {error,terminated}; - {'DOWN', Mref, _, _, _} -> - receive - {'EXIT', From, _What} -> true - after 0 -> true - end, - {error,terminated} - end. - - %% io_requests(Requests) %% Transform requests into correct i/o server messages. Only handle the %% one we KNOW must be changed, others, including incorrect ones, are diff --git a/lib/stdlib/src/ms_transform.erl b/lib/stdlib/src/ms_transform.erl index 27dfcf52e1..97564e2e44 100644 --- a/lib/stdlib/src/ms_transform.erl +++ b/lib/stdlib/src/ms_transform.erl @@ -725,10 +725,10 @@ transform_head([V],OuterBound) -> th(NewV,NewBind,OuterBound). -toplevel_head_match({match,Line,{var,_,VName},Expr},B,OB) -> +toplevel_head_match({match,_,{var,Line,VName},Expr},B,OB) -> warn_var_clash(Line,VName,OB), {Expr,new_bind({VName,'$_'},B)}; -toplevel_head_match({match,Line,Expr,{var,_,VName}},B,OB) -> +toplevel_head_match({match,_,Expr,{var,Line,VName}},B,OB) -> warn_var_clash(Line,VName,OB), {Expr,new_bind({VName,'$_'},B)}; toplevel_head_match(Other,B,_OB) -> diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index 99d9b8b431..7802ea884f 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -17,11 +17,11 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"2\\.1(\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.1 + [{<<"2\\.[1-2](\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.1-17.3 {<<"2\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.0 {<<"1\\.19(\\.[0-9]+)*">>,[restart_new_emulator]}],%% R16 %% Down to - max one major revision back - [{<<"2\\.1(\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.1 + [{<<"2\\.[1-2](\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.1-17.3 {<<"2\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.0 {<<"1\\.19(\\.[0-9]+)*">>,[restart_new_emulator]}] %% R16 }. diff --git a/lib/stdlib/test/dets_SUITE.erl b/lib/stdlib/test/dets_SUITE.erl index 119b4dc7cb..3b08ac165e 100644 --- a/lib/stdlib/test/dets_SUITE.erl +++ b/lib/stdlib/test/dets_SUITE.erl @@ -223,8 +223,7 @@ open(Config, Version) -> ?format("Crashing dets server \n", []), process_flag(trap_exit, true), - Procs = [whereis(?DETS_SERVER) | map(fun(Tab) -> dets:info(Tab, pid) end, - Tabs)], + Procs = [whereis(?DETS_SERVER) | [dets:info(Tab, pid) || Tab <- Tabs]], foreach(fun(Pid) -> exit(Pid, kill) end, Procs), timer:sleep(100), c:flush(), %% flush all the EXIT sigs @@ -235,18 +234,32 @@ open(Config, Version) -> open_files(1, All, Version), ?format("Checking contents of repaired files \n", []), check(Tabs, Data), - - close_all(Tabs), + close_all(Tabs), delete_files(All), - P1 = pps(), + {Ports0, Procs0} = P0, - {Ports1, Procs1} = P1, - true = Ports1 =:= Ports0, - %% The dets_server process has been restarted: - [_] = Procs0 -- Procs1, - [_] = Procs1 -- Procs0, - ok. + Test = fun() -> + P1 = pps(), + {Ports1, Procs1} = P1, + show("Old port", Ports0 -- Ports1), + show("New port", Ports1 -- Ports0), + show("Old procs", Procs0 -- Procs1), + show("New procs", Procs1 -- Procs0), + io:format("Remaining Dets-pids (should be nil): ~p~n", + [find_dets_pids()]), + true = Ports1 =:= Ports0, + %% The dets_server process has been restarted: + [_] = Procs0 -- Procs1, + [_] = Procs1 -- Procs0, + ok + end, + case catch Test() of + ok -> ok; + _ -> + timer:sleep(500), + ok = Test() + end. check(Tabs, Data) -> foreach(fun(Tab) -> @@ -3275,12 +3288,22 @@ simultaneous_open(Config) -> File = filename(Tab, Config), ok = monit(Tab, File), - ok = kill_while_repairing(Tab, File), - ok = kill_while_init(Tab, File), - ok = open_ro(Tab, File), - ok = open_w(Tab, File, 0, Config), - ok = open_w(Tab, File, 100, Config), - ok. + case feasible() of + false -> {comment, "OK, but did not run all of the test"}; + true -> + ok = kill_while_repairing(Tab, File), + ok = kill_while_init(Tab, File), + ok = open_ro(Tab, File), + ok = open_w(Tab, File, 0, Config), + ok = open_w(Tab, File, 100, Config) + end. + +feasible() -> + LP = erlang:system_info(logical_processors), + (is_integer(LP) + andalso LP >= erlang:system_info(schedulers_online) + andalso not erlang:system_info(debug_compiled) + andalso not erlang:system_info(lock_checking)). %% One process logs and another process closes the log. Before %% monitors were used, this would make the client never return. @@ -3307,7 +3330,6 @@ kill_while_repairing(Tab, File) -> Delay = 1000, dets:start(), Parent = self(), - Ps = processes(), F = fun() -> R = (catch dets:open_file(Tab, [{file,File}])), timer:sleep(Delay), @@ -3318,7 +3340,7 @@ kill_while_repairing(Tab, File) -> P1 = spawn(F), P2 = spawn(F), P3 = spawn(F), - DetsPid = find_dets_pid([P1, P2, P3 | Ps]), + DetsPid = find_dets_pid(), exit(DetsPid, kill), receive {P1,R1} -> R1 end, @@ -3342,12 +3364,6 @@ kill_while_repairing(Tab, File) -> file:delete(File), ok. -find_dets_pid(P0) -> - case lists:sort(processes() -- P0) of - [P, _] -> P; - _ -> timer:sleep(100), find_dets_pid(P0) - end. - find_dets_pid() -> case find_dets_pids() of [] -> @@ -3421,6 +3437,13 @@ open_ro(Tab, File) -> open_w(Tab, File, Delay, Config) -> create_opened_log(File), + + Tab2 = t2, + File2 = filename(Tab2, Config), + file:delete(File2), + {ok,Tab2} = dets:open_file(Tab2, [{file,File2}]), + ok = dets:close(Tab2), + Parent = self(), F = fun() -> R = dets:open_file(Tab, [{file,File}]), @@ -3430,16 +3453,16 @@ open_w(Tab, File, Delay, Config) -> Pid1 = spawn(F), Pid2 = spawn(F), Pid3 = spawn(F), - undefined = dets:info(Tab), % is repairing now - 0 = qlen(), - Tab2 = t2, - File2 = filename(Tab2, Config), - file:delete(File2), + ok = wait_for_repair_to_start(Tab), + + %% It is assumed that it takes some time to repair the file. {ok,Tab2} = dets:open_file(Tab2, [{file,File2}]), + %% The Dets server managed to handle to open_file request. + 0 = qlen(), % still repairing + ok = dets:close(Tab2), file:delete(File2), - 0 = qlen(), % still repairing receive {Pid1,R1} -> {ok, Tab} = R1 end, receive {Pid2,R2} -> {ok, Tab} = R2 end, @@ -3456,6 +3479,15 @@ open_w(Tab, File, Delay, Config) -> file:delete(File), ok. +wait_for_repair_to_start(Tab) -> + case catch dets_server:get_pid(Tab) of + {'EXIT', _} -> + timer:sleep(1), + wait_for_repair_to_start(Tab); + Pid when is_pid(Pid) -> + ok + end. + qlen() -> {_, {_, N}} = lists:keysearch(message_queue_len, 1, process_info(self())), N. @@ -4350,6 +4382,7 @@ check_badarg({'EXIT', {badarg, [{M,F,A,_} | _]}}, M, F, Args) -> true = test_server:is_native(M) andalso length(Args) =:= A. check_pps({Ports0,Procs0} = P0) -> + ok = check_dets_tables(), case pps() of P0 -> ok; @@ -4375,13 +4408,45 @@ check_pps({Ports0,Procs0} = P0) -> end end. +%% Copied from dets_server.erl: +-define(REGISTRY, dets_registry). +-define(OWNERS, dets_owners). +-define(STORE, dets). + +check_dets_tables() -> + Store = [T || + T <- ets:all(), + ets:info(T, name) =:= ?STORE, + owner(T) =:= dets], + S = case Store of + [Tab] -> ets:tab2list(Tab); + [] -> [] + end, + case {ets:tab2list(?REGISTRY), ets:tab2list(?OWNERS), S} of + {[], [], []} -> ok; + {R, O, _} -> + io:format("Registry: ~p~n", [R]), + io:format("Owners: ~p~n", [O]), + io:format("Store: ~p~n", [S]), + not_ok + end. + +owner(Tab) -> + Owner = ets:info(Tab, owner), + case process_info(Owner, registered_name) of + {registered_name, Name} -> Name; + _ -> Owner + end. + show(_S, []) -> ok; -show(S, [Pid|Pids]) when is_pid(Pid) -> - io:format("~s: ~p~n", [S, erlang:process_info(Pid)]), +show(S, [{Pid, Name, InitCall}|Pids]) when is_pid(Pid) -> + io:format("~s: ~w (~w), ~w: ~p~n", + [S, Pid, proc_reg_name(Name), InitCall, + erlang:process_info(Pid)]), show(S, Pids); -show(S, [Port|Ports]) when is_port(Port)-> - io:format("~s: ~p~n", [S, erlang:port_info(Port)]), +show(S, [{Port, _}|Ports]) when is_port(Port)-> + io:format("~s: ~w: ~p~n", [S, Port, erlang:port_info(Port)]), show(S, Ports). pps() -> @@ -4397,5 +4462,8 @@ process_list() -> safe_second_element(process_info(P, initial_call))} || P <- processes()]. +proc_reg_name({registered_name, Name}) -> Name; +proc_reg_name([]) -> no_reg_name. + safe_second_element({_,Info}) -> Info; safe_second_element(Other) -> Other. diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl index babf3a49eb..927fe0b595 100644 --- a/lib/stdlib/test/erl_pp_SUITE.erl +++ b/lib/stdlib/test/erl_pp_SUITE.erl @@ -604,20 +604,20 @@ import_export(Config) when is_list(Config) -> misc_attrs(suite) -> []; misc_attrs(Config) when is_list(Config) -> - ?line ok = pp_forms(<<"-module(m). ">>), - ?line ok = pp_forms(<<"-module(m, [Aafjlksfjdlsjflsdfjlsdjflkdsfjlk," - "Blsjfdlslfjsdf]). ">>), - ?line ok = pp_forms(<<"-export([]). ">>), - ?line ok = pp_forms(<<"-export([foo/2, bar/0]). ">>), - ?line ok = pp_forms(<<"-export([bar/0]). ">>), - ?line ok = pp_forms(<<"-import(lists, []). ">>), - ?line ok = pp_forms(<<"-import(lists, [map/2]). ">>), - ?line ok = pp_forms(<<"-import(lists, [map/2, foreach/2]). ">>), - ?line ok = pp_forms(<<"-'wild '({attr2,3}). ">>), - ?line ok = pp_forms(<<"-record(a, {b,c}). ">>), - ?line ok = pp_forms(<<"-record(' a ', {}). ">>), - ?line ok = pp_forms(<<"-record(' a ', {foo = foo:bar()}). ">>), - + ok = pp_forms(<<"-module(m). ">>), + ok = pp_forms(<<"-module(m, [Aafjlksfjdlsjflsdfjlsdjflkdsfjlk," + "Blsjfdlslfjsdf]). ">>), + ok = pp_forms(<<"-export([]). ">>), + ok = pp_forms(<<"-export([foo/2, bar/0]). ">>), + ok = pp_forms(<<"-export([bar/0]). ">>), + ok = pp_forms(<<"-import(lists, []). ">>), + ok = pp_forms(<<"-import(lists, [map/2]). ">>), + ok = pp_forms(<<"-import(lists, [map/2, foreach/2]). ">>), + ok = pp_forms(<<"-'wild '({attr2,3}). ">>), + ok = pp_forms(<<"-record(a, {b,c}). ">>), + ok = pp_forms(<<"-record(' a ', {}). ">>), + ok = pp_forms(<<"-record(' a ', {foo = foo:bar()}). ">>), + ok = pp_forms(<<"-custom1(#{test1 => init/2, test2 => [val/1, val/2]}). ">>), ok. dialyzer_attrs(suite) -> diff --git a/lib/stdlib/test/filelib_SUITE.erl b/lib/stdlib/test/filelib_SUITE.erl index 040ae1effc..bd313390b3 100644 --- a/lib/stdlib/test/filelib_SUITE.erl +++ b/lib/stdlib/test/filelib_SUITE.erl @@ -88,6 +88,7 @@ wildcard_two(Config) when is_list(Config) -> ?line ok = file:make_dir(Dir), ?line do_wildcard_1(Dir, fun(Wc) -> io:format("~p~n",[{Wc,Dir, X = filelib:wildcard(Wc, Dir)}]),X end), ?line do_wildcard_1(Dir, fun(Wc) -> filelib:wildcard(Wc, Dir++"/") end), + ?line do_wildcard_1(Dir, fun(Wc) -> filelib:wildcard(Wc, Dir++"/.") end), case os:type() of {win32,_} -> ok; diff --git a/lib/stdlib/test/gen_server_SUITE.erl b/lib/stdlib/test/gen_server_SUITE.erl index 42694d8b5d..0f03fda30a 100644 --- a/lib/stdlib/test/gen_server_SUITE.erl +++ b/lib/stdlib/test/gen_server_SUITE.erl @@ -275,7 +275,9 @@ crash(Config) when is_list(Config) -> receive {error,_GroupLeader4,{Pid4, "** Generic server"++_, - [Pid4,crash,{formatted, state4},crashed]}} -> + [Pid4,crash,{formatted, state4}, + {crashed,[{?MODULE,handle_call,3,_} + |_Stacktrace]}]}} -> ok; Other4a -> ?line io:format("Unexpected: ~p", [Other4a]), @@ -1026,7 +1028,9 @@ error_format_status(Config) when is_list(Config) -> receive {error,_GroupLeader,{Pid, "** Generic server"++_, - [Pid,crash,{formatted, State},crashed]}} -> + [Pid,crash,{formatted, State}, + {crashed,[{?MODULE,handle_call,3,_} + |_Stacktrace]}]}} -> ok; Other -> ?line io:format("Unexpected: ~p", [Other]), @@ -1048,7 +1052,9 @@ terminate_crash_format(Config) when is_list(Config) -> receive {error,_GroupLeader,{Pid, "** Generic server"++_, - [Pid,stop, {formatted, State},{crash, terminate}]}} -> + [Pid,stop, {formatted, State}, + {{crash, terminate},[{?MODULE,terminate,2,_} + |_Stacktrace]}]}} -> ok; Other -> io:format("Unexpected: ~p", [Other]), diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl index 3a76275f31..2203dd8f51 100644 --- a/lib/stdlib/test/io_SUITE.erl +++ b/lib/stdlib/test/io_SUITE.erl @@ -30,7 +30,8 @@ io_fread_newlines/1, otp_8989/1, io_lib_fread_literal/1, printable_range/1, io_lib_print_binary_depth_one/1, otp_10302/1, otp_10755/1, - otp_10836/1, io_lib_width_too_small/1]). + otp_10836/1, io_lib_width_too_small/1, + io_with_huge_message_queue/1]). -export([pretty/2]). @@ -70,7 +71,7 @@ all() -> io_fread_newlines, otp_8989, io_lib_fread_literal, printable_range, io_lib_print_binary_depth_one, otp_10302, otp_10755, otp_10836, - io_lib_width_too_small]. + io_lib_width_too_small, io_with_huge_message_queue]. groups() -> []. @@ -2219,3 +2220,44 @@ io_lib_width_too_small(Config) -> "**" = lists:flatten(io_lib:format("~2.3w", [3.14])), "**" = lists:flatten(io_lib:format("~2.5w", [3.14])), ok. + +%% Test that the time for a huge message queue is not +%% significantly slower than with an empty message queue. +io_with_huge_message_queue(Config) when is_list(Config) -> + case test_server:is_native(gen) of + true -> + {skip, + "gen is native - huge message queue optimization " + "is not implemented"}; + false -> + do_io_with_huge_message_queue(Config) + end. + +do_io_with_huge_message_queue(Config) -> + PrivDir = ?privdir(Config), + File = filename:join(PrivDir, "slask"), + {ok, F1} = file:open(File, [write]), + + {Time,ok} = timer:tc(fun() -> writes(1000, F1) end), + + [self() ! {msg,N} || N <- lists:seq(1, 500000)], + erlang:garbage_collect(), + {NewTime,ok} = timer:tc(fun() -> writes(1000, F1) end), + file:close(F1), + io:format("Time for empty message queue: ~p", [Time]), + io:format("Time for huge message queue: ~p", [NewTime]), + + IsCover = test_server:is_cover(), + case (NewTime+1) / (Time+1) of + Q when Q < 10; IsCover -> + ok; + Q -> + io:format("Q = ~p", [Q]), + ?t:fail() + end, + ok. + +writes(0, _) -> ok; +writes(N, F1) -> + file:write(F1, "hello\n"), + writes(N - 1, F1). diff --git a/lib/stdlib/test/ms_transform_SUITE.erl b/lib/stdlib/test/ms_transform_SUITE.erl index 4ec13ed472..1577caa80f 100644 --- a/lib/stdlib/test/ms_transform_SUITE.erl +++ b/lib/stdlib/test/ms_transform_SUITE.erl @@ -91,21 +91,23 @@ warnings(Config) when is_list(Config) -> " end)">>, ?line [{_,[{_,ms_transform,{?WARN_NUMBER_SHADOW,'A'}}]}] = compile_ww(Prog), - Prog2 = <<"C=5, " - "ets:fun2ms(fun({A,B} = C) " - " when is_integer(A) and (A+5 > B) -> " - " {A andalso B,C} " - " end)">>, - ?line [{_,[{_,ms_transform,{?WARN_NUMBER_SHADOW,'C'}}]}] = + Prog2 = <<"C = 5, + ets:fun2ms(fun ({A,B} = + C) when is_integer(A) and (A+5 > B) -> + {A andalso B,C} + end)">>, + [{_,[{3,ms_transform,{?WARN_NUMBER_SHADOW,'C'}}]}] = compile_ww(Prog2), Rec3 = <<"-record(a,{a,b,c,d=foppa}).">>, - Prog3 = <<"A=3,C=5, " - "ets:fun2ms(fun(#a{a = A, b = B} = C) " - " when is_integer(A) and (A+5 > B) -> " - " {A andalso B,C} " - " end)">>, - ?line [{_,[{_,ms_transform,{?WARN_NUMBER_SHADOW,'A'}}, - {_,ms_transform,{?WARN_NUMBER_SHADOW,'C'}}]}] = + Prog3 = <<"A = 3, + C = 5, + ets:fun2ms(fun (C + = #a{a = A, b = B}) + when is_integer(A) and (A+5 > B) -> + {A andalso B,C} + end)">>, + [{_,[{3,ms_transform,{?WARN_NUMBER_SHADOW,'C'}}, + {4,ms_transform,{?WARN_NUMBER_SHADOW,'A'}}]}] = compile_ww(Rec3,Prog3), Rec4 = <<"-record(a,{a,b,c,d=foppa}).">>, Prog4 = <<"A=3,C=5, " @@ -867,6 +869,7 @@ compile_ww(Records,Expr) -> "-include_lib(\"stdlib/include/ms_transform.hrl\").\n", "-export([tmp/0]).\n", Records/binary,"\n", + "-file(?FILE, 0). ", "tmp() ->\n", Expr/binary,".\n">>, FN=temp_name(), diff --git a/lib/stdlib/test/stdlib_SUITE.erl b/lib/stdlib/test/stdlib_SUITE.erl index 59821220b4..3d09bd27ff 100644 --- a/lib/stdlib/test/stdlib_SUITE.erl +++ b/lib/stdlib/test/stdlib_SUITE.erl @@ -78,17 +78,29 @@ appup_test(_Config) -> appup_tests(_App,{[],[]}) -> {skip,"no previous releases available"}; -appup_tests(App,{OkVsns,NokVsns}) -> +appup_tests(App,{OkVsns0,NokVsns}) -> application:load(App), {_,_,Vsn} = lists:keyfind(App,1,application:loaded_applications()), AppupFileName = atom_to_list(App) ++ ".appup", AppupFile = filename:join([code:lib_dir(App),ebin,AppupFileName]), {ok,[{Vsn,UpFrom,DownTo}=AppupScript]} = file:consult(AppupFile), ct:log("~p~n",[AppupScript]), - ct:log("Testing ok versions: ~p~n",[OkVsns]), + OkVsns = + case OkVsns0 -- [Vsn] of + OkVsns0 -> + OkVsns0; + Ok -> + ct:log("Current version, ~p, is same as in previous release.~n" + "Removing this from the list of ok versions.", + [Vsn]), + Ok + end, + ct:log("Testing that appup allows upgrade from these versions: ~p~n", + [OkVsns]), check_appup(OkVsns,UpFrom,{ok,[restart_new_emulator]}), check_appup(OkVsns,DownTo,{ok,[restart_new_emulator]}), - ct:log("Testing not ok versions: ~p~n",[NokVsns]), + ct:log("Testing that appup does not allow upgrade from these versions: ~p~n", + [NokVsns]), check_appup(NokVsns,UpFrom,error), check_appup(NokVsns,DownTo,error), ok. diff --git a/lib/stdlib/test/tar_SUITE.erl b/lib/stdlib/test/tar_SUITE.erl index 6349139925..9b6d65011e 100644 --- a/lib/stdlib/test/tar_SUITE.erl +++ b/lib/stdlib/test/tar_SUITE.erl @@ -654,6 +654,7 @@ open_add_close(Config) when is_list(Config) -> ?line ok = erl_tar:add(AD, FileOne, []), ?line ok = erl_tar:add(AD, FileTwo, "second file", []), ?line ok = erl_tar:add(AD, FileThree, [verbose]), + ?line ok = erl_tar:add(AD, FileThree, "chunked", [{chunks,11411},verbose]), ?line ok = erl_tar:add(AD, ADir, [verbose]), ?line ok = erl_tar:add(AD, AnotherDir, [verbose]), ?line ok = erl_tar:close(AD), @@ -661,7 +662,7 @@ open_add_close(Config) when is_list(Config) -> ?line ok = erl_tar:t(TarOne), ?line ok = erl_tar:tt(TarOne), - ?line {ok,[FileOne,"second file",FileThree,ADir,SomeContent]} = erl_tar:table(TarOne), + ?line {ok,[FileOne,"second file",FileThree,"chunked",ADir,SomeContent]} = erl_tar:table(TarOne), ?line delete_files(["oac_file","oac_small","oac_big",Dir,AnotherDir,ADir]), |