Merge tag 'OTP-19.0' into sverker/19/binary_to_atom-utf8-crash/ERL-474/OTP-14590

1 files changed, 145 insertions, 51 deletions

diff --git a/lib/stdlib/src/erl_tar.erl b/lib/stdlib/src/erl_tar.erl
index 4b654833ed..a383a0fc67 100644
--- a/lib/stdlib/src/erl_tar.erl
+++ b/lib/stdlib/src/erl_tar.erl
@@ -1,18 +1,19 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2013. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
 %% 
-%% The contents of this file are subject to the Erlang Public License,
-%% Version 1.1, (the "License"); you may not use this file except in
-%% compliance with the License. You should have received a copy of the
-%% Erlang Public License along with this software. If not, it can be
-%% retrieved online at http://www.erlang.org/.
-%% 
-%% Software distributed under the License is distributed on an "AS IS"
-%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-%% the License for the specific language governing rights and limitations
-%% under the License.
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
 %% 
 %% %CopyrightEnd%
 %%
@@ -22,7 +23,7 @@
 %% Purpose: Unix tar (tape archive) utility.
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
--export([create/2, create/3, extract/1, extract/2, table/1, table/2,
+-export([init/3, create/2, create/3, extract/1, extract/2, table/1, table/2,
 	 open/2, close/1, add/3, add/4,
 	 t/1, tt/1, format_error/1]).
 
@@ -30,10 +31,16 @@
 
 -record(add_opts,
 	{read_info,				% Fun to use for read file/link info.
+	 chunk_size = 0,  % For file reading when sending to sftp. 0=do not chunk
 	 verbose = false :: boolean()}).	% Verbose on/off.
 
 %% Opens a tar archive.
 
+init(UsrHandle, AccessMode, Fun) when is_function(Fun,2) ->   
+    {ok, {AccessMode,{tar_descriptor,UsrHandle,Fun}}}.
+
+%%%================================================================		   
+%%% The open function with friends is to keep the file and binary api of this module
 open(Name, Mode) ->
     case open_mode(Mode) of
 	{ok, Access, Raw, Opts} ->
@@ -45,31 +52,38 @@ open(Name, Mode) ->
 open1({binary,Bin}, read, _Raw, Opts) ->
     case file:open(Bin, [ram,binary,read]) of
 	{ok,File} ->
-	    case Opts of
-		[compressed] -> ram_file:uncompress(File);
-		[] -> ok
-	    end,
-	    {ok,{read,File}};
+            _ = [ram_file:uncompress(File) || Opts =:= [compressed]],
+	    init(File,read,file_fun());
 	Error ->
 	    Error
     end;
 open1({file, Fd}, read, _Raw, _Opts) ->
-    {ok, {read, Fd}};
+    init(Fd, read, file_fun());
 open1(Name, Access, Raw, Opts) ->
     case file:open(Name, Raw ++ [binary, Access|Opts]) of
 	{ok, File} ->
-	    {ok, {Access, File}};
+	    init(File, Access, file_fun());
 	{error, Reason} ->
 	    {error, {Name, Reason}}
     end.
 
+file_fun() ->
+    fun(write, {Fd,Data}) ->  file:write(Fd, Data);
+       (position, {Fd,Pos}) -> file:position(Fd, Pos);
+       (read2, {Fd,Size}) -> file:read(Fd,Size);
+       (close, Fd) -> file:close(Fd)
+    end.
+
+%%% End of file and binary api (except for open_mode/1 downwards
+%%%================================================================		   
+
 %% Closes a tar archive.
 
 close({read, File}) ->
-    ok = file:close(File);
+    ok = do_close(File);
 close({write, File}) ->
     PadResult = pad_file(File),
-    ok = file:close(File),
+    ok = do_close(File),
     PadResult;
 close(_) ->
     {error, einval}.
@@ -78,7 +92,6 @@ close(_) ->
 
 add(File, Name, Options) ->
     add(File, Name, Name, Options).
-
 add({write, File}, Name, NameInArchive, Options) ->
     Opts = #add_opts{read_info=fun(F) -> file:read_link_info(F) end},
     add1(File, Name, NameInArchive, add_opts(Options, Opts));
@@ -91,6 +104,8 @@ add_opts([dereference|T], Opts) ->
     add_opts(T, Opts#add_opts{read_info=fun(F) -> file:read_file_info(F) end});
 add_opts([verbose|T], Opts) ->
     add_opts(T, Opts#add_opts{verbose=true});
+add_opts([{chunks,N}|T], Opts) ->
+    add_opts(T, Opts#add_opts{chunk_size=N});
 add_opts([_|T], Opts) ->
     add_opts(T, Opts);
 add_opts([], Opts) ->
@@ -222,7 +237,7 @@ format_error(Atom) when is_atom(Atom) ->
 format_error(Term) ->
     lists:flatten(io_lib:format("~tp", [Term])).
 
-
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%
 %%%	Useful definitions (also start of implementation).
@@ -286,7 +301,7 @@ format_error(Term) ->
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 add1(TarFile, Bin, NameInArchive, Opts) when is_binary(Bin) ->
-    Now = calendar:now_to_local_time(now()),
+    Now = calendar:now_to_local_time(erlang:timestamp()),
     Info = #file_info{size = byte_size(Bin),
 		      type = regular,
 		      access = read_write,
@@ -324,16 +339,46 @@ add1(TarFile, Name, NameInArchive, Opts) ->
 	    {error, {Name, Reason}}
     end.
 
+add1(Tar, Name, Header, chunked, Options) ->
+    add_verbose(Options, "a ~ts [chunked ", [Name]),
+    try
+	ok = do_write(Tar, Header),
+	{ok,D} = file:open(Name, [read,binary]),
+	{ok,NumBytes} = add_read_write_chunks(D, Tar, Options#add_opts.chunk_size, 0, Options),
+	_ = file:close(D),
+	ok = do_write(Tar, padding(NumBytes,?record_size))
+    of 
+	ok ->
+	    add_verbose(Options, "~n", []),
+	    ok
+    catch
+	error:{badmatch,{error,Error}} ->
+	    add_verbose(Options, "~n", []),
+	    {error,{Name,Error}}
+    end;
 add1(Tar, Name, Header, Bin, Options) ->
     add_verbose(Options, "a ~ts~n", [Name]),
-    file:write(Tar, [Header, Bin, padding(byte_size(Bin), ?record_size)]).
+    do_write(Tar, [Header, Bin, padding(byte_size(Bin), ?record_size)]).
+
+add_read_write_chunks(D, Tar, ChunkSize, SumNumBytes, Options) ->
+    case file:read(D, ChunkSize) of
+	{ok,Bin} -> 
+	    ok = do_write(Tar, Bin),
+	    add_verbose(Options, ".", []),
+	    add_read_write_chunks(D, Tar, ChunkSize, SumNumBytes+byte_size(Bin), Options);
+	eof ->
+	    add_verbose(Options, "]", []),
+	    {ok,SumNumBytes};
+	Other ->
+	    Other
+    end.
 
 add_directory(TarFile, DirName, NameInArchive, Info, Options) ->
     case file:list_dir(DirName) of
 	{ok, []} ->
 	    add_verbose(Options, "a ~ts~n", [DirName]),
 	    Header = create_header(NameInArchive, Info),
-	    file:write(TarFile, Header);
+	    do_write(TarFile, Header);
 	{ok, Files} ->
 	    Add = fun (File) ->
 			  add1(TarFile,
@@ -384,7 +429,12 @@ to_octal(Int, Count, Result) ->
     to_octal(Int div 8, Count-1, [Int rem 8 + $0|Result]).
 
 to_string(Str0, Count) ->
-    Str = list_to_binary(Str0),
+    Str = case file:native_name_encoding() of
+	      utf8 ->
+		  unicode:characters_to_binary(Str0);
+	      latin1 ->
+		  list_to_binary(Str0)
+	  end,
     case byte_size(Str) of
 	Size when Size < Count ->
 	    [Str|zeroes(Count-Size)];
@@ -394,10 +444,18 @@ to_string(Str0, Count) ->
 %% Pads out end of file.
 
 pad_file(File) ->
-    {ok,Position} = file:position(File, {cur,0}),
-    %% There must be at least one empty record at the end of the file.
-    Zeros = zeroes(?block_size - (Position rem ?block_size)),
-    file:write(File, Zeros).
+    {ok,Position} = do_position(File, {cur,0}),
+    %% There must be at least two zero records at the end.
+    Fill = case ?block_size - (Position rem ?block_size) of
+	       Fill0 when Fill0 < 2*?record_size ->
+		   %% We need to another block here to ensure that there
+		   %% are at least two zero records at the end.
+		   Fill0 + ?block_size;
+	       Fill0 ->
+		   %% Large enough.
+		   Fill0
+	   end,
+    do_write(File, zeroes(Fill)).
 
 split_filename(Name) when length(Name) =< ?th_name_len ->
     {"", Name};
@@ -412,7 +470,7 @@ split_filename([Comp|Rest], Prefix, Suffix, Len) ->
 split_filename([], Prefix, Suffix, _) ->
     {filename:join(Prefix),filename:join(Suffix)}.
 
-
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%
 %%% 	Retrieving files from a tape archive.
@@ -475,27 +533,36 @@ read_opts([_|Rest], Opts) ->
 read_opts([], Opts) ->
     Opts.
 
+foldl_read({AccessMode,TD={tar_descriptor,_UsrHandle,_AccessFun}}, Fun, Accu, Opts) ->
+    case AccessMode of
+	read ->
+	    foldl_read0(TD, Fun, Accu, Opts);
+	_ ->
+	    {error,{read_mode_expected,AccessMode}}
+    end;
 foldl_read(TarName, Fun, Accu, Opts) ->
     case open(TarName, [read|Opts#read_opts.open_mode]) of
 	{ok, {read, File}} ->
-	    Result = 
-		case catch foldl_read1(Fun, Accu, File, Opts) of
-		    {'EXIT', Reason} ->
-			exit(Reason);
-		    {error, {Reason, Format, Args}} ->
-			read_verbose(Opts, Format, Args),
-			{error, Reason};
-		    {error, Reason} ->
-			{error, Reason};
-		    Ok ->
-			Ok
-		end,
-	    ok = file:close(File),
+	    Result = foldl_read0(File, Fun, Accu, Opts),
+	    ok = do_close(File),
 	    Result;
 	Error ->
 	    Error
     end.
 
+foldl_read0(File, Fun, Accu, Opts) ->
+    case catch foldl_read1(Fun, Accu, File, Opts) of
+	{'EXIT', Reason} ->
+	    exit(Reason);
+	{error, {Reason, Format, Args}} ->
+	    read_verbose(Opts, Format, Args),
+	    {error, Reason};
+	{error, Reason} ->
+	    {error, Reason};
+	Ok ->
+	    Ok
+    end.
+
 foldl_read1(Fun, Accu0, File, Opts) ->
     case get_header(File) of
 	eof ->
@@ -549,7 +616,7 @@ check_extract(Name, #read_opts{files=Files}) ->
     ordsets:is_element(Name, Files).
 
 get_header(File) ->
-    case file:read(File, ?record_size) of
+    case do_read(File, ?record_size) of
 	eof ->
 	    throw({error,eof});
 	{ok, Bin} when is_binary(Bin) ->
@@ -611,7 +678,22 @@ typeflag(Bin) ->
 %% Get the name of the file from the prefix and name fields of the
 %% tar header.
 
-get_name(Bin) ->
+get_name(Bin0) ->
+    List0 = get_name_raw(Bin0),
+    case file:native_name_encoding() of
+	utf8 ->
+	    Bin = list_to_binary(List0),
+	    case unicode:characters_to_list(Bin) of
+		{error,_,_} ->
+		    List0;
+		List when is_list(List) ->
+		    List
+	    end;
+	latin1 ->
+	    List0
+    end.
+
+get_name_raw(Bin) ->
     Name = from_string(Bin, ?th_name, ?th_name_len),
     case binary_to_list(Bin, ?th_prefix+1, ?th_prefix+1) of
 	[0] ->
@@ -665,7 +747,7 @@ get_element(File, #tar_header{size = 0}) ->
     skip_to_next(File),
     {ok,<<>>};
 get_element(File, #tar_header{size = Size}) ->
-    case file:read(File, Size) of
+    case do_read(File, Size) of
 	{ok,Bin}=Res when byte_size(Bin) =:= Size ->
 	    skip_to_next(File),
 	    Res;
@@ -855,7 +937,7 @@ skip(File, Size) ->
     %% Note: There is no point in handling failure to get the current position
     %% in the file.  If it doesn't work, something serious is wrong.
     Amount = ((Size + ?record_size - 1) div ?record_size) * ?record_size,
-    {ok,_} = file:position(File, {cur, Amount}),
+    {ok,_} = do_position(File, {cur, Amount}),
     ok.
 
 %% Skips to the next record in the file.
@@ -863,9 +945,9 @@ skip(File, Size) ->
 skip_to_next(File) ->
     %% Note: There is no point in handling failure to get the current position
     %% in the file.  If it doesn't work, something serious is wrong.
-    {ok, Position} = file:position(File, {cur, 0}),
+    {ok, Position} = do_position(File, {cur, 0}),
     NewPosition = ((Position + ?record_size - 1) div ?record_size) * ?record_size,
-    {ok,NewPosition} = file:position(File, NewPosition),
+    {ok,NewPosition} = do_position(File, NewPosition),
     ok.
 
 %% Prints the message on if the verbose option is given.
@@ -891,6 +973,9 @@ posix_to_erlang_time(Sec) ->
 read_file_and_info(Name, Opts) ->
     ReadInfo = Opts#add_opts.read_info,
     case ReadInfo(Name) of
+	{ok,Info} when Info#file_info.type =:= regular,
+		       Opts#add_opts.chunk_size>0 ->
+	    {ok,chunked,Info};
 	{ok,Info} when Info#file_info.type =:= regular ->
 	    case file:read_file(Name) of
 		{ok,Bin} ->
@@ -937,3 +1022,12 @@ open_mode([], Access, Raw, Opts) ->
     {ok, Access, Raw, Opts};
 open_mode(_, _, _, _) ->
     {error, einval}.
+
+%%%================================================================
+do_write({tar_descriptor,UsrHandle,Fun}, Data) -> Fun(write,{UsrHandle,Data}).
+
+do_position({tar_descriptor,UsrHandle,Fun}, Pos) -> Fun(position,{UsrHandle,Pos}).
+
+do_read({tar_descriptor,UsrHandle,Fun}, Len) -> Fun(read2,{UsrHandle,Len}).
+
+do_close({tar_descriptor,UsrHandle,Fun}) -> Fun(close,UsrHandle).