ssh: crypto framework for ssh_sftp:open_tar added

2 files changed, 613 insertions, 92 deletions

diff --git a/lib/ssh/src/ssh_sftp.erl b/lib/ssh/src/ssh_sftp.erl
index 3b80f5326c..613f8f25b2 100644
--- a/lib/ssh/src/ssh_sftp.erl
+++ b/lib/ssh/src/ssh_sftp.erl
@@ -69,6 +69,18 @@
 	  mode
 	 }).
 
+-record(bufinf,
+	{
+	  mode,			 % read | write  (=from or to buffer by user)
+	  crypto_state,
+	  crypto_fun,            % For encode or decode depending on the mode field
+	  size = 0,		 % # bytes "before" the current buffer for the postion call
+
+	  chunksize,		 % The size of the chunks to be sent or received
+	  enc_text_buf = <<>>,	 % Encrypted text
+	  plain_text_buf = <<>>	 % Decrypted text
+	}).
+	  
 -define(FILEOP_TIMEOUT, infinity).
 
 -define(NEXT_REQID(S),
@@ -164,24 +176,73 @@ open(Pid, File, Mode, FileOpTimeout) ->
 
 open_tar(Pid, File, Mode) ->
     open_tar(Pid, File, Mode, ?FILEOP_TIMEOUT).
-open_tar(Pid, File, Mode=[write], FileOpTimeout) ->
-    {ok,R} = open(Pid, File, Mode, FileOpTimeout),
-    erl_tar:init({Pid,R,FileOpTimeout}, write,
-		 fun(write, {{P,H,T},Data}) ->
-			 Bin = if is_list(Data) -> list_to_binary(Data);
-				  is_binary(Data) -> Data
-			       end,
-			 {ok,{_Window,Packet}} = send_window(P, T),
-			 write_file_loop(P, H, 0, Bin, size(Bin), Packet, T);
-		    (position, {{P,H,T},Pos}) -> 
-			 position(P, H, Pos, T);
-		    (close, {P,H,T}) -> 
-			 close(P, H, T)
-		 end);
-open_tar(_Pid, _File, Mode, _FileOpTimeout) ->
-    {error,{illegal_mode,Mode}}.
-
-
+open_tar(Pid, File, Mode, FileOpTimeout) ->
+    case {lists:member(write,Mode),
+	  lists:member(read,Mode),
+	  Mode -- [read,write]} of
+	{true,false,[]} ->
+	    {ok,Handle} = open(Pid, File, [write], FileOpTimeout),
+	    erl_tar:init(Pid, write,
+			 fun(write, {_,Data}) ->
+				 write_to_remote_tar(Pid, Handle, to_bin(Data), FileOpTimeout);
+			    (position, {_,Pos}) -> 
+				 position(Pid, Handle, Pos, FileOpTimeout);
+			    (close, _) -> 
+				 close(Pid, Handle, FileOpTimeout)
+			 end);
+	{true,false,[{crypto,{CryptoInitFun,CryptoEncryptFun,CryptoEndFun}}]} ->
+	    {ok,SftpHandle} = open(Pid, File, [write], FileOpTimeout),
+	    BI = #bufinf{mode = write,
+			 crypto_fun = CryptoEncryptFun},
+	    {ok,BufHandle} = open_buf(Pid, CryptoInitFun, BI, FileOpTimeout),
+	    erl_tar:init(Pid, write,
+			 fun(write, {_,Data}) ->
+				 write_buf(Pid, SftpHandle, BufHandle,  to_bin(Data), FileOpTimeout);
+			    (position, {_,Pos}) ->
+				 position_buf(Pid, SftpHandle, BufHandle, Pos, FileOpTimeout);
+			    (close, _) ->
+				 {ok,#bufinf{
+					plain_text_buf = PlainBuf0,
+					enc_text_buf = EncBuf0,
+					crypto_state = CState0
+				       }}  = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout),
+				 {ok,EncTextTail} = CryptoEndFun(PlainBuf0, CState0),
+				 EncTextBuf = <<EncBuf0/binary, EncTextTail/binary>>,
+				 case write(Pid, SftpHandle, EncTextBuf, FileOpTimeout) of
+				     ok ->
+					 call(Pid, {erase_bufinf,BufHandle}, FileOpTimeout),
+					 close(Pid, SftpHandle, FileOpTimeout);
+				     Other ->
+					 Other
+				 end
+			 end);
+	{false,true,[]} ->
+	    {ok,Handle} = open(Pid, File, [read,binary], FileOpTimeout),
+	    erl_tar:init(Pid, read,
+			 fun(read2, {_,Len}) ->
+				 read_repeat(Pid, Handle, Len, FileOpTimeout);
+			    (position, {_,Pos}) -> 
+				 position(Pid, Handle, Pos, FileOpTimeout);
+			    (close, _) -> 
+				 close(Pid, Handle, FileOpTimeout)
+			 end);
+	{false,true,[{crypto,{CryptoInitFun,CryptoDecryptFun}}]} ->
+	    {ok,SftpHandle} = open(Pid, File, [read,binary], FileOpTimeout),
+	    BI = #bufinf{mode = read,
+			 crypto_fun = CryptoDecryptFun},
+	    {ok,BufHandle} = open_buf(Pid, CryptoInitFun, BI, FileOpTimeout),
+	    erl_tar:init(Pid, read,
+			 fun(read2, {_,Len}) ->
+				 read_buf(Pid, SftpHandle, BufHandle, Len, FileOpTimeout);
+			    (position, {_,Pos}) -> 
+				 position_buf(Pid, SftpHandle, BufHandle, Pos, FileOpTimeout);
+			    (close, _) -> 
+				 call(Pid, {erase_bufinf,BufHandle}, FileOpTimeout),
+				 close(Pid, SftpHandle, FileOpTimeout)
+                         end);
+	_ ->
+	    {error,{illegal_mode,Mode}}
+    end.
 
 
 opendir(Pid, Path) ->
@@ -469,6 +530,15 @@ handle_cast(_,State) ->
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
+do_handle_call({get_bufinf,BufHandle}, _From, S=#state{inf=I0}) ->
+    {reply, dict:find(BufHandle,I0), S};
+
+do_handle_call({put_bufinf,BufHandle,B}, _From, S=#state{inf=I0}) ->
+    {reply, ok, S#state{inf=dict:store(BufHandle,B,I0)}};
+
+do_handle_call({erase_bufinf,BufHandle}, _From, S=#state{inf=I0}) ->
+    {reply, ok, S#state{inf=dict:erase(BufHandle,I0)}};
+
 do_handle_call({open, Async,FileName,Mode}, From, #state{xf = XF} = State) ->
     {Access,Flags,Attrs} = open_mode(XF#ssh_xfer.vsn, Mode),
     ReqID = State#state.req_id,
@@ -573,12 +643,7 @@ do_handle_call({read,Async,Handle,Length}, From, State) ->
 do_handle_call({pwrite,Async,Handle,At,Data0}, From, State) ->
     case lseek_position(Handle, At, State) of
 	{ok,Offset} ->
-	    Data = if 
-		       is_binary(Data0) -> 
-			   Data0;
-		       is_list(Data0) -> 
-			   list_to_binary(Data0)
-		   end,
+	    Data = to_bin(Data0),
 	    ReqID = State#state.req_id,
 	    Size = size(Data),
 	    ssh_xfer:write(?XF(State),ReqID,Handle,Offset,Data),
@@ -591,12 +656,7 @@ do_handle_call({pwrite,Async,Handle,At,Data0}, From, State) ->
 do_handle_call({write,Async,Handle,Data0}, From, State) ->
     case lseek_position(Handle, cur, State) of
 	{ok,Offset} ->
-	    Data = if 
-		       is_binary(Data0) ->
-			   Data0;
-		       is_list(Data0) ->
-			   list_to_binary(Data0)
-		   end,
+	    Data = to_bin(Data0),
 	    ReqID = State#state.req_id,
 	    Size = size(Data),
 	    ssh_xfer:write(?XF(State),ReqID,Handle,Offset,Data),
@@ -1148,5 +1208,207 @@ lseek_pos({eof, Offset}, _CurOffset, CurSize)
     end;
 lseek_pos(_, _, _) ->
     {error, einval}. 
- 
 
+%%%================================================================
+%%%
+to_bin(Data) when is_list(Data) -> list_to_binary(Data);
+to_bin(Data) when is_binary(Data) -> Data.
+
+
+read_repeat(Pid, Handle, Len, FileOpTimeout) ->
+    {ok,{_WindowSz,PacketSz}} = recv_window(Pid, FileOpTimeout),
+    read_rpt(Pid, Handle, Len, PacketSz, FileOpTimeout, <<>>).
+
+read_rpt(Pid, Handle, WantedLen, PacketSz, FileOpTimeout, Acc) when WantedLen > 0 ->
+    case read(Pid, Handle, min(WantedLen,PacketSz), FileOpTimeout) of
+	{ok, Data}  ->
+	    read_rpt(Pid, Handle, WantedLen-size(Data), PacketSz, FileOpTimeout, <<Acc/binary, Data/binary>>);
+	eof ->
+	    {ok, Acc};
+	Error ->
+	    Error
+    end;
+read_rpt(_Pid, _Handle, WantedLen, _PacketSz, _FileOpTimeout, Acc) when WantedLen >= 0 ->
+    {ok,Acc}.
+
+
+write_to_remote_tar(_Pid, _SftpHandle, <<>>, _FileOpTimeout) ->
+    ok;
+write_to_remote_tar(Pid, SftpHandle, Bin, FileOpTimeout) ->
+    {ok,{_Window,Packet}} = send_window(Pid, FileOpTimeout),
+    write_file_loop(Pid, SftpHandle, 0, Bin, size(Bin), Packet, FileOpTimeout).
+
+position_buf(Pid, SftpHandle, BufHandle, Pos, FileOpTimeout) ->
+    {ok,#bufinf{mode = Mode,
+		plain_text_buf = Buf0,
+		size = Size}} = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout),
+    case Pos of
+	{cur,0} when Mode==write ->
+	    {ok,Size+size(Buf0)};
+	
+	{cur,0} when Mode==read ->
+	    {ok,Size};
+	
+	_ when Mode==read, is_integer(Pos) ->
+	    Skip = Pos-Size,
+	    if 
+		Skip < 0 ->
+		    {error, cannot_rewind};
+		Skip == 0 ->
+		    %% Optimization
+		    {ok,Pos};
+		Skip > 0 ->
+		    case read_buf(Pid, SftpHandle, BufHandle, Skip, FileOpTimeout) of
+			%% A bit innefficient to fetch the bufinf again, but there are lots of
+			%% other more important optimizations waiting....
+			{ok,_} ->
+			    {ok,Pos};
+			 Other ->
+			    Other
+		    end
+	    end;
+
+	_ ->
+	    {error,{not_yet_implemented,{pos,Pos}}}
+      end.
+
+read_buf(Pid, SftpHandle, BufHandle, WantedLen, FileOpTimeout) ->
+    {ok,{_Window,Packet}} = send_window(Pid, FileOpTimeout),
+    {ok,B0}  = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout),
+    case do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, B0) of
+	{ok,ResultBin,B} ->
+	    call(Pid, {put_bufinf,BufHandle,B}, FileOpTimeout),
+	    {ok,ResultBin};
+	{error,Error} ->
+	    {error,Error};
+	{eof,B} ->
+	    call(Pid, {put_bufinf,BufHandle,B}, FileOpTimeout),
+	    eof
+      end.
+
+do_the_read_buf(_Pid, _SftpHandle, WantedLen, _Packet, _FileOpTimeout, 
+		B=#bufinf{plain_text_buf=PlainBuf0,
+			  size = Size})
+    when size(PlainBuf0) >= WantedLen ->
+    %% We already have the wanted number of bytes decoded and ready!
+    <<ResultBin:WantedLen/binary, PlainBuf/binary>> = PlainBuf0,
+    {ok,ResultBin,B#bufinf{plain_text_buf=PlainBuf,
+			   size = Size + WantedLen}};
+
+do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, 
+		B0=#bufinf{plain_text_buf = PlainBuf0,
+			   enc_text_buf = EncBuf0,
+			   chunksize = undefined
+			  })
+  when size(EncBuf0) > 0 ->
+    %% We have (at least) one decodable byte waiting for decodeing.
+    {ok,DecodedBin,B} = apply_crypto(EncBuf0, B0),
+    do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, 
+		    B#bufinf{plain_text_buf = <<PlainBuf0/binary, DecodedBin/binary>>,
+			     enc_text_buf = <<>>
+			    });
+    
+do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, 
+		B0=#bufinf{plain_text_buf = PlainBuf0,
+			   enc_text_buf = EncBuf0,
+			   chunksize = ChunkSize0
+			  })
+  when size(EncBuf0) >= ChunkSize0 ->
+    %% We have (at least) one chunk of decodable bytes waiting for decodeing.
+    <<ToDecode:ChunkSize0/binary, EncBuf/binary>> = EncBuf0,
+    {ok,DecodedBin,B} = apply_crypto(ToDecode, B0),
+    do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, 
+		    B#bufinf{plain_text_buf = <<PlainBuf0/binary, DecodedBin/binary>>,
+			     enc_text_buf = EncBuf
+			    });
+    
+do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, B=#bufinf{enc_text_buf = EncBuf0}) ->
+    %% We must read more bytes and append to the buffer of encoded bytes.
+    case read(Pid, SftpHandle, Packet, FileOpTimeout) of
+	{ok,EncryptedBin} ->
+	    do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout,
+			    B#bufinf{enc_text_buf = <<EncBuf0/binary, EncryptedBin/binary>>});
+	eof ->
+	    {eof,B};
+	Other ->
+	    Other
+    end.
+
+
+write_buf(Pid, SftpHandle, BufHandle, PlainBin, FileOpTimeout) ->
+    {ok,{_Window,Packet}} = send_window(Pid, FileOpTimeout),
+    {ok,B0=#bufinf{plain_text_buf=PTB}}  = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout),
+    case do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, 
+			  B0#bufinf{plain_text_buf = <<PTB/binary,PlainBin/binary>>}) of
+	{ok, B} ->
+	    call(Pid, {put_bufinf,BufHandle,B}, FileOpTimeout),
+	    ok;
+	{error,Error} ->
+	    {error,Error}
+    end.
+
+do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, 
+		 B=#bufinf{enc_text_buf = EncBuf0,
+			   size = Size})
+  when size(EncBuf0) >= Packet ->
+    <<BinToWrite:Packet/binary, EncBuf/binary>> = EncBuf0,
+    case write(Pid, SftpHandle, BinToWrite, FileOpTimeout) of
+	ok ->
+	    do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout,
+			     B#bufinf{enc_text_buf = EncBuf,
+				      size = Size + Packet});
+	Other ->
+	    Other
+    end;
+
+do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout,
+		 B0=#bufinf{plain_text_buf = PlainBuf0,
+			    enc_text_buf = EncBuf0,
+			    chunksize = undefined})
+  when size(PlainBuf0) > 0 ->
+     {ok,EncodedBin,B} = apply_crypto(PlainBuf0, B0),
+     do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout,
+		     B#bufinf{plain_text_buf = <<>>,
+			      enc_text_buf = <<EncBuf0/binary, EncodedBin/binary>>});
+
+do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout,
+		 B0=#bufinf{plain_text_buf = PlainBuf0,
+			    enc_text_buf = EncBuf0,
+			    chunksize = ChunkSize0
+			   })
+  when size(PlainBuf0) >= ChunkSize0 ->
+    <<ToEncode:ChunkSize0/binary, PlainBuf/binary>> = PlainBuf0,
+    {ok,EncodedBin,B} = apply_crypto(ToEncode, B0),
+    do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout,
+		     B#bufinf{plain_text_buf = PlainBuf,
+			      enc_text_buf = <<EncBuf0/binary, EncodedBin/binary>>});
+
+do_the_write_buf(_Pid, _SftpHandle, _Packet, _FileOpTimeout, B) ->
+    {ok,B}.
+
+apply_crypto(In, B=#bufinf{crypto_state = CState0,
+			   crypto_fun = F}) ->
+    case F(In,CState0) of
+	{ok,EncodedBin,CState} -> 
+	    {ok, EncodedBin, B#bufinf{crypto_state=CState}};
+	{ok,EncodedBin,CState,ChunkSize} -> 
+	    {ok, EncodedBin, B#bufinf{crypto_state=CState,
+				      chunksize=ChunkSize}}
+    end.
+
+open_buf(Pid, CryptoInitFun, BufInfo0, FileOpTimeout) ->
+    case CryptoInitFun() of
+	{ok,CryptoState} ->
+	    open_buf1(Pid, BufInfo0, FileOpTimeout, CryptoState, undefined);
+	{ok,CryptoState,ChunkSize} ->
+	    open_buf1(Pid, BufInfo0, FileOpTimeout, CryptoState, ChunkSize);
+	Other ->
+	    Other
+    end.
+
+open_buf1(Pid, BufInfo0, FileOpTimeout, CryptoState, ChunkSize) ->
+    BufInfo = BufInfo0#bufinf{crypto_state = CryptoState,
+			      chunksize = ChunkSize},
+    BufHandle = make_ref(),
+    call(Pid, {put_bufinf,BufHandle,BufInfo}, FileOpTimeout),
+    {ok,BufHandle}.
diff --git a/lib/ssh/test/ssh_sftp_SUITE.erl b/lib/ssh/test/ssh_sftp_SUITE.erl
index 559fa721fd..cb74a27638 100644
--- a/lib/ssh/test/ssh_sftp_SUITE.erl
+++ b/lib/ssh/test/ssh_sftp_SUITE.erl
@@ -65,19 +65,25 @@ end_per_suite(Config) ->
 %%--------------------------------------------------------------------
 groups() -> 
     [{erlang_server, [], [open_close_file, open_close_dir, read_file, read_dir,
-			  write_file, write_big_file, rename_file, mk_rm_dir, remove_file, links,
+			  write_file, write_big_file, sftp_read_big_file,
+			  rename_file, mk_rm_dir, remove_file, links,
 			  retrieve_attributes, set_attributes, async_read,
 			  async_write, position, pos_read, pos_write, version_option,
-			  {group,remote_tar_write}
-			 ]},
+			  {group,remote_tar}]},
+
      {openssh_server, [], [open_close_file, open_close_dir, read_file, read_dir,
-			   write_file, write_big_file, rename_file, mk_rm_dir, remove_file, links,
+			   write_file, write_big_file, sftp_read_big_file,
+			   rename_file, mk_rm_dir, remove_file, links,
 			   retrieve_attributes, set_attributes, async_read,
 			   async_write, position, pos_read, pos_write,
-			   {group,remote_tar_write}]},
-
-     {remote_tar_write, [], [create_empty_tar, files_to_tar, big_file_to_tar, files_chunked_to_tar,
-			     directory_to_tar, binaries_to_tar]}
+			   {group,remote_tar}]},
+
+     {remote_tar, [], [create_empty_tar, files_to_tar, big_file_to_tar, files_chunked_to_tar,
+		       directory_to_tar, binaries_to_tar, null_crypto_tar, 
+		       simple_crypto_tar_small, simple_crypto_tar_big,
+		       read_tar, read_null_crypto_tar, read_crypto_tar, 
+		       aes_cbc256_crypto_tar, aes_ctr_stream_crypto_tar
+		      ]}
     ].
      
 
@@ -104,7 +110,7 @@ init_per_group(openssh_server, Config) ->
 	    {skip, "No openssh server"} 
     end;
 
-init_per_group(remote_tar_write, Config) ->
+init_per_group(remote_tar, Config) ->
     {Host,Port} =  ?config(peer, Config),
     ct:log("Server (~p) at ~p:~p",[?config(group,Config),Host,Port]),
     {ok, Connection} =
@@ -120,7 +126,7 @@ init_per_group(remote_tar_write, Config) ->
 			    [{user_interaction, false},
 			     {silently_accept_hosts, true}])
 	end,
-    [{remote_tar_write, true}, 
+    [{remote_tar, true}, 
      {connection, Connection} | Config].
 
 end_per_group(erlang_server, Config) ->
@@ -187,16 +193,12 @@ init_per_testcase(Case, Config0) ->
 		[{sftp, Sftp}, {watchdog, Dog} | Config2]
 	end,
 
-    case catch ?config(remote_tar_write,Config) of
+    case catch ?config(remote_tar,Config) of
 	%% The 'catch' is for the case of Config={skip,...}
 	true ->
-	    %% Provide a tar Handle *independent* of the sftp-channel already opened!
-	    %% This Handle will be closed (as well as ChannelPid2) in the testcase
-	    {ok,ChannelPid2} = 
-		ssh_sftp:start_channel(?config(connection,Config)),
-	    {ok,Handle} =
-		ssh_sftp:open_tar(ChannelPid2, fnp(?tar_file_name,Config), [write]),
-	    [{handle,Handle} | Config];
+	    %% Provide a ChannelPid independent of the sftp-channel already opened.
+	    {ok,ChPid2} = ssh_sftp:start_channel(?config(connection,Config)),
+	    [{channel_pid2,ChPid2} | Config];
 	_ ->
 	    Config
     end.
@@ -214,6 +216,7 @@ end_per_testcase(_, Config) ->
 end_per_testcase(Config) ->
     {Sftp, Connection} = ?config(sftp, Config),
     ssh_sftp:stop_channel(Sftp),
+    catch ssh_sftp:stop_channel(?config(channel_pid2, Config)),
     ssh:close(Connection).
 
 %%--------------------------------------------------------------------
@@ -258,6 +261,7 @@ read_file(Config) when is_list(Config) ->
     FileName = filename:join(PrivDir, "sftp.txt"),
     {Sftp, _} = ?config(sftp, Config),
     {ok, Data} = ssh_sftp:read_file(Sftp, FileName),
+    {ok, Data} = ssh_sftp:read_file(Sftp, FileName),
     {ok, Data} = file:read_file(FileName).
 
 %%--------------------------------------------------------------------
@@ -294,6 +298,19 @@ write_big_file(Config) when is_list(Config) ->
     {ok, Data} = file:read_file(FileName).
 
 %%--------------------------------------------------------------------
+sftp_read_big_file() ->
+    [{doc, "Test API function read_file/2 with big data"}].
+sftp_read_big_file(Config) when is_list(Config) ->
+    PrivDir =  ?config(priv_dir, Config),
+    FileName = filename:join(PrivDir, "sftp.txt"),
+    {Sftp, _} = ?config(sftp, Config),
+
+    Data = list_to_binary(lists:duplicate(750000,"a")),
+    ct:log("Data size to write is ~p bytes",[size(Data)]),
+    ssh_sftp:write_file(Sftp, FileName, [Data]),
+    {ok, Data} = ssh_sftp:read_file(Sftp, FileName).
+
+%%--------------------------------------------------------------------
 remove_file() ->
     [{doc,"Test API function delete/2"}].
 remove_file(Config) when is_list(Config) ->
@@ -527,53 +544,247 @@ version_option(Config) when is_list(Config) ->
 
 %%--------------------------------------------------------------------
 create_empty_tar(Config) ->
-    {ChPid,_} = ?config(sftp,Config),
-    Handle = ?config(handle,Config),
+    ChPid2 = ?config(channel_pid2, Config),
+    {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]),
     erl_tar:close(Handle),
+    {ChPid,_} = ?config(sftp,Config),
     {ok, #file_info{type=regular}} =
 	ssh_sftp:read_file_info(ChPid,fnp(?tar_file_name,Config)).
-    
+
 %%--------------------------------------------------------------------
 files_to_tar(Config) ->
-    Handle = ?config(handle,Config),
-    ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", []),
-    ok = erl_tar:add(Handle, fn("f2.txt",Config), "f2.txt", []),
+    ChPid2 = ?config(channel_pid2, Config),
+    {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]),
+    ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose]),
+    ok = erl_tar:add(Handle, fn("f2.txt",Config), "f2.txt", [verbose]),
     ok = erl_tar:close(Handle),
     chk_tar(["f1.txt", "f2.txt"], Config).
 
-
 %%--------------------------------------------------------------------
 big_file_to_tar(Config) ->
-    Handle = ?config(handle,Config),
-    ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", []),
+    ChPid2 = ?config(channel_pid2, Config),
+    {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), 
+    ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [verbose]),
     ok = erl_tar:close(Handle),
     chk_tar(["big.txt"], Config).
 
 
 %%--------------------------------------------------------------------
 files_chunked_to_tar(Config) ->
-    Handle = ?config(handle,Config),
-    ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [{chunks,2}]),
-    ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [{chunks,15000}]),
+    ChPid2 = ?config(channel_pid2, Config),
+    {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]),
+    ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]),
     ok = erl_tar:close(Handle),
-    chk_tar(["f1.txt", "big.txt"], Config).
+    chk_tar(["f1.txt"], Config).
 
 %%--------------------------------------------------------------------
 directory_to_tar(Config) ->
-    Handle = ?config(handle,Config),
-    ok = erl_tar:add(Handle, fn("d1",Config), "d1", []),
+    ChPid2 = ?config(channel_pid2, Config),
+    {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]),
+    ok = erl_tar:add(Handle, fn("d1",Config), "d1", [verbose]),
     ok = erl_tar:close(Handle),
-    chk_tar(["d1/f1", "d1/f2"], Config).
+    chk_tar(["d1"], Config).
     
 %%--------------------------------------------------------------------
 binaries_to_tar(Config) ->
-    Handle = ?config(handle,Config),
+    ChPid2 = ?config(channel_pid2, Config),
+    {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]),
     Bin = <<"A binary">>,
-    ok = erl_tar:add(Handle, Bin, "b1", []),
+    ok = erl_tar:add(Handle, Bin, "b1", [verbose]),
     ok = erl_tar:close(Handle),
     chk_tar([{"b1",Bin}], Config).
 
 %%--------------------------------------------------------------------
+null_crypto_tar(Config) ->
+    ChPid2 = ?config(channel_pid2, Config),
+    Cinit = fun() -> {ok, no_state, _SendSize=5} end,
+    Cenc = fun(Bin,CState) -> {ok,Bin,CState,_SendSize=5} end,
+    Cend = fun(Bin,_CState) -> {ok,Bin} end,
+    C = {Cinit,Cenc,Cend},
+    {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,C}]),
+    Bin = <<"A binary">>,
+    ok = erl_tar:add(Handle, Bin, "b1", [verbose]),
+    ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]),
+    ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [verbose,{chunks,15000}]),
+    ok = erl_tar:close(Handle),
+    chk_tar([{"b1",Bin}, "f1.txt", "big.txt"], Config).
+
+%%--------------------------------------------------------------------
+simple_crypto_tar_small(Config) ->
+    ChPid2 = ?config(channel_pid2, Config),
+    Cinit = fun() -> {ok, no_state, _Size=6} end,
+    Cenc = fun(Bin,CState) -> {ok,stuff(Bin),CState,_SendSize=5} end,
+    Cdec = fun(Bin,CState) -> {ok,unstuff(Bin),CState,_Size=4} end,
+    Cend = fun(Bin,_CState) -> {ok,stuff(Bin)} end,
+    C = {Cinit,Cenc,Cend},
+    {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,C}]),
+    Bin = <<"A binary">>,
+    ok = erl_tar:add(Handle, Bin, "b1", [verbose]),
+    ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]),
+    ok = erl_tar:close(Handle),
+    chk_tar([{"b1",Bin}, "f1.txt"], Config, [{crypto,{Cinit,Cdec}}]).
+
+%%--------------------------------------------------------------------
+simple_crypto_tar_big(Config) ->
+    ChPid2 = ?config(channel_pid2, Config),
+    Cinit = fun() -> {ok, no_state, _SendSize=6} end,
+    Cenc = fun(Bin,CState) -> {ok,stuff(Bin),CState,_SendSize=5} end,
+    Cdec = fun(Bin,CState) -> {ok,unstuff(Bin),CState,_SendSize=4} end,
+    Cend = fun(Bin,_CState) -> {ok,stuff(Bin)} end,
+    C = {Cinit,Cenc,Cend},
+    {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,C}]),
+    Bin = <<"A binary">>,
+    ok = erl_tar:add(Handle, Bin, "b1", [verbose]),
+    ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]),
+    ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [verbose,{chunks,15000}]),
+    ok = erl_tar:close(Handle),
+    chk_tar([{"b1",Bin}, "f1.txt", "big.txt"], Config, [{crypto,{Cinit,Cdec}}]).
+
+stuff(Bin) -> << <<C,C>> || <<C>> <= Bin >>.
+    
+unstuff(Bin) -> << <<C>> || <<C,C>> <= Bin >>.
+     
+%%--------------------------------------------------------------------
+read_tar(Config) ->
+    ChPid2 = ?config(channel_pid2, Config),
+    NameBins = lists:sort(
+		 [{"b1",<<"A binary">>},
+		  {"b2",list_to_binary(lists:duplicate(750000,"a"))}
+		 ]),
+    {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]),
+    [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose])
+     ||	{Name,Bin} <- NameBins],
+    ok = erl_tar:close(HandleWrite),
+
+    chk_tar(NameBins, Config).
+
+%%--------------------------------------------------------------------
+read_null_crypto_tar(Config) ->
+    ChPid2 = ?config(channel_pid2, Config),
+    NameBins = lists:sort(
+		 [{"b1",<<"A binary">>},
+		  {"b2",list_to_binary(lists:duplicate(750000,"a"))}
+		 ]),
+    Cinitw = fun() -> {ok, no_state, _SendSize=5} end,
+    Cinitr = fun() -> {ok, no_state, _FetchSize=42} end,
+    Cenc = fun(Bin,CState) -> {ok,Bin,CState,_SendSize=42*42} end,
+    Cdec = fun(Bin,CState) -> {ok,Bin,CState,_FetchSize=19} end,
+    Cendw = fun(Bin,_CState) -> {ok,Bin} end,
+    Cw = {Cinitw,Cenc,Cendw},
+    Cr = {Cinitr,Cdec},
+
+    {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]),
+    [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose])
+     ||	{Name,Bin} <- NameBins],
+    ok = erl_tar:close(HandleWrite),
+
+    chk_tar(NameBins, Config, [{crypto,Cr}]).
+
+%%--------------------------------------------------------------------
+read_crypto_tar(Config) ->
+    ChPid2 = ?config(channel_pid2, Config),
+    NameBins = lists:sort(
+		 [{"b1",<<"A binary">>},
+		  {"b2",list_to_binary(lists:duplicate(750000,"a"))}
+		 ]),
+    Cinitw = fun() -> {ok, no_state, _SendSize=5} end,
+    Cinitr = fun() -> {ok, no_state, _FetchSize=42} end,
+
+    Cenc = fun(Bin,CState) -> {ok,stuff(Bin),CState,_SendSize=42*42} end,
+    Cdec = fun(Bin,CState) -> {ok,unstuff(Bin),CState,_FetchSize=120} end,
+    Cendw = fun(Bin,_CState) -> {ok,stuff(Bin)} end,
+    Cw = {Cinitw,Cenc,Cendw},
+    Cr = {Cinitr,Cdec},
+
+    {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]),
+    [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose])
+     ||	{Name,Bin} <- NameBins],
+    ok = erl_tar:close(HandleWrite),
+
+    chk_tar(NameBins, Config, [{crypto,Cr}]).
+
+%%--------------------------------------------------------------------
+aes_cbc256_crypto_tar(Config) ->
+    ChPid2 = ?config(channel_pid2, Config),
+    NameBins = lists:sort(
+		 [{"b1",<<"A binary">>},
+		  {"b2",list_to_binary(lists:duplicate(750000,"a"))},
+		  {"d1",fn("d1",Config)}  % Dir
+		 ]),
+    Key = <<"This is a 256 bit key. Boring...">>,
+    Ivec0 = crypto:rand_bytes(16),
+    DataSize = 1024,  % data_size rem 16 = 0 for aes_cbc
+
+    Cinitw = fun() -> {ok, Ivec0, DataSize} end,
+    Cinitr = fun() -> {ok, Ivec0, DataSize} end,
+
+    Cenc = fun(PlainBin,Ivec) -> 
+		   CipherBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, PlainBin),
+		   {ok, CipherBin, crypto:next_iv(aes_cbc,CipherBin), DataSize}
+	   end,
+    Cdec = fun(CipherBin,Ivec) ->
+		   PlainBin = crypto:block_decrypt(aes_cbc256, Key, Ivec, CipherBin),
+		   {ok, PlainBin, crypto:next_iv(aes_cbc,CipherBin), DataSize}
+	   end,
+
+    Cendw = fun(PlainBin, _) when PlainBin == <<>> -> {ok, <<>>};
+	       (PlainBin, Ivec) ->
+		    CipherBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, 
+						     pad(16,PlainBin)), %% Last chunk
+		    {ok, CipherBin} 
+	    end,
+
+    Cw = {Cinitw,Cenc,Cendw},
+    {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]),
+    [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) || {Name,Bin} <- NameBins],
+    ok = erl_tar:close(HandleWrite),
+
+    Cr = {Cinitr,Cdec},
+    chk_tar(NameBins, Config, [{crypto,Cr}]).
+
+
+pad(BlockSize, Bin) ->
+    PadSize = (BlockSize - (size(Bin) rem BlockSize)) rem BlockSize,
+    list_to_binary( lists:duplicate(PadSize,0) ).
+
+%%--------------------------------------------------------------------
+aes_ctr_stream_crypto_tar(Config) ->
+    ChPid2 = ?config(channel_pid2, Config),
+    NameBins = lists:sort(
+		 [{"b1",<<"A binary">>},
+		  {"b2",list_to_binary(lists:duplicate(750000,"a"))},
+		  {"d1",fn("d1",Config)}  % Dir
+		 ]),
+    Key = <<"This is a 256 bit key. Boring...">>,
+    Ivec0 = crypto:rand_bytes(16),
+
+    Cinitw = Cinitr = fun() -> {ok, crypto:stream_init(aes_ctr,Key,Ivec0)} end,
+
+    Cenc = fun(PlainBin,State) -> 
+		   {NewState,CipherBin} = crypto:stream_encrypt(State, PlainBin),
+		   {ok, CipherBin, NewState}
+	   end,
+    Cdec = fun(CipherBin,State) ->
+		   {NewState,PlainBin} = crypto:stream_decrypt(State, CipherBin),
+		   {ok, PlainBin, NewState}
+	   end,
+
+    Cendw = fun(PlainBin, _) when PlainBin == <<>> -> {ok, <<>>};
+	       (PlainBin, Ivec) ->
+		    CipherBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, 
+						     pad(16,PlainBin)), %% Last chunk
+		    {ok, CipherBin} 
+	    end,
+
+    Cw = {Cinitw,Cenc,Cendw},
+    {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]),
+    [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) || {Name,Bin} <- NameBins],
+    ok = erl_tar:close(HandleWrite),
+
+    Cr = {Cinitr,Cdec},
+    chk_tar(NameBins, Config, [{crypto,Cr}]).
+
+%%--------------------------------------------------------------------
 %% Internal functions ------------------------------------------------
 %%--------------------------------------------------------------------
 prep(Config) ->
@@ -597,34 +808,82 @@ prep(Config) ->
 			      FileInfo#file_info{mode = Mode}).
 
 
+
 chk_tar(Items, Config) ->
-    %% FIXME: ought to check that no more than expected is present...
+    chk_tar(Items, Config, []).
+
+chk_tar(Items, Config, Opts) ->
+    chk_tar(Items, fnp(?tar_file_name,Config), Config, Opts).
+
+chk_tar(Items, TarFileName, Config, Opts)  when is_list(Opts) ->
+    tar_size(TarFileName, Config),
     {ChPid,_} = ?config(sftp,Config),
-    ok = file:set_cwd(?config(priv_dir,Config)),
-    file:make_dir("tar_chk"), % May already exist
-    ok = file:set_cwd("tar_chk"),
-    {ok,Data} = ssh_sftp:read_file(ChPid, fnp(?tar_file_name,Config)),
-    ok = file:write_file(?tar_file_name, Data),
-    os:cmd("tar xf "++?tar_file_name),
-    lists:foreach(fun(Item) -> chk_contents(Item,Config) end,
-		  Items).
-
-	      
-chk_contents({Name,ExpectBin}, _Config) -> 
-    case file:read_file(Name) of
-	{ok,ExpectBin} ->
-	    ok;
-	{ok,OtherBin} ->
-	    ct:log("File: ~p~n   Got: ~p~nExpect: ~p",[Name,OtherBin,ExpectBin]),
-	    ct:fail("Bad contents in file ~p",[Name]);
-	Other ->
-	    ct:log("File: ~p~nOther: ~p",[Name,Other]),
-	    ct:fail("Error reading of file ~p",[Name])
-    end;
-chk_contents(Name, Config) -> 
-    {ok,Bin} = file:read_file(fn(Name,Config)),
-    chk_contents({Name,Bin}, Config).
+    {ok,HandleRead} = ssh_sftp:open_tar(ChPid, TarFileName, [read|Opts]),
+    {ok,NameValueList} = erl_tar:extract(HandleRead,[memory,verbose]),
+    ok = erl_tar:close(HandleRead),
+    case {lists:sort(expand_items(Items,Config)), lists:sort(NameValueList)} of
+	{L,L} -> 
+	    true;
+	{Expect,Actual} -> 
+	    ct:log("Expect: ~p",[Expect]), ct:log("Actual: ~p",[Actual]),
+	    case erl_tar:table(TarFileName) of
+		{ok,Names} -> ct:log("names: ~p",[Names]);
+		Other -> ct:log("~p",[Other])
+	    end,
+	    ct:log("~s",[analyze_report(Expect, Actual)]),
+	    ct:fail(bad_tar_contents)
+    end.
 
+analyze_report([E={NameE,BinE}|Es], [A={NameA,BinA}|As]) ->
+    if
+	NameE == NameA,
+	BinE =/= BinA-> 
+	    [["Component ",NameE," differs. \n  Expected: ",BinE,"\n  Actual: ",BinA,"\n\n"]
+	     | analyze_report(Es,As)];
+
+	NameE < NameA ->
+	    [["Component ",NameE," is missing.\n\n"]
+	     | analyze_report(Es,[A|As])];
+	
+	NameE > NameA ->
+	    [["Component ",NameA," is not expected.\n\n"]
+	     | analyze_report([E|Es],As)];
+	true ->
+	    analyze_report(Es, As)
+    end;
+analyze_report([{NameE,_BinE}|Es], []) ->
+    [["Component ",NameE," missing.\n\n"] | analyze_report(Es,[])];
+analyze_report([], [{NameA,_BinA}|As]) ->
+    [["Component ",NameA," not expected.\n\n"] | analyze_report([],As)];
+analyze_report([], []) ->
+    "".
+	
+tar_size(TarFileName, Config) ->
+    {ChPid,_} = ?config(sftp,Config),
+    {ok,Data} = ssh_sftp:read_file(ChPid, TarFileName),
+    io:format('Tar file ~p is~n ~p bytes.~n',[TarFileName, size(Data)]).
+
+expand_items(Items, Config) ->    
+    lists:flatten(
+      [case Item of
+	   {_Name,Bin} when is_binary(Bin) -> 
+	       Item;
+	   {Name,FileName} when is_list(FileName) ->
+	       read_item_contents(Name, fn(FileName,Config));
+	   FileName  when is_list(FileName) ->
+	       read_item_contents(FileName, fn(FileName,Config))
+       end || Item <- Items]).
+
+read_item_contents(ItemName, FileName) ->
+    case file:read_file(FileName) of
+	{ok,Bin} ->
+	    {ItemName, Bin};
+	{error,eisdir} ->
+	    {ok,FileNames} = file:list_dir(FileName),
+	    [read_item_contents(filename:join(ItemName,Name),
+				filename:join(FileName,Name)) 
+	     || Name<-FileNames]
+    end.
 
 fn(Name, Config) ->
     Dir = ?config(data_dir, Config),