diff options
Diffstat (limited to 'lib/ssh')
-rw-r--r-- | lib/ssh/doc/src/notes.xml | 94 | ||||
-rw-r--r-- | lib/ssh/doc/src/ssh_sftp.xml | 106 | ||||
-rw-r--r-- | lib/ssh/src/ssh.app.src | 2 | ||||
-rw-r--r-- | lib/ssh/src/ssh.appup.src | 12 | ||||
-rw-r--r-- | lib/ssh/src/ssh_connection.erl | 12 | ||||
-rw-r--r-- | lib/ssh/src/ssh_sftp.erl | 324 | ||||
-rw-r--r-- | lib/ssh/test/ssh_connection_SUITE.erl | 27 | ||||
-rw-r--r-- | lib/ssh/test/ssh_sftp_SUITE.erl | 381 |
8 files changed, 824 insertions, 134 deletions
diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index f3db05192e..3aa61aa9ec 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -29,6 +29,100 @@ <file>notes.xml</file> </header> +<section><title>Ssh 3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Make sure the clean rule for ssh, ssl, eunit and otp_mibs + actually removes generated files.</p> + <p> + Own Id: OTP-12200</p> + </item> + <item> + <p> + Improved Property Tests (Thanks to Thomas, John and + Tobias at QuviQ)</p> + <p> + Own Id: OTP-12256</p> + </item> + <item> + <p> + Correct typo of renegotiate that could cause rekeying to + fail</p> + <p> + Own Id: OTP-12277 Aux Id: seq12736 </p> + </item> + <item> + <p> + The {timeout, Timeout} option passed to + ssh_sftp:start_channel was not applied to the early + phases of the SSH protocol. This patch passes the Timeout + through to ssh:connect. In case the timeout occurs during + these phases, {error, timeout} is returned. (Thanks to + Simon Cornish)</p> + <p> + Own Id: OTP-12306</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Added API functions ptty_alloc/3 and ptty_alloc/4, to + allocate a pseudo tty.</p> + <p> + Own Id: OTP-11542 Aux Id: seq12493, OTP-11631 </p> + </item> + <item> + <p> + Supports tar file creation on other media than file + systems mounted on the local machine.</p> + <p> + The <c>erl_tar</c> api is extended with + <c>erl_tar:init/3</c> that enables usage of user provided + media storage routines. A ssh-specific set of such + routines is hidden in the new function + <c>ssh_sftp:open_tar/3</c> to simplify creating a tar + archive on a remote ssh server.</p> + <p> + A chunked file reading option is added to + <c>erl_tar:add/3,4</c> to save memory on e.g small + embedded systems. The size of the slices read from a file + in that case can be specified.</p> + <p> + Own Id: OTP-12180 Aux Id: seq12715 </p> + </item> + <item> + <p> + Always send SSH_DISCONNECT protocol messages when peer + sends corrupt messages.</p> + <p> + Own Id: OTP-12185</p> + </item> + <item> + <p> + Hooks for funs that can change binaries sent to remote + sites from erl_tar for renote tar file creation are + added. See <c>ssh_sftp:open_tar/3,4</c> for details. The + hooks could also be used to read remote tar files that + need transformation before file extraction.</p> + <p> + Those hooks are intended for encryption and decryption of + tar files. Effort is put into memory, disk and network + resource economy.</p> + <p> + Own Id: OTP-12312 Aux Id: OTP-12180 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 3.0.8</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/doc/src/ssh_sftp.xml b/lib/ssh/doc/src/ssh_sftp.xml index 251f5a4be3..ab111562f9 100644 --- a/lib/ssh/doc/src/ssh_sftp.xml +++ b/lib/ssh/doc/src/ssh_sftp.xml @@ -196,19 +196,113 @@ </func> <func> - <name>open_tar(ChannelPid, Path) -></name> - <name>open_tar(ChannelPid, Path, Timeout) -> {ok, Handle} | {error, Reason}</name> - <fsummary>Open a tar file on the server to which <v>ChannelPid</v> is connected and return a handle</fsummary> + <name>open_tar(ChannelPid, Path, Mode) -></name> + <name>open_tar(ChannelPid, Path, Mode, Timeout) -> {ok, Handle} | {error, Reason}</name> + <fsummary>Opens a tar file on the server to which <v>ChannelPid</v> is connected and returns a handle</fsummary> <type> <v>ChannelPid = pid()</v> <v>Path = string()</v> + <v>Mode = [read] | [write] | [read,EncryptOpt] | [write,DecryptOpt] </v> + <v>EncryptOpt = {crypto,{InitFun,EncryptFun,CloseFun}}</v> + <v>DecryptOpt = {crypto,{InitFun,DecryptFun}}</v> + <v>InitFun = (fun() -> {ok,CryptoState}) | (fun() -> {ok,CryptoState,ChunkSize}) </v> + <v>CryptoState = any()</v> + <v>ChunkSize = undefined | pos_integer()</v> + <v>EncryptFun = (fun(PlainBin,CryptoState) -> EncryptResult)</v> + <v>EncryptResult = {ok,EncryptedBin,CryptoState} | {ok,EncryptedBin,CryptoState,ChunkSize}</v> + <v>PlainBin = binary()</v> + <v>EncryptedBin = binary()</v> + <v>DecryptFun = (fun(EncryptedBin,CryptoState) -> DecryptResult)</v> + <v>DecryptResult = {ok,PlainBin,CryptoState} | {ok,PlainBin,CryptoState,ChunkSize}</v> + <v>CloseFun = (fun(PlainBin,CryptoState) -> {ok,EncryptedBin})</v> <v>Timeout = timeout()</v> <v>Reason = term()</v> </type> <desc> - <p>Opens a handle to a tar file on the server, the handle - can be used for remote tar manipulation as defined by the - <seealso marker="stdlib:erl_tar#init/3">erl_tar:init/3</seealso> function.</p> + <p>Opens a handle to a tar file on the server associated with <c>ChannelPid</c>. The handle + can be used for remote tar creation and extraction as defined by the + <seealso marker="stdlib:erl_tar#init/3">erl_tar:init/3</seealso> function. + </p> + <p>An example of writing and then reading a tar file:</p> + <code type="none"> + {ok,HandleWrite} = ssh_sftp:open_tar(ChannelPid, ?tar_file_name, [write]), + ok = erl_tar:add(HandleWrite, .... ), + ok = erl_tar:add(HandleWrite, .... ), + ... + ok = erl_tar:add(HandleWrite, .... ), + ok = erl_tar:close(HandleWrite), + + %% And for reading + {ok,HandleRead} = ssh_sftp:open_tar(ChannelPid, ?tar_file_name, [read]), + {ok,NameValueList} = erl_tar:extract(HandleRead,[memory]), + ok = erl_tar:close(HandleRead), + </code> + + <p>The <c>crypto</c> mode option is applied to the generated stream of bytes just prior to sending + them to the sftp server. This is intended for encryption but could of course be used for other + purposes. + </p> + <p>The <c>InitFun</c> is applied once + prior to any other crypto operation. The returned <c>CryptoState</c> is then folded into + repeated applications of the <c>EncryptFun</c> or <c>DecryptFun</c>. The binary returned + from those Funs are sent further to the remote sftp server. Finally - if doing encryption + - the <c>CloseFun</c> is applied to the last piece of data. The <c>CloseFun</c> is + responsible for padding (if needed) and encryption of that last piece. + </p> + <p>The <c>ChunkSize</c> defines the size of the <c>PlainBin</c>s that <c>EncodeFun</c> is applied + to. If the <c>ChunkSize</c> is <c>undefined</c> the size of the <c>PlainBin</c>s varies because + this is inteded for stream crypto while a fixed <c>ChunkSize</c> is intended for block crypto. It + is possible to change the <c>ChunkSize</c>s in the return from the <c>EncryptFun</c> or + <c>DecryptFun</c>. It is in fact possible to change the value between <c>pos_integer()</c> and + <c>undefined</c>. + </p> + <p>The write and read example above can be extended with encryption and decryption:</p> + <code type="none"> + %% First three parameters depending on which crypto type we select: + Key = <<"This is a 256 bit key. abcdefghi">>, + Ivec0 = crypto:rand_bytes(16), + DataSize = 1024, % DataSize rem 16 = 0 for aes_cbc + + %% Initialization of the CryptoState, in this case it is the Ivector. + InitFun = fun() -> {ok, Ivec0, DataSize} end, + + %% How to encrypt: + EncryptFun = + fun(PlainBin,Ivec) -> + EncryptedBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, PlainBin), + {ok, EncryptedBin, crypto:next_iv(aes_cbc,EncryptedBin)} + end, + + %% What to do with the very last block: + CloseFun = + fun(PlainBin, Ivec) -> + EncryptedBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, + pad(16,PlainBin) %% Last chunk + ), + {ok, EncryptedBin} + end, + + Cw = {InitFun,EncryptFun,CloseFun}, + {ok,HandleWrite} = ssh_sftp:open_tar(ChannelPid, ?tar_file_name, [write,{crypto,Cw}]), + ok = erl_tar:add(HandleWrite, .... ), + ok = erl_tar:add(HandleWrite, .... ), + ... + ok = erl_tar:add(HandleWrite, .... ), + ok = erl_tar:close(HandleWrite), + + %% And for decryption (in this crypto example we could use the same InitFun + %% as for encryption): + DecryptFun = + fun(EncryptedBin,Ivec) -> + PlainBin = crypto:block_decrypt(aes_cbc256, Key, Ivec, EncryptedBin), + {ok, PlainBin, crypto:next_iv(aes_cbc,EncryptedBin)} + end, + + Cr = {InitFun,DecryptFun}, + {ok,HandleRead} = ssh_sftp:open_tar(ChannelPid, ?tar_file_name, [read,{crypto,Cw}]), + {ok,NameValueList} = erl_tar:extract(HandleRead,[memory]), + ok = erl_tar:close(HandleRead), + </code> </desc> </func> diff --git a/lib/ssh/src/ssh.app.src b/lib/ssh/src/ssh.app.src index 4ad55b34ca..bc01c539e0 100644 --- a/lib/ssh/src/ssh.app.src +++ b/lib/ssh/src/ssh.app.src @@ -40,7 +40,7 @@ {applications, [kernel, stdlib, crypto, public_key]}, {env, []}, {mod, {ssh_app, []}}, - {runtime_dependencies, ["stdlib-2.0","public_key-0.22","kernel-3.0", + {runtime_dependencies, ["stdlib-2.3","public_key-0.22","kernel-3.0", "erts-6.0","crypto-3.3"]}]}. diff --git a/lib/ssh/src/ssh.appup.src b/lib/ssh/src/ssh.appup.src index 600c01454c..b2b2994eed 100644 --- a/lib/ssh/src/ssh.appup.src +++ b/lib/ssh/src/ssh.appup.src @@ -19,6 +19,12 @@ {"%VSN%", [ + {"3.0.8", [{load_module, ssh_connection, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_sftp, soft_purge, soft_purge, [ssh_xfer]}, + {load_module, ssh_connection_handler, soft_purge, soft_purge, []}, + {load_module, ssh, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_xfer, soft_purge, soft_purge, []} + ]}, {"3.0.7", [{load_module, ssh_auth, soft_purge, soft_purge, [ssh_connection_handler]}, {load_module, ssh_acceptor, soft_purge, soft_purge, [ssh_connection_handler]}, {load_module, ssh_channel, soft_purge, soft_purge, [ssh_connection_handler]}, @@ -42,6 +48,12 @@ {<<".*">>, [{restart_application, ssh}]} ], [ + {"3.0.8", [{load_module, ssh_connection, soft_purge, soft_purge, [ssh_connection_handler]}, + {load_module, ssh_sftp, soft_purge, soft_purge, []}, + {load_module, ssh_connection_handler, soft_purge, soft_purge, []}, + {load_module, ssh, soft_purge, soft_purge, []}, + {load_module, ssh_xfer, soft_purge, soft_purge, []} + ]}, {"3.0.7", [{load_module, ssh_auth, soft_purge, soft_purge, [ssh_connection_handler]}, {load_module, ssh_acceptor, soft_purge, soft_purge, [ssh_connection_handler]}, {load_module, ssh_channel, soft_purge, soft_purge, [ssh_connection_handler]}, diff --git a/lib/ssh/src/ssh_connection.erl b/lib/ssh/src/ssh_connection.erl index 593443e11c..01141622d6 100644 --- a/lib/ssh/src/ssh_connection.erl +++ b/lib/ssh/src/ssh_connection.erl @@ -107,15 +107,9 @@ shell(ConnectionHandler, ChannelId) -> %% Description: Executes a predefined subsystem. %%-------------------------------------------------------------------- subsystem(ConnectionHandler, ChannelId, SubSystem, TimeOut) -> - case ssh_connection_handler:request(ConnectionHandler, self(), - ChannelId, "subsystem", - true, [?string(SubSystem)], TimeOut) of - success -> success; - failure -> failure; - {error,timeout} -> {error,timeout}; - _ -> failure - end. - + ssh_connection_handler:request(ConnectionHandler, self(), + ChannelId, "subsystem", + true, [?string(SubSystem)], TimeOut). %%-------------------------------------------------------------------- -spec send(pid(), channel_id(), iodata()) -> ok | {error, closed}. diff --git a/lib/ssh/src/ssh_sftp.erl b/lib/ssh/src/ssh_sftp.erl index 3b80f5326c..613f8f25b2 100644 --- a/lib/ssh/src/ssh_sftp.erl +++ b/lib/ssh/src/ssh_sftp.erl @@ -69,6 +69,18 @@ mode }). +-record(bufinf, + { + mode, % read | write (=from or to buffer by user) + crypto_state, + crypto_fun, % For encode or decode depending on the mode field + size = 0, % # bytes "before" the current buffer for the postion call + + chunksize, % The size of the chunks to be sent or received + enc_text_buf = <<>>, % Encrypted text + plain_text_buf = <<>> % Decrypted text + }). + -define(FILEOP_TIMEOUT, infinity). -define(NEXT_REQID(S), @@ -164,24 +176,73 @@ open(Pid, File, Mode, FileOpTimeout) -> open_tar(Pid, File, Mode) -> open_tar(Pid, File, Mode, ?FILEOP_TIMEOUT). -open_tar(Pid, File, Mode=[write], FileOpTimeout) -> - {ok,R} = open(Pid, File, Mode, FileOpTimeout), - erl_tar:init({Pid,R,FileOpTimeout}, write, - fun(write, {{P,H,T},Data}) -> - Bin = if is_list(Data) -> list_to_binary(Data); - is_binary(Data) -> Data - end, - {ok,{_Window,Packet}} = send_window(P, T), - write_file_loop(P, H, 0, Bin, size(Bin), Packet, T); - (position, {{P,H,T},Pos}) -> - position(P, H, Pos, T); - (close, {P,H,T}) -> - close(P, H, T) - end); -open_tar(_Pid, _File, Mode, _FileOpTimeout) -> - {error,{illegal_mode,Mode}}. - - +open_tar(Pid, File, Mode, FileOpTimeout) -> + case {lists:member(write,Mode), + lists:member(read,Mode), + Mode -- [read,write]} of + {true,false,[]} -> + {ok,Handle} = open(Pid, File, [write], FileOpTimeout), + erl_tar:init(Pid, write, + fun(write, {_,Data}) -> + write_to_remote_tar(Pid, Handle, to_bin(Data), FileOpTimeout); + (position, {_,Pos}) -> + position(Pid, Handle, Pos, FileOpTimeout); + (close, _) -> + close(Pid, Handle, FileOpTimeout) + end); + {true,false,[{crypto,{CryptoInitFun,CryptoEncryptFun,CryptoEndFun}}]} -> + {ok,SftpHandle} = open(Pid, File, [write], FileOpTimeout), + BI = #bufinf{mode = write, + crypto_fun = CryptoEncryptFun}, + {ok,BufHandle} = open_buf(Pid, CryptoInitFun, BI, FileOpTimeout), + erl_tar:init(Pid, write, + fun(write, {_,Data}) -> + write_buf(Pid, SftpHandle, BufHandle, to_bin(Data), FileOpTimeout); + (position, {_,Pos}) -> + position_buf(Pid, SftpHandle, BufHandle, Pos, FileOpTimeout); + (close, _) -> + {ok,#bufinf{ + plain_text_buf = PlainBuf0, + enc_text_buf = EncBuf0, + crypto_state = CState0 + }} = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout), + {ok,EncTextTail} = CryptoEndFun(PlainBuf0, CState0), + EncTextBuf = <<EncBuf0/binary, EncTextTail/binary>>, + case write(Pid, SftpHandle, EncTextBuf, FileOpTimeout) of + ok -> + call(Pid, {erase_bufinf,BufHandle}, FileOpTimeout), + close(Pid, SftpHandle, FileOpTimeout); + Other -> + Other + end + end); + {false,true,[]} -> + {ok,Handle} = open(Pid, File, [read,binary], FileOpTimeout), + erl_tar:init(Pid, read, + fun(read2, {_,Len}) -> + read_repeat(Pid, Handle, Len, FileOpTimeout); + (position, {_,Pos}) -> + position(Pid, Handle, Pos, FileOpTimeout); + (close, _) -> + close(Pid, Handle, FileOpTimeout) + end); + {false,true,[{crypto,{CryptoInitFun,CryptoDecryptFun}}]} -> + {ok,SftpHandle} = open(Pid, File, [read,binary], FileOpTimeout), + BI = #bufinf{mode = read, + crypto_fun = CryptoDecryptFun}, + {ok,BufHandle} = open_buf(Pid, CryptoInitFun, BI, FileOpTimeout), + erl_tar:init(Pid, read, + fun(read2, {_,Len}) -> + read_buf(Pid, SftpHandle, BufHandle, Len, FileOpTimeout); + (position, {_,Pos}) -> + position_buf(Pid, SftpHandle, BufHandle, Pos, FileOpTimeout); + (close, _) -> + call(Pid, {erase_bufinf,BufHandle}, FileOpTimeout), + close(Pid, SftpHandle, FileOpTimeout) + end); + _ -> + {error,{illegal_mode,Mode}} + end. opendir(Pid, Path) -> @@ -469,6 +530,15 @@ handle_cast(_,State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. +do_handle_call({get_bufinf,BufHandle}, _From, S=#state{inf=I0}) -> + {reply, dict:find(BufHandle,I0), S}; + +do_handle_call({put_bufinf,BufHandle,B}, _From, S=#state{inf=I0}) -> + {reply, ok, S#state{inf=dict:store(BufHandle,B,I0)}}; + +do_handle_call({erase_bufinf,BufHandle}, _From, S=#state{inf=I0}) -> + {reply, ok, S#state{inf=dict:erase(BufHandle,I0)}}; + do_handle_call({open, Async,FileName,Mode}, From, #state{xf = XF} = State) -> {Access,Flags,Attrs} = open_mode(XF#ssh_xfer.vsn, Mode), ReqID = State#state.req_id, @@ -573,12 +643,7 @@ do_handle_call({read,Async,Handle,Length}, From, State) -> do_handle_call({pwrite,Async,Handle,At,Data0}, From, State) -> case lseek_position(Handle, At, State) of {ok,Offset} -> - Data = if - is_binary(Data0) -> - Data0; - is_list(Data0) -> - list_to_binary(Data0) - end, + Data = to_bin(Data0), ReqID = State#state.req_id, Size = size(Data), ssh_xfer:write(?XF(State),ReqID,Handle,Offset,Data), @@ -591,12 +656,7 @@ do_handle_call({pwrite,Async,Handle,At,Data0}, From, State) -> do_handle_call({write,Async,Handle,Data0}, From, State) -> case lseek_position(Handle, cur, State) of {ok,Offset} -> - Data = if - is_binary(Data0) -> - Data0; - is_list(Data0) -> - list_to_binary(Data0) - end, + Data = to_bin(Data0), ReqID = State#state.req_id, Size = size(Data), ssh_xfer:write(?XF(State),ReqID,Handle,Offset,Data), @@ -1148,5 +1208,207 @@ lseek_pos({eof, Offset}, _CurOffset, CurSize) end; lseek_pos(_, _, _) -> {error, einval}. - +%%%================================================================ +%%% +to_bin(Data) when is_list(Data) -> list_to_binary(Data); +to_bin(Data) when is_binary(Data) -> Data. + + +read_repeat(Pid, Handle, Len, FileOpTimeout) -> + {ok,{_WindowSz,PacketSz}} = recv_window(Pid, FileOpTimeout), + read_rpt(Pid, Handle, Len, PacketSz, FileOpTimeout, <<>>). + +read_rpt(Pid, Handle, WantedLen, PacketSz, FileOpTimeout, Acc) when WantedLen > 0 -> + case read(Pid, Handle, min(WantedLen,PacketSz), FileOpTimeout) of + {ok, Data} -> + read_rpt(Pid, Handle, WantedLen-size(Data), PacketSz, FileOpTimeout, <<Acc/binary, Data/binary>>); + eof -> + {ok, Acc}; + Error -> + Error + end; +read_rpt(_Pid, _Handle, WantedLen, _PacketSz, _FileOpTimeout, Acc) when WantedLen >= 0 -> + {ok,Acc}. + + +write_to_remote_tar(_Pid, _SftpHandle, <<>>, _FileOpTimeout) -> + ok; +write_to_remote_tar(Pid, SftpHandle, Bin, FileOpTimeout) -> + {ok,{_Window,Packet}} = send_window(Pid, FileOpTimeout), + write_file_loop(Pid, SftpHandle, 0, Bin, size(Bin), Packet, FileOpTimeout). + +position_buf(Pid, SftpHandle, BufHandle, Pos, FileOpTimeout) -> + {ok,#bufinf{mode = Mode, + plain_text_buf = Buf0, + size = Size}} = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout), + case Pos of + {cur,0} when Mode==write -> + {ok,Size+size(Buf0)}; + + {cur,0} when Mode==read -> + {ok,Size}; + + _ when Mode==read, is_integer(Pos) -> + Skip = Pos-Size, + if + Skip < 0 -> + {error, cannot_rewind}; + Skip == 0 -> + %% Optimization + {ok,Pos}; + Skip > 0 -> + case read_buf(Pid, SftpHandle, BufHandle, Skip, FileOpTimeout) of + %% A bit innefficient to fetch the bufinf again, but there are lots of + %% other more important optimizations waiting.... + {ok,_} -> + {ok,Pos}; + Other -> + Other + end + end; + + _ -> + {error,{not_yet_implemented,{pos,Pos}}} + end. + +read_buf(Pid, SftpHandle, BufHandle, WantedLen, FileOpTimeout) -> + {ok,{_Window,Packet}} = send_window(Pid, FileOpTimeout), + {ok,B0} = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout), + case do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, B0) of + {ok,ResultBin,B} -> + call(Pid, {put_bufinf,BufHandle,B}, FileOpTimeout), + {ok,ResultBin}; + {error,Error} -> + {error,Error}; + {eof,B} -> + call(Pid, {put_bufinf,BufHandle,B}, FileOpTimeout), + eof + end. + +do_the_read_buf(_Pid, _SftpHandle, WantedLen, _Packet, _FileOpTimeout, + B=#bufinf{plain_text_buf=PlainBuf0, + size = Size}) + when size(PlainBuf0) >= WantedLen -> + %% We already have the wanted number of bytes decoded and ready! + <<ResultBin:WantedLen/binary, PlainBuf/binary>> = PlainBuf0, + {ok,ResultBin,B#bufinf{plain_text_buf=PlainBuf, + size = Size + WantedLen}}; + +do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, + B0=#bufinf{plain_text_buf = PlainBuf0, + enc_text_buf = EncBuf0, + chunksize = undefined + }) + when size(EncBuf0) > 0 -> + %% We have (at least) one decodable byte waiting for decodeing. + {ok,DecodedBin,B} = apply_crypto(EncBuf0, B0), + do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, + B#bufinf{plain_text_buf = <<PlainBuf0/binary, DecodedBin/binary>>, + enc_text_buf = <<>> + }); + +do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, + B0=#bufinf{plain_text_buf = PlainBuf0, + enc_text_buf = EncBuf0, + chunksize = ChunkSize0 + }) + when size(EncBuf0) >= ChunkSize0 -> + %% We have (at least) one chunk of decodable bytes waiting for decodeing. + <<ToDecode:ChunkSize0/binary, EncBuf/binary>> = EncBuf0, + {ok,DecodedBin,B} = apply_crypto(ToDecode, B0), + do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, + B#bufinf{plain_text_buf = <<PlainBuf0/binary, DecodedBin/binary>>, + enc_text_buf = EncBuf + }); + +do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, B=#bufinf{enc_text_buf = EncBuf0}) -> + %% We must read more bytes and append to the buffer of encoded bytes. + case read(Pid, SftpHandle, Packet, FileOpTimeout) of + {ok,EncryptedBin} -> + do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, + B#bufinf{enc_text_buf = <<EncBuf0/binary, EncryptedBin/binary>>}); + eof -> + {eof,B}; + Other -> + Other + end. + + +write_buf(Pid, SftpHandle, BufHandle, PlainBin, FileOpTimeout) -> + {ok,{_Window,Packet}} = send_window(Pid, FileOpTimeout), + {ok,B0=#bufinf{plain_text_buf=PTB}} = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout), + case do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B0#bufinf{plain_text_buf = <<PTB/binary,PlainBin/binary>>}) of + {ok, B} -> + call(Pid, {put_bufinf,BufHandle,B}, FileOpTimeout), + ok; + {error,Error} -> + {error,Error} + end. + +do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B=#bufinf{enc_text_buf = EncBuf0, + size = Size}) + when size(EncBuf0) >= Packet -> + <<BinToWrite:Packet/binary, EncBuf/binary>> = EncBuf0, + case write(Pid, SftpHandle, BinToWrite, FileOpTimeout) of + ok -> + do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B#bufinf{enc_text_buf = EncBuf, + size = Size + Packet}); + Other -> + Other + end; + +do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B0=#bufinf{plain_text_buf = PlainBuf0, + enc_text_buf = EncBuf0, + chunksize = undefined}) + when size(PlainBuf0) > 0 -> + {ok,EncodedBin,B} = apply_crypto(PlainBuf0, B0), + do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B#bufinf{plain_text_buf = <<>>, + enc_text_buf = <<EncBuf0/binary, EncodedBin/binary>>}); + +do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B0=#bufinf{plain_text_buf = PlainBuf0, + enc_text_buf = EncBuf0, + chunksize = ChunkSize0 + }) + when size(PlainBuf0) >= ChunkSize0 -> + <<ToEncode:ChunkSize0/binary, PlainBuf/binary>> = PlainBuf0, + {ok,EncodedBin,B} = apply_crypto(ToEncode, B0), + do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B#bufinf{plain_text_buf = PlainBuf, + enc_text_buf = <<EncBuf0/binary, EncodedBin/binary>>}); + +do_the_write_buf(_Pid, _SftpHandle, _Packet, _FileOpTimeout, B) -> + {ok,B}. + +apply_crypto(In, B=#bufinf{crypto_state = CState0, + crypto_fun = F}) -> + case F(In,CState0) of + {ok,EncodedBin,CState} -> + {ok, EncodedBin, B#bufinf{crypto_state=CState}}; + {ok,EncodedBin,CState,ChunkSize} -> + {ok, EncodedBin, B#bufinf{crypto_state=CState, + chunksize=ChunkSize}} + end. + +open_buf(Pid, CryptoInitFun, BufInfo0, FileOpTimeout) -> + case CryptoInitFun() of + {ok,CryptoState} -> + open_buf1(Pid, BufInfo0, FileOpTimeout, CryptoState, undefined); + {ok,CryptoState,ChunkSize} -> + open_buf1(Pid, BufInfo0, FileOpTimeout, CryptoState, ChunkSize); + Other -> + Other + end. + +open_buf1(Pid, BufInfo0, FileOpTimeout, CryptoState, ChunkSize) -> + BufInfo = BufInfo0#bufinf{crypto_state = CryptoState, + chunksize = ChunkSize}, + BufHandle = make_ref(), + call(Pid, {put_bufinf,BufHandle,BufInfo}, FileOpTimeout), + {ok,BufHandle}. diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index 553d0f5720..85bd2c75d4 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -37,7 +37,7 @@ all() -> [ {group, openssh}, - start_subsystem_on_closed_channel, + {group, openssh_payload}, interrupted_send, start_shell, start_shell_exec, @@ -286,32 +286,7 @@ ptty_alloc_pixel(Config) when is_list(Config) -> ssh:close(ConnectionRef). %%-------------------------------------------------------------------- -start_subsystem_on_closed_channel(Config) -> - PrivDir = ?config(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), - SysDir = ?config(data_dir, Config), - {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, UserDir}, - {password, "morot"}, - {subsystems, [{"echo_n", {ssh_echo_server, [4000000]}}]}]), - - ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, - {user, "foo"}, - {password, "morot"}, - {user_interaction, false}, - {user_dir, UserDir}]), - - {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), - - ok = ssh_connection:close(ConnectionRef, ChannelId), - failure = ssh_connection:subsystem(ConnectionRef, ChannelId, "echo_n", infinity), - - ssh:close(ConnectionRef), - ssh:stop_daemon(Pid). - -%%-------------------------------------------------------------------- interrupted_send() -> [{doc, "Use a subsystem that echos n char and then sends eof to cause a channel exit partway through a large send."}]. diff --git a/lib/ssh/test/ssh_sftp_SUITE.erl b/lib/ssh/test/ssh_sftp_SUITE.erl index 559fa721fd..cb74a27638 100644 --- a/lib/ssh/test/ssh_sftp_SUITE.erl +++ b/lib/ssh/test/ssh_sftp_SUITE.erl @@ -65,19 +65,25 @@ end_per_suite(Config) -> %%-------------------------------------------------------------------- groups() -> [{erlang_server, [], [open_close_file, open_close_dir, read_file, read_dir, - write_file, write_big_file, rename_file, mk_rm_dir, remove_file, links, + write_file, write_big_file, sftp_read_big_file, + rename_file, mk_rm_dir, remove_file, links, retrieve_attributes, set_attributes, async_read, async_write, position, pos_read, pos_write, version_option, - {group,remote_tar_write} - ]}, + {group,remote_tar}]}, + {openssh_server, [], [open_close_file, open_close_dir, read_file, read_dir, - write_file, write_big_file, rename_file, mk_rm_dir, remove_file, links, + write_file, write_big_file, sftp_read_big_file, + rename_file, mk_rm_dir, remove_file, links, retrieve_attributes, set_attributes, async_read, async_write, position, pos_read, pos_write, - {group,remote_tar_write}]}, - - {remote_tar_write, [], [create_empty_tar, files_to_tar, big_file_to_tar, files_chunked_to_tar, - directory_to_tar, binaries_to_tar]} + {group,remote_tar}]}, + + {remote_tar, [], [create_empty_tar, files_to_tar, big_file_to_tar, files_chunked_to_tar, + directory_to_tar, binaries_to_tar, null_crypto_tar, + simple_crypto_tar_small, simple_crypto_tar_big, + read_tar, read_null_crypto_tar, read_crypto_tar, + aes_cbc256_crypto_tar, aes_ctr_stream_crypto_tar + ]} ]. @@ -104,7 +110,7 @@ init_per_group(openssh_server, Config) -> {skip, "No openssh server"} end; -init_per_group(remote_tar_write, Config) -> +init_per_group(remote_tar, Config) -> {Host,Port} = ?config(peer, Config), ct:log("Server (~p) at ~p:~p",[?config(group,Config),Host,Port]), {ok, Connection} = @@ -120,7 +126,7 @@ init_per_group(remote_tar_write, Config) -> [{user_interaction, false}, {silently_accept_hosts, true}]) end, - [{remote_tar_write, true}, + [{remote_tar, true}, {connection, Connection} | Config]. end_per_group(erlang_server, Config) -> @@ -187,16 +193,12 @@ init_per_testcase(Case, Config0) -> [{sftp, Sftp}, {watchdog, Dog} | Config2] end, - case catch ?config(remote_tar_write,Config) of + case catch ?config(remote_tar,Config) of %% The 'catch' is for the case of Config={skip,...} true -> - %% Provide a tar Handle *independent* of the sftp-channel already opened! - %% This Handle will be closed (as well as ChannelPid2) in the testcase - {ok,ChannelPid2} = - ssh_sftp:start_channel(?config(connection,Config)), - {ok,Handle} = - ssh_sftp:open_tar(ChannelPid2, fnp(?tar_file_name,Config), [write]), - [{handle,Handle} | Config]; + %% Provide a ChannelPid independent of the sftp-channel already opened. + {ok,ChPid2} = ssh_sftp:start_channel(?config(connection,Config)), + [{channel_pid2,ChPid2} | Config]; _ -> Config end. @@ -214,6 +216,7 @@ end_per_testcase(_, Config) -> end_per_testcase(Config) -> {Sftp, Connection} = ?config(sftp, Config), ssh_sftp:stop_channel(Sftp), + catch ssh_sftp:stop_channel(?config(channel_pid2, Config)), ssh:close(Connection). %%-------------------------------------------------------------------- @@ -258,6 +261,7 @@ read_file(Config) when is_list(Config) -> FileName = filename:join(PrivDir, "sftp.txt"), {Sftp, _} = ?config(sftp, Config), {ok, Data} = ssh_sftp:read_file(Sftp, FileName), + {ok, Data} = ssh_sftp:read_file(Sftp, FileName), {ok, Data} = file:read_file(FileName). %%-------------------------------------------------------------------- @@ -294,6 +298,19 @@ write_big_file(Config) when is_list(Config) -> {ok, Data} = file:read_file(FileName). %%-------------------------------------------------------------------- +sftp_read_big_file() -> + [{doc, "Test API function read_file/2 with big data"}]. +sftp_read_big_file(Config) when is_list(Config) -> + PrivDir = ?config(priv_dir, Config), + FileName = filename:join(PrivDir, "sftp.txt"), + {Sftp, _} = ?config(sftp, Config), + + Data = list_to_binary(lists:duplicate(750000,"a")), + ct:log("Data size to write is ~p bytes",[size(Data)]), + ssh_sftp:write_file(Sftp, FileName, [Data]), + {ok, Data} = ssh_sftp:read_file(Sftp, FileName). + +%%-------------------------------------------------------------------- remove_file() -> [{doc,"Test API function delete/2"}]. remove_file(Config) when is_list(Config) -> @@ -527,53 +544,247 @@ version_option(Config) when is_list(Config) -> %%-------------------------------------------------------------------- create_empty_tar(Config) -> - {ChPid,_} = ?config(sftp,Config), - Handle = ?config(handle,Config), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), erl_tar:close(Handle), + {ChPid,_} = ?config(sftp,Config), {ok, #file_info{type=regular}} = ssh_sftp:read_file_info(ChPid,fnp(?tar_file_name,Config)). - + %%-------------------------------------------------------------------- files_to_tar(Config) -> - Handle = ?config(handle,Config), - ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", []), - ok = erl_tar:add(Handle, fn("f2.txt",Config), "f2.txt", []), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), + ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose]), + ok = erl_tar:add(Handle, fn("f2.txt",Config), "f2.txt", [verbose]), ok = erl_tar:close(Handle), chk_tar(["f1.txt", "f2.txt"], Config). - %%-------------------------------------------------------------------- big_file_to_tar(Config) -> - Handle = ?config(handle,Config), - ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", []), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), + ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [verbose]), ok = erl_tar:close(Handle), chk_tar(["big.txt"], Config). %%-------------------------------------------------------------------- files_chunked_to_tar(Config) -> - Handle = ?config(handle,Config), - ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [{chunks,2}]), - ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [{chunks,15000}]), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), + ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]), ok = erl_tar:close(Handle), - chk_tar(["f1.txt", "big.txt"], Config). + chk_tar(["f1.txt"], Config). %%-------------------------------------------------------------------- directory_to_tar(Config) -> - Handle = ?config(handle,Config), - ok = erl_tar:add(Handle, fn("d1",Config), "d1", []), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), + ok = erl_tar:add(Handle, fn("d1",Config), "d1", [verbose]), ok = erl_tar:close(Handle), - chk_tar(["d1/f1", "d1/f2"], Config). + chk_tar(["d1"], Config). %%-------------------------------------------------------------------- binaries_to_tar(Config) -> - Handle = ?config(handle,Config), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), Bin = <<"A binary">>, - ok = erl_tar:add(Handle, Bin, "b1", []), + ok = erl_tar:add(Handle, Bin, "b1", [verbose]), ok = erl_tar:close(Handle), chk_tar([{"b1",Bin}], Config). %%-------------------------------------------------------------------- +null_crypto_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + Cinit = fun() -> {ok, no_state, _SendSize=5} end, + Cenc = fun(Bin,CState) -> {ok,Bin,CState,_SendSize=5} end, + Cend = fun(Bin,_CState) -> {ok,Bin} end, + C = {Cinit,Cenc,Cend}, + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,C}]), + Bin = <<"A binary">>, + ok = erl_tar:add(Handle, Bin, "b1", [verbose]), + ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]), + ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [verbose,{chunks,15000}]), + ok = erl_tar:close(Handle), + chk_tar([{"b1",Bin}, "f1.txt", "big.txt"], Config). + +%%-------------------------------------------------------------------- +simple_crypto_tar_small(Config) -> + ChPid2 = ?config(channel_pid2, Config), + Cinit = fun() -> {ok, no_state, _Size=6} end, + Cenc = fun(Bin,CState) -> {ok,stuff(Bin),CState,_SendSize=5} end, + Cdec = fun(Bin,CState) -> {ok,unstuff(Bin),CState,_Size=4} end, + Cend = fun(Bin,_CState) -> {ok,stuff(Bin)} end, + C = {Cinit,Cenc,Cend}, + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,C}]), + Bin = <<"A binary">>, + ok = erl_tar:add(Handle, Bin, "b1", [verbose]), + ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]), + ok = erl_tar:close(Handle), + chk_tar([{"b1",Bin}, "f1.txt"], Config, [{crypto,{Cinit,Cdec}}]). + +%%-------------------------------------------------------------------- +simple_crypto_tar_big(Config) -> + ChPid2 = ?config(channel_pid2, Config), + Cinit = fun() -> {ok, no_state, _SendSize=6} end, + Cenc = fun(Bin,CState) -> {ok,stuff(Bin),CState,_SendSize=5} end, + Cdec = fun(Bin,CState) -> {ok,unstuff(Bin),CState,_SendSize=4} end, + Cend = fun(Bin,_CState) -> {ok,stuff(Bin)} end, + C = {Cinit,Cenc,Cend}, + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,C}]), + Bin = <<"A binary">>, + ok = erl_tar:add(Handle, Bin, "b1", [verbose]), + ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]), + ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [verbose,{chunks,15000}]), + ok = erl_tar:close(Handle), + chk_tar([{"b1",Bin}, "f1.txt", "big.txt"], Config, [{crypto,{Cinit,Cdec}}]). + +stuff(Bin) -> << <<C,C>> || <<C>> <= Bin >>. + +unstuff(Bin) -> << <<C>> || <<C,C>> <= Bin >>. + +%%-------------------------------------------------------------------- +read_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + NameBins = lists:sort( + [{"b1",<<"A binary">>}, + {"b2",list_to_binary(lists:duplicate(750000,"a"))} + ]), + {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), + [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) + || {Name,Bin} <- NameBins], + ok = erl_tar:close(HandleWrite), + + chk_tar(NameBins, Config). + +%%-------------------------------------------------------------------- +read_null_crypto_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + NameBins = lists:sort( + [{"b1",<<"A binary">>}, + {"b2",list_to_binary(lists:duplicate(750000,"a"))} + ]), + Cinitw = fun() -> {ok, no_state, _SendSize=5} end, + Cinitr = fun() -> {ok, no_state, _FetchSize=42} end, + Cenc = fun(Bin,CState) -> {ok,Bin,CState,_SendSize=42*42} end, + Cdec = fun(Bin,CState) -> {ok,Bin,CState,_FetchSize=19} end, + Cendw = fun(Bin,_CState) -> {ok,Bin} end, + Cw = {Cinitw,Cenc,Cendw}, + Cr = {Cinitr,Cdec}, + + {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]), + [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) + || {Name,Bin} <- NameBins], + ok = erl_tar:close(HandleWrite), + + chk_tar(NameBins, Config, [{crypto,Cr}]). + +%%-------------------------------------------------------------------- +read_crypto_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + NameBins = lists:sort( + [{"b1",<<"A binary">>}, + {"b2",list_to_binary(lists:duplicate(750000,"a"))} + ]), + Cinitw = fun() -> {ok, no_state, _SendSize=5} end, + Cinitr = fun() -> {ok, no_state, _FetchSize=42} end, + + Cenc = fun(Bin,CState) -> {ok,stuff(Bin),CState,_SendSize=42*42} end, + Cdec = fun(Bin,CState) -> {ok,unstuff(Bin),CState,_FetchSize=120} end, + Cendw = fun(Bin,_CState) -> {ok,stuff(Bin)} end, + Cw = {Cinitw,Cenc,Cendw}, + Cr = {Cinitr,Cdec}, + + {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]), + [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) + || {Name,Bin} <- NameBins], + ok = erl_tar:close(HandleWrite), + + chk_tar(NameBins, Config, [{crypto,Cr}]). + +%%-------------------------------------------------------------------- +aes_cbc256_crypto_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + NameBins = lists:sort( + [{"b1",<<"A binary">>}, + {"b2",list_to_binary(lists:duplicate(750000,"a"))}, + {"d1",fn("d1",Config)} % Dir + ]), + Key = <<"This is a 256 bit key. Boring...">>, + Ivec0 = crypto:rand_bytes(16), + DataSize = 1024, % data_size rem 16 = 0 for aes_cbc + + Cinitw = fun() -> {ok, Ivec0, DataSize} end, + Cinitr = fun() -> {ok, Ivec0, DataSize} end, + + Cenc = fun(PlainBin,Ivec) -> + CipherBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, PlainBin), + {ok, CipherBin, crypto:next_iv(aes_cbc,CipherBin), DataSize} + end, + Cdec = fun(CipherBin,Ivec) -> + PlainBin = crypto:block_decrypt(aes_cbc256, Key, Ivec, CipherBin), + {ok, PlainBin, crypto:next_iv(aes_cbc,CipherBin), DataSize} + end, + + Cendw = fun(PlainBin, _) when PlainBin == <<>> -> {ok, <<>>}; + (PlainBin, Ivec) -> + CipherBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, + pad(16,PlainBin)), %% Last chunk + {ok, CipherBin} + end, + + Cw = {Cinitw,Cenc,Cendw}, + {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]), + [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) || {Name,Bin} <- NameBins], + ok = erl_tar:close(HandleWrite), + + Cr = {Cinitr,Cdec}, + chk_tar(NameBins, Config, [{crypto,Cr}]). + + +pad(BlockSize, Bin) -> + PadSize = (BlockSize - (size(Bin) rem BlockSize)) rem BlockSize, + list_to_binary( lists:duplicate(PadSize,0) ). + +%%-------------------------------------------------------------------- +aes_ctr_stream_crypto_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + NameBins = lists:sort( + [{"b1",<<"A binary">>}, + {"b2",list_to_binary(lists:duplicate(750000,"a"))}, + {"d1",fn("d1",Config)} % Dir + ]), + Key = <<"This is a 256 bit key. Boring...">>, + Ivec0 = crypto:rand_bytes(16), + + Cinitw = Cinitr = fun() -> {ok, crypto:stream_init(aes_ctr,Key,Ivec0)} end, + + Cenc = fun(PlainBin,State) -> + {NewState,CipherBin} = crypto:stream_encrypt(State, PlainBin), + {ok, CipherBin, NewState} + end, + Cdec = fun(CipherBin,State) -> + {NewState,PlainBin} = crypto:stream_decrypt(State, CipherBin), + {ok, PlainBin, NewState} + end, + + Cendw = fun(PlainBin, _) when PlainBin == <<>> -> {ok, <<>>}; + (PlainBin, Ivec) -> + CipherBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, + pad(16,PlainBin)), %% Last chunk + {ok, CipherBin} + end, + + Cw = {Cinitw,Cenc,Cendw}, + {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]), + [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) || {Name,Bin} <- NameBins], + ok = erl_tar:close(HandleWrite), + + Cr = {Cinitr,Cdec}, + chk_tar(NameBins, Config, [{crypto,Cr}]). + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- prep(Config) -> @@ -597,34 +808,82 @@ prep(Config) -> FileInfo#file_info{mode = Mode}). + chk_tar(Items, Config) -> - %% FIXME: ought to check that no more than expected is present... + chk_tar(Items, Config, []). + +chk_tar(Items, Config, Opts) -> + chk_tar(Items, fnp(?tar_file_name,Config), Config, Opts). + +chk_tar(Items, TarFileName, Config, Opts) when is_list(Opts) -> + tar_size(TarFileName, Config), {ChPid,_} = ?config(sftp,Config), - ok = file:set_cwd(?config(priv_dir,Config)), - file:make_dir("tar_chk"), % May already exist - ok = file:set_cwd("tar_chk"), - {ok,Data} = ssh_sftp:read_file(ChPid, fnp(?tar_file_name,Config)), - ok = file:write_file(?tar_file_name, Data), - os:cmd("tar xf "++?tar_file_name), - lists:foreach(fun(Item) -> chk_contents(Item,Config) end, - Items). - - -chk_contents({Name,ExpectBin}, _Config) -> - case file:read_file(Name) of - {ok,ExpectBin} -> - ok; - {ok,OtherBin} -> - ct:log("File: ~p~n Got: ~p~nExpect: ~p",[Name,OtherBin,ExpectBin]), - ct:fail("Bad contents in file ~p",[Name]); - Other -> - ct:log("File: ~p~nOther: ~p",[Name,Other]), - ct:fail("Error reading of file ~p",[Name]) - end; -chk_contents(Name, Config) -> - {ok,Bin} = file:read_file(fn(Name,Config)), - chk_contents({Name,Bin}, Config). + {ok,HandleRead} = ssh_sftp:open_tar(ChPid, TarFileName, [read|Opts]), + {ok,NameValueList} = erl_tar:extract(HandleRead,[memory,verbose]), + ok = erl_tar:close(HandleRead), + case {lists:sort(expand_items(Items,Config)), lists:sort(NameValueList)} of + {L,L} -> + true; + {Expect,Actual} -> + ct:log("Expect: ~p",[Expect]), ct:log("Actual: ~p",[Actual]), + case erl_tar:table(TarFileName) of + {ok,Names} -> ct:log("names: ~p",[Names]); + Other -> ct:log("~p",[Other]) + end, + ct:log("~s",[analyze_report(Expect, Actual)]), + ct:fail(bad_tar_contents) + end. +analyze_report([E={NameE,BinE}|Es], [A={NameA,BinA}|As]) -> + if + NameE == NameA, + BinE =/= BinA-> + [["Component ",NameE," differs. \n Expected: ",BinE,"\n Actual: ",BinA,"\n\n"] + | analyze_report(Es,As)]; + + NameE < NameA -> + [["Component ",NameE," is missing.\n\n"] + | analyze_report(Es,[A|As])]; + + NameE > NameA -> + [["Component ",NameA," is not expected.\n\n"] + | analyze_report([E|Es],As)]; + true -> + analyze_report(Es, As) + end; +analyze_report([{NameE,_BinE}|Es], []) -> + [["Component ",NameE," missing.\n\n"] | analyze_report(Es,[])]; +analyze_report([], [{NameA,_BinA}|As]) -> + [["Component ",NameA," not expected.\n\n"] | analyze_report([],As)]; +analyze_report([], []) -> + "". + +tar_size(TarFileName, Config) -> + {ChPid,_} = ?config(sftp,Config), + {ok,Data} = ssh_sftp:read_file(ChPid, TarFileName), + io:format('Tar file ~p is~n ~p bytes.~n',[TarFileName, size(Data)]). + +expand_items(Items, Config) -> + lists:flatten( + [case Item of + {_Name,Bin} when is_binary(Bin) -> + Item; + {Name,FileName} when is_list(FileName) -> + read_item_contents(Name, fn(FileName,Config)); + FileName when is_list(FileName) -> + read_item_contents(FileName, fn(FileName,Config)) + end || Item <- Items]). + +read_item_contents(ItemName, FileName) -> + case file:read_file(FileName) of + {ok,Bin} -> + {ItemName, Bin}; + {error,eisdir} -> + {ok,FileNames} = file:list_dir(FileName), + [read_item_contents(filename:join(ItemName,Name), + filename:join(FileName,Name)) + || Name<-FileNames] + end. fn(Name, Config) -> Dir = ?config(data_dir, Config), |