From 3005b2aadb5f756503cf6d05d604d8c64eb2c786 Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Mon, 1 Dec 2014 10:50:19 +0100 Subject: ssh: crypto framework for ssh_sftp:open_tar added --- lib/ssh/src/ssh_sftp.erl | 324 ++++++++++++++++++++++++++++++---- lib/ssh/test/ssh_sftp_SUITE.erl | 381 +++++++++++++++++++++++++++++++++------- 2 files changed, 613 insertions(+), 92 deletions(-) (limited to 'lib/ssh') diff --git a/lib/ssh/src/ssh_sftp.erl b/lib/ssh/src/ssh_sftp.erl index 3b80f5326c..613f8f25b2 100644 --- a/lib/ssh/src/ssh_sftp.erl +++ b/lib/ssh/src/ssh_sftp.erl @@ -69,6 +69,18 @@ mode }). +-record(bufinf, + { + mode, % read | write (=from or to buffer by user) + crypto_state, + crypto_fun, % For encode or decode depending on the mode field + size = 0, % # bytes "before" the current buffer for the postion call + + chunksize, % The size of the chunks to be sent or received + enc_text_buf = <<>>, % Encrypted text + plain_text_buf = <<>> % Decrypted text + }). + -define(FILEOP_TIMEOUT, infinity). -define(NEXT_REQID(S), @@ -164,24 +176,73 @@ open(Pid, File, Mode, FileOpTimeout) -> open_tar(Pid, File, Mode) -> open_tar(Pid, File, Mode, ?FILEOP_TIMEOUT). -open_tar(Pid, File, Mode=[write], FileOpTimeout) -> - {ok,R} = open(Pid, File, Mode, FileOpTimeout), - erl_tar:init({Pid,R,FileOpTimeout}, write, - fun(write, {{P,H,T},Data}) -> - Bin = if is_list(Data) -> list_to_binary(Data); - is_binary(Data) -> Data - end, - {ok,{_Window,Packet}} = send_window(P, T), - write_file_loop(P, H, 0, Bin, size(Bin), Packet, T); - (position, {{P,H,T},Pos}) -> - position(P, H, Pos, T); - (close, {P,H,T}) -> - close(P, H, T) - end); -open_tar(_Pid, _File, Mode, _FileOpTimeout) -> - {error,{illegal_mode,Mode}}. - - +open_tar(Pid, File, Mode, FileOpTimeout) -> + case {lists:member(write,Mode), + lists:member(read,Mode), + Mode -- [read,write]} of + {true,false,[]} -> + {ok,Handle} = open(Pid, File, [write], FileOpTimeout), + erl_tar:init(Pid, write, + fun(write, {_,Data}) -> + write_to_remote_tar(Pid, Handle, to_bin(Data), FileOpTimeout); + (position, {_,Pos}) -> + position(Pid, Handle, Pos, FileOpTimeout); + (close, _) -> + close(Pid, Handle, FileOpTimeout) + end); + {true,false,[{crypto,{CryptoInitFun,CryptoEncryptFun,CryptoEndFun}}]} -> + {ok,SftpHandle} = open(Pid, File, [write], FileOpTimeout), + BI = #bufinf{mode = write, + crypto_fun = CryptoEncryptFun}, + {ok,BufHandle} = open_buf(Pid, CryptoInitFun, BI, FileOpTimeout), + erl_tar:init(Pid, write, + fun(write, {_,Data}) -> + write_buf(Pid, SftpHandle, BufHandle, to_bin(Data), FileOpTimeout); + (position, {_,Pos}) -> + position_buf(Pid, SftpHandle, BufHandle, Pos, FileOpTimeout); + (close, _) -> + {ok,#bufinf{ + plain_text_buf = PlainBuf0, + enc_text_buf = EncBuf0, + crypto_state = CState0 + }} = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout), + {ok,EncTextTail} = CryptoEndFun(PlainBuf0, CState0), + EncTextBuf = <>, + case write(Pid, SftpHandle, EncTextBuf, FileOpTimeout) of + ok -> + call(Pid, {erase_bufinf,BufHandle}, FileOpTimeout), + close(Pid, SftpHandle, FileOpTimeout); + Other -> + Other + end + end); + {false,true,[]} -> + {ok,Handle} = open(Pid, File, [read,binary], FileOpTimeout), + erl_tar:init(Pid, read, + fun(read2, {_,Len}) -> + read_repeat(Pid, Handle, Len, FileOpTimeout); + (position, {_,Pos}) -> + position(Pid, Handle, Pos, FileOpTimeout); + (close, _) -> + close(Pid, Handle, FileOpTimeout) + end); + {false,true,[{crypto,{CryptoInitFun,CryptoDecryptFun}}]} -> + {ok,SftpHandle} = open(Pid, File, [read,binary], FileOpTimeout), + BI = #bufinf{mode = read, + crypto_fun = CryptoDecryptFun}, + {ok,BufHandle} = open_buf(Pid, CryptoInitFun, BI, FileOpTimeout), + erl_tar:init(Pid, read, + fun(read2, {_,Len}) -> + read_buf(Pid, SftpHandle, BufHandle, Len, FileOpTimeout); + (position, {_,Pos}) -> + position_buf(Pid, SftpHandle, BufHandle, Pos, FileOpTimeout); + (close, _) -> + call(Pid, {erase_bufinf,BufHandle}, FileOpTimeout), + close(Pid, SftpHandle, FileOpTimeout) + end); + _ -> + {error,{illegal_mode,Mode}} + end. opendir(Pid, Path) -> @@ -469,6 +530,15 @@ handle_cast(_,State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. +do_handle_call({get_bufinf,BufHandle}, _From, S=#state{inf=I0}) -> + {reply, dict:find(BufHandle,I0), S}; + +do_handle_call({put_bufinf,BufHandle,B}, _From, S=#state{inf=I0}) -> + {reply, ok, S#state{inf=dict:store(BufHandle,B,I0)}}; + +do_handle_call({erase_bufinf,BufHandle}, _From, S=#state{inf=I0}) -> + {reply, ok, S#state{inf=dict:erase(BufHandle,I0)}}; + do_handle_call({open, Async,FileName,Mode}, From, #state{xf = XF} = State) -> {Access,Flags,Attrs} = open_mode(XF#ssh_xfer.vsn, Mode), ReqID = State#state.req_id, @@ -573,12 +643,7 @@ do_handle_call({read,Async,Handle,Length}, From, State) -> do_handle_call({pwrite,Async,Handle,At,Data0}, From, State) -> case lseek_position(Handle, At, State) of {ok,Offset} -> - Data = if - is_binary(Data0) -> - Data0; - is_list(Data0) -> - list_to_binary(Data0) - end, + Data = to_bin(Data0), ReqID = State#state.req_id, Size = size(Data), ssh_xfer:write(?XF(State),ReqID,Handle,Offset,Data), @@ -591,12 +656,7 @@ do_handle_call({pwrite,Async,Handle,At,Data0}, From, State) -> do_handle_call({write,Async,Handle,Data0}, From, State) -> case lseek_position(Handle, cur, State) of {ok,Offset} -> - Data = if - is_binary(Data0) -> - Data0; - is_list(Data0) -> - list_to_binary(Data0) - end, + Data = to_bin(Data0), ReqID = State#state.req_id, Size = size(Data), ssh_xfer:write(?XF(State),ReqID,Handle,Offset,Data), @@ -1148,5 +1208,207 @@ lseek_pos({eof, Offset}, _CurOffset, CurSize) end; lseek_pos(_, _, _) -> {error, einval}. - +%%%================================================================ +%%% +to_bin(Data) when is_list(Data) -> list_to_binary(Data); +to_bin(Data) when is_binary(Data) -> Data. + + +read_repeat(Pid, Handle, Len, FileOpTimeout) -> + {ok,{_WindowSz,PacketSz}} = recv_window(Pid, FileOpTimeout), + read_rpt(Pid, Handle, Len, PacketSz, FileOpTimeout, <<>>). + +read_rpt(Pid, Handle, WantedLen, PacketSz, FileOpTimeout, Acc) when WantedLen > 0 -> + case read(Pid, Handle, min(WantedLen,PacketSz), FileOpTimeout) of + {ok, Data} -> + read_rpt(Pid, Handle, WantedLen-size(Data), PacketSz, FileOpTimeout, <>); + eof -> + {ok, Acc}; + Error -> + Error + end; +read_rpt(_Pid, _Handle, WantedLen, _PacketSz, _FileOpTimeout, Acc) when WantedLen >= 0 -> + {ok,Acc}. + + +write_to_remote_tar(_Pid, _SftpHandle, <<>>, _FileOpTimeout) -> + ok; +write_to_remote_tar(Pid, SftpHandle, Bin, FileOpTimeout) -> + {ok,{_Window,Packet}} = send_window(Pid, FileOpTimeout), + write_file_loop(Pid, SftpHandle, 0, Bin, size(Bin), Packet, FileOpTimeout). + +position_buf(Pid, SftpHandle, BufHandle, Pos, FileOpTimeout) -> + {ok,#bufinf{mode = Mode, + plain_text_buf = Buf0, + size = Size}} = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout), + case Pos of + {cur,0} when Mode==write -> + {ok,Size+size(Buf0)}; + + {cur,0} when Mode==read -> + {ok,Size}; + + _ when Mode==read, is_integer(Pos) -> + Skip = Pos-Size, + if + Skip < 0 -> + {error, cannot_rewind}; + Skip == 0 -> + %% Optimization + {ok,Pos}; + Skip > 0 -> + case read_buf(Pid, SftpHandle, BufHandle, Skip, FileOpTimeout) of + %% A bit innefficient to fetch the bufinf again, but there are lots of + %% other more important optimizations waiting.... + {ok,_} -> + {ok,Pos}; + Other -> + Other + end + end; + + _ -> + {error,{not_yet_implemented,{pos,Pos}}} + end. + +read_buf(Pid, SftpHandle, BufHandle, WantedLen, FileOpTimeout) -> + {ok,{_Window,Packet}} = send_window(Pid, FileOpTimeout), + {ok,B0} = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout), + case do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, B0) of + {ok,ResultBin,B} -> + call(Pid, {put_bufinf,BufHandle,B}, FileOpTimeout), + {ok,ResultBin}; + {error,Error} -> + {error,Error}; + {eof,B} -> + call(Pid, {put_bufinf,BufHandle,B}, FileOpTimeout), + eof + end. + +do_the_read_buf(_Pid, _SftpHandle, WantedLen, _Packet, _FileOpTimeout, + B=#bufinf{plain_text_buf=PlainBuf0, + size = Size}) + when size(PlainBuf0) >= WantedLen -> + %% We already have the wanted number of bytes decoded and ready! + <> = PlainBuf0, + {ok,ResultBin,B#bufinf{plain_text_buf=PlainBuf, + size = Size + WantedLen}}; + +do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, + B0=#bufinf{plain_text_buf = PlainBuf0, + enc_text_buf = EncBuf0, + chunksize = undefined + }) + when size(EncBuf0) > 0 -> + %% We have (at least) one decodable byte waiting for decodeing. + {ok,DecodedBin,B} = apply_crypto(EncBuf0, B0), + do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, + B#bufinf{plain_text_buf = <>, + enc_text_buf = <<>> + }); + +do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, + B0=#bufinf{plain_text_buf = PlainBuf0, + enc_text_buf = EncBuf0, + chunksize = ChunkSize0 + }) + when size(EncBuf0) >= ChunkSize0 -> + %% We have (at least) one chunk of decodable bytes waiting for decodeing. + <> = EncBuf0, + {ok,DecodedBin,B} = apply_crypto(ToDecode, B0), + do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, + B#bufinf{plain_text_buf = <>, + enc_text_buf = EncBuf + }); + +do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, B=#bufinf{enc_text_buf = EncBuf0}) -> + %% We must read more bytes and append to the buffer of encoded bytes. + case read(Pid, SftpHandle, Packet, FileOpTimeout) of + {ok,EncryptedBin} -> + do_the_read_buf(Pid, SftpHandle, WantedLen, Packet, FileOpTimeout, + B#bufinf{enc_text_buf = <>}); + eof -> + {eof,B}; + Other -> + Other + end. + + +write_buf(Pid, SftpHandle, BufHandle, PlainBin, FileOpTimeout) -> + {ok,{_Window,Packet}} = send_window(Pid, FileOpTimeout), + {ok,B0=#bufinf{plain_text_buf=PTB}} = call(Pid, {get_bufinf,BufHandle}, FileOpTimeout), + case do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B0#bufinf{plain_text_buf = <>}) of + {ok, B} -> + call(Pid, {put_bufinf,BufHandle,B}, FileOpTimeout), + ok; + {error,Error} -> + {error,Error} + end. + +do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B=#bufinf{enc_text_buf = EncBuf0, + size = Size}) + when size(EncBuf0) >= Packet -> + <> = EncBuf0, + case write(Pid, SftpHandle, BinToWrite, FileOpTimeout) of + ok -> + do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B#bufinf{enc_text_buf = EncBuf, + size = Size + Packet}); + Other -> + Other + end; + +do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B0=#bufinf{plain_text_buf = PlainBuf0, + enc_text_buf = EncBuf0, + chunksize = undefined}) + when size(PlainBuf0) > 0 -> + {ok,EncodedBin,B} = apply_crypto(PlainBuf0, B0), + do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B#bufinf{plain_text_buf = <<>>, + enc_text_buf = <>}); + +do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B0=#bufinf{plain_text_buf = PlainBuf0, + enc_text_buf = EncBuf0, + chunksize = ChunkSize0 + }) + when size(PlainBuf0) >= ChunkSize0 -> + <> = PlainBuf0, + {ok,EncodedBin,B} = apply_crypto(ToEncode, B0), + do_the_write_buf(Pid, SftpHandle, Packet, FileOpTimeout, + B#bufinf{plain_text_buf = PlainBuf, + enc_text_buf = <>}); + +do_the_write_buf(_Pid, _SftpHandle, _Packet, _FileOpTimeout, B) -> + {ok,B}. + +apply_crypto(In, B=#bufinf{crypto_state = CState0, + crypto_fun = F}) -> + case F(In,CState0) of + {ok,EncodedBin,CState} -> + {ok, EncodedBin, B#bufinf{crypto_state=CState}}; + {ok,EncodedBin,CState,ChunkSize} -> + {ok, EncodedBin, B#bufinf{crypto_state=CState, + chunksize=ChunkSize}} + end. + +open_buf(Pid, CryptoInitFun, BufInfo0, FileOpTimeout) -> + case CryptoInitFun() of + {ok,CryptoState} -> + open_buf1(Pid, BufInfo0, FileOpTimeout, CryptoState, undefined); + {ok,CryptoState,ChunkSize} -> + open_buf1(Pid, BufInfo0, FileOpTimeout, CryptoState, ChunkSize); + Other -> + Other + end. + +open_buf1(Pid, BufInfo0, FileOpTimeout, CryptoState, ChunkSize) -> + BufInfo = BufInfo0#bufinf{crypto_state = CryptoState, + chunksize = ChunkSize}, + BufHandle = make_ref(), + call(Pid, {put_bufinf,BufHandle,BufInfo}, FileOpTimeout), + {ok,BufHandle}. diff --git a/lib/ssh/test/ssh_sftp_SUITE.erl b/lib/ssh/test/ssh_sftp_SUITE.erl index 559fa721fd..cb74a27638 100644 --- a/lib/ssh/test/ssh_sftp_SUITE.erl +++ b/lib/ssh/test/ssh_sftp_SUITE.erl @@ -65,19 +65,25 @@ end_per_suite(Config) -> %%-------------------------------------------------------------------- groups() -> [{erlang_server, [], [open_close_file, open_close_dir, read_file, read_dir, - write_file, write_big_file, rename_file, mk_rm_dir, remove_file, links, + write_file, write_big_file, sftp_read_big_file, + rename_file, mk_rm_dir, remove_file, links, retrieve_attributes, set_attributes, async_read, async_write, position, pos_read, pos_write, version_option, - {group,remote_tar_write} - ]}, + {group,remote_tar}]}, + {openssh_server, [], [open_close_file, open_close_dir, read_file, read_dir, - write_file, write_big_file, rename_file, mk_rm_dir, remove_file, links, + write_file, write_big_file, sftp_read_big_file, + rename_file, mk_rm_dir, remove_file, links, retrieve_attributes, set_attributes, async_read, async_write, position, pos_read, pos_write, - {group,remote_tar_write}]}, - - {remote_tar_write, [], [create_empty_tar, files_to_tar, big_file_to_tar, files_chunked_to_tar, - directory_to_tar, binaries_to_tar]} + {group,remote_tar}]}, + + {remote_tar, [], [create_empty_tar, files_to_tar, big_file_to_tar, files_chunked_to_tar, + directory_to_tar, binaries_to_tar, null_crypto_tar, + simple_crypto_tar_small, simple_crypto_tar_big, + read_tar, read_null_crypto_tar, read_crypto_tar, + aes_cbc256_crypto_tar, aes_ctr_stream_crypto_tar + ]} ]. @@ -104,7 +110,7 @@ init_per_group(openssh_server, Config) -> {skip, "No openssh server"} end; -init_per_group(remote_tar_write, Config) -> +init_per_group(remote_tar, Config) -> {Host,Port} = ?config(peer, Config), ct:log("Server (~p) at ~p:~p",[?config(group,Config),Host,Port]), {ok, Connection} = @@ -120,7 +126,7 @@ init_per_group(remote_tar_write, Config) -> [{user_interaction, false}, {silently_accept_hosts, true}]) end, - [{remote_tar_write, true}, + [{remote_tar, true}, {connection, Connection} | Config]. end_per_group(erlang_server, Config) -> @@ -187,16 +193,12 @@ init_per_testcase(Case, Config0) -> [{sftp, Sftp}, {watchdog, Dog} | Config2] end, - case catch ?config(remote_tar_write,Config) of + case catch ?config(remote_tar,Config) of %% The 'catch' is for the case of Config={skip,...} true -> - %% Provide a tar Handle *independent* of the sftp-channel already opened! - %% This Handle will be closed (as well as ChannelPid2) in the testcase - {ok,ChannelPid2} = - ssh_sftp:start_channel(?config(connection,Config)), - {ok,Handle} = - ssh_sftp:open_tar(ChannelPid2, fnp(?tar_file_name,Config), [write]), - [{handle,Handle} | Config]; + %% Provide a ChannelPid independent of the sftp-channel already opened. + {ok,ChPid2} = ssh_sftp:start_channel(?config(connection,Config)), + [{channel_pid2,ChPid2} | Config]; _ -> Config end. @@ -214,6 +216,7 @@ end_per_testcase(_, Config) -> end_per_testcase(Config) -> {Sftp, Connection} = ?config(sftp, Config), ssh_sftp:stop_channel(Sftp), + catch ssh_sftp:stop_channel(?config(channel_pid2, Config)), ssh:close(Connection). %%-------------------------------------------------------------------- @@ -258,6 +261,7 @@ read_file(Config) when is_list(Config) -> FileName = filename:join(PrivDir, "sftp.txt"), {Sftp, _} = ?config(sftp, Config), {ok, Data} = ssh_sftp:read_file(Sftp, FileName), + {ok, Data} = ssh_sftp:read_file(Sftp, FileName), {ok, Data} = file:read_file(FileName). %%-------------------------------------------------------------------- @@ -293,6 +297,19 @@ write_big_file(Config) when is_list(Config) -> ssh_sftp:write_file(Sftp, FileName, [Data]), {ok, Data} = file:read_file(FileName). +%%-------------------------------------------------------------------- +sftp_read_big_file() -> + [{doc, "Test API function read_file/2 with big data"}]. +sftp_read_big_file(Config) when is_list(Config) -> + PrivDir = ?config(priv_dir, Config), + FileName = filename:join(PrivDir, "sftp.txt"), + {Sftp, _} = ?config(sftp, Config), + + Data = list_to_binary(lists:duplicate(750000,"a")), + ct:log("Data size to write is ~p bytes",[size(Data)]), + ssh_sftp:write_file(Sftp, FileName, [Data]), + {ok, Data} = ssh_sftp:read_file(Sftp, FileName). + %%-------------------------------------------------------------------- remove_file() -> [{doc,"Test API function delete/2"}]. @@ -527,52 +544,246 @@ version_option(Config) when is_list(Config) -> %%-------------------------------------------------------------------- create_empty_tar(Config) -> - {ChPid,_} = ?config(sftp,Config), - Handle = ?config(handle,Config), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), erl_tar:close(Handle), + {ChPid,_} = ?config(sftp,Config), {ok, #file_info{type=regular}} = ssh_sftp:read_file_info(ChPid,fnp(?tar_file_name,Config)). - + %%-------------------------------------------------------------------- files_to_tar(Config) -> - Handle = ?config(handle,Config), - ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", []), - ok = erl_tar:add(Handle, fn("f2.txt",Config), "f2.txt", []), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), + ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose]), + ok = erl_tar:add(Handle, fn("f2.txt",Config), "f2.txt", [verbose]), ok = erl_tar:close(Handle), chk_tar(["f1.txt", "f2.txt"], Config). - %%-------------------------------------------------------------------- big_file_to_tar(Config) -> - Handle = ?config(handle,Config), - ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", []), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), + ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [verbose]), ok = erl_tar:close(Handle), chk_tar(["big.txt"], Config). %%-------------------------------------------------------------------- files_chunked_to_tar(Config) -> - Handle = ?config(handle,Config), - ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [{chunks,2}]), - ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [{chunks,15000}]), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), + ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]), ok = erl_tar:close(Handle), - chk_tar(["f1.txt", "big.txt"], Config). + chk_tar(["f1.txt"], Config). %%-------------------------------------------------------------------- directory_to_tar(Config) -> - Handle = ?config(handle,Config), - ok = erl_tar:add(Handle, fn("d1",Config), "d1", []), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), + ok = erl_tar:add(Handle, fn("d1",Config), "d1", [verbose]), ok = erl_tar:close(Handle), - chk_tar(["d1/f1", "d1/f2"], Config). + chk_tar(["d1"], Config). %%-------------------------------------------------------------------- binaries_to_tar(Config) -> - Handle = ?config(handle,Config), + ChPid2 = ?config(channel_pid2, Config), + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), Bin = <<"A binary">>, - ok = erl_tar:add(Handle, Bin, "b1", []), + ok = erl_tar:add(Handle, Bin, "b1", [verbose]), ok = erl_tar:close(Handle), chk_tar([{"b1",Bin}], Config). +%%-------------------------------------------------------------------- +null_crypto_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + Cinit = fun() -> {ok, no_state, _SendSize=5} end, + Cenc = fun(Bin,CState) -> {ok,Bin,CState,_SendSize=5} end, + Cend = fun(Bin,_CState) -> {ok,Bin} end, + C = {Cinit,Cenc,Cend}, + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,C}]), + Bin = <<"A binary">>, + ok = erl_tar:add(Handle, Bin, "b1", [verbose]), + ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]), + ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [verbose,{chunks,15000}]), + ok = erl_tar:close(Handle), + chk_tar([{"b1",Bin}, "f1.txt", "big.txt"], Config). + +%%-------------------------------------------------------------------- +simple_crypto_tar_small(Config) -> + ChPid2 = ?config(channel_pid2, Config), + Cinit = fun() -> {ok, no_state, _Size=6} end, + Cenc = fun(Bin,CState) -> {ok,stuff(Bin),CState,_SendSize=5} end, + Cdec = fun(Bin,CState) -> {ok,unstuff(Bin),CState,_Size=4} end, + Cend = fun(Bin,_CState) -> {ok,stuff(Bin)} end, + C = {Cinit,Cenc,Cend}, + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,C}]), + Bin = <<"A binary">>, + ok = erl_tar:add(Handle, Bin, "b1", [verbose]), + ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]), + ok = erl_tar:close(Handle), + chk_tar([{"b1",Bin}, "f1.txt"], Config, [{crypto,{Cinit,Cdec}}]). + +%%-------------------------------------------------------------------- +simple_crypto_tar_big(Config) -> + ChPid2 = ?config(channel_pid2, Config), + Cinit = fun() -> {ok, no_state, _SendSize=6} end, + Cenc = fun(Bin,CState) -> {ok,stuff(Bin),CState,_SendSize=5} end, + Cdec = fun(Bin,CState) -> {ok,unstuff(Bin),CState,_SendSize=4} end, + Cend = fun(Bin,_CState) -> {ok,stuff(Bin)} end, + C = {Cinit,Cenc,Cend}, + {ok,Handle} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,C}]), + Bin = <<"A binary">>, + ok = erl_tar:add(Handle, Bin, "b1", [verbose]), + ok = erl_tar:add(Handle, fn("f1.txt",Config), "f1.txt", [verbose,{chunks,2}]), + ok = erl_tar:add(Handle, fn("big.txt",Config), "big.txt", [verbose,{chunks,15000}]), + ok = erl_tar:close(Handle), + chk_tar([{"b1",Bin}, "f1.txt", "big.txt"], Config, [{crypto,{Cinit,Cdec}}]). + +stuff(Bin) -> << <> || <> <= Bin >>. + +unstuff(Bin) -> << <> || <> <= Bin >>. + +%%-------------------------------------------------------------------- +read_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + NameBins = lists:sort( + [{"b1",<<"A binary">>}, + {"b2",list_to_binary(lists:duplicate(750000,"a"))} + ]), + {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write]), + [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) + || {Name,Bin} <- NameBins], + ok = erl_tar:close(HandleWrite), + + chk_tar(NameBins, Config). + +%%-------------------------------------------------------------------- +read_null_crypto_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + NameBins = lists:sort( + [{"b1",<<"A binary">>}, + {"b2",list_to_binary(lists:duplicate(750000,"a"))} + ]), + Cinitw = fun() -> {ok, no_state, _SendSize=5} end, + Cinitr = fun() -> {ok, no_state, _FetchSize=42} end, + Cenc = fun(Bin,CState) -> {ok,Bin,CState,_SendSize=42*42} end, + Cdec = fun(Bin,CState) -> {ok,Bin,CState,_FetchSize=19} end, + Cendw = fun(Bin,_CState) -> {ok,Bin} end, + Cw = {Cinitw,Cenc,Cendw}, + Cr = {Cinitr,Cdec}, + + {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]), + [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) + || {Name,Bin} <- NameBins], + ok = erl_tar:close(HandleWrite), + + chk_tar(NameBins, Config, [{crypto,Cr}]). + +%%-------------------------------------------------------------------- +read_crypto_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + NameBins = lists:sort( + [{"b1",<<"A binary">>}, + {"b2",list_to_binary(lists:duplicate(750000,"a"))} + ]), + Cinitw = fun() -> {ok, no_state, _SendSize=5} end, + Cinitr = fun() -> {ok, no_state, _FetchSize=42} end, + + Cenc = fun(Bin,CState) -> {ok,stuff(Bin),CState,_SendSize=42*42} end, + Cdec = fun(Bin,CState) -> {ok,unstuff(Bin),CState,_FetchSize=120} end, + Cendw = fun(Bin,_CState) -> {ok,stuff(Bin)} end, + Cw = {Cinitw,Cenc,Cendw}, + Cr = {Cinitr,Cdec}, + + {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]), + [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) + || {Name,Bin} <- NameBins], + ok = erl_tar:close(HandleWrite), + + chk_tar(NameBins, Config, [{crypto,Cr}]). + +%%-------------------------------------------------------------------- +aes_cbc256_crypto_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + NameBins = lists:sort( + [{"b1",<<"A binary">>}, + {"b2",list_to_binary(lists:duplicate(750000,"a"))}, + {"d1",fn("d1",Config)} % Dir + ]), + Key = <<"This is a 256 bit key. Boring...">>, + Ivec0 = crypto:rand_bytes(16), + DataSize = 1024, % data_size rem 16 = 0 for aes_cbc + + Cinitw = fun() -> {ok, Ivec0, DataSize} end, + Cinitr = fun() -> {ok, Ivec0, DataSize} end, + + Cenc = fun(PlainBin,Ivec) -> + CipherBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, PlainBin), + {ok, CipherBin, crypto:next_iv(aes_cbc,CipherBin), DataSize} + end, + Cdec = fun(CipherBin,Ivec) -> + PlainBin = crypto:block_decrypt(aes_cbc256, Key, Ivec, CipherBin), + {ok, PlainBin, crypto:next_iv(aes_cbc,CipherBin), DataSize} + end, + + Cendw = fun(PlainBin, _) when PlainBin == <<>> -> {ok, <<>>}; + (PlainBin, Ivec) -> + CipherBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, + pad(16,PlainBin)), %% Last chunk + {ok, CipherBin} + end, + + Cw = {Cinitw,Cenc,Cendw}, + {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]), + [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) || {Name,Bin} <- NameBins], + ok = erl_tar:close(HandleWrite), + + Cr = {Cinitr,Cdec}, + chk_tar(NameBins, Config, [{crypto,Cr}]). + + +pad(BlockSize, Bin) -> + PadSize = (BlockSize - (size(Bin) rem BlockSize)) rem BlockSize, + list_to_binary( lists:duplicate(PadSize,0) ). + +%%-------------------------------------------------------------------- +aes_ctr_stream_crypto_tar(Config) -> + ChPid2 = ?config(channel_pid2, Config), + NameBins = lists:sort( + [{"b1",<<"A binary">>}, + {"b2",list_to_binary(lists:duplicate(750000,"a"))}, + {"d1",fn("d1",Config)} % Dir + ]), + Key = <<"This is a 256 bit key. Boring...">>, + Ivec0 = crypto:rand_bytes(16), + + Cinitw = Cinitr = fun() -> {ok, crypto:stream_init(aes_ctr,Key,Ivec0)} end, + + Cenc = fun(PlainBin,State) -> + {NewState,CipherBin} = crypto:stream_encrypt(State, PlainBin), + {ok, CipherBin, NewState} + end, + Cdec = fun(CipherBin,State) -> + {NewState,PlainBin} = crypto:stream_decrypt(State, CipherBin), + {ok, PlainBin, NewState} + end, + + Cendw = fun(PlainBin, _) when PlainBin == <<>> -> {ok, <<>>}; + (PlainBin, Ivec) -> + CipherBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, + pad(16,PlainBin)), %% Last chunk + {ok, CipherBin} + end, + + Cw = {Cinitw,Cenc,Cendw}, + {ok,HandleWrite} = ssh_sftp:open_tar(ChPid2, fnp(?tar_file_name,Config), [write,{crypto,Cw}]), + [ok = erl_tar:add(HandleWrite, Bin, Name, [verbose]) || {Name,Bin} <- NameBins], + ok = erl_tar:close(HandleWrite), + + Cr = {Cinitr,Cdec}, + chk_tar(NameBins, Config, [{crypto,Cr}]). + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -597,34 +808,82 @@ prep(Config) -> FileInfo#file_info{mode = Mode}). + chk_tar(Items, Config) -> - %% FIXME: ought to check that no more than expected is present... + chk_tar(Items, Config, []). + +chk_tar(Items, Config, Opts) -> + chk_tar(Items, fnp(?tar_file_name,Config), Config, Opts). + +chk_tar(Items, TarFileName, Config, Opts) when is_list(Opts) -> + tar_size(TarFileName, Config), {ChPid,_} = ?config(sftp,Config), - ok = file:set_cwd(?config(priv_dir,Config)), - file:make_dir("tar_chk"), % May already exist - ok = file:set_cwd("tar_chk"), - {ok,Data} = ssh_sftp:read_file(ChPid, fnp(?tar_file_name,Config)), - ok = file:write_file(?tar_file_name, Data), - os:cmd("tar xf "++?tar_file_name), - lists:foreach(fun(Item) -> chk_contents(Item,Config) end, - Items). - - -chk_contents({Name,ExpectBin}, _Config) -> - case file:read_file(Name) of - {ok,ExpectBin} -> - ok; - {ok,OtherBin} -> - ct:log("File: ~p~n Got: ~p~nExpect: ~p",[Name,OtherBin,ExpectBin]), - ct:fail("Bad contents in file ~p",[Name]); - Other -> - ct:log("File: ~p~nOther: ~p",[Name,Other]), - ct:fail("Error reading of file ~p",[Name]) - end; -chk_contents(Name, Config) -> - {ok,Bin} = file:read_file(fn(Name,Config)), - chk_contents({Name,Bin}, Config). + {ok,HandleRead} = ssh_sftp:open_tar(ChPid, TarFileName, [read|Opts]), + {ok,NameValueList} = erl_tar:extract(HandleRead,[memory,verbose]), + ok = erl_tar:close(HandleRead), + case {lists:sort(expand_items(Items,Config)), lists:sort(NameValueList)} of + {L,L} -> + true; + {Expect,Actual} -> + ct:log("Expect: ~p",[Expect]), ct:log("Actual: ~p",[Actual]), + case erl_tar:table(TarFileName) of + {ok,Names} -> ct:log("names: ~p",[Names]); + Other -> ct:log("~p",[Other]) + end, + ct:log("~s",[analyze_report(Expect, Actual)]), + ct:fail(bad_tar_contents) + end. +analyze_report([E={NameE,BinE}|Es], [A={NameA,BinA}|As]) -> + if + NameE == NameA, + BinE =/= BinA-> + [["Component ",NameE," differs. \n Expected: ",BinE,"\n Actual: ",BinA,"\n\n"] + | analyze_report(Es,As)]; + + NameE < NameA -> + [["Component ",NameE," is missing.\n\n"] + | analyze_report(Es,[A|As])]; + + NameE > NameA -> + [["Component ",NameA," is not expected.\n\n"] + | analyze_report([E|Es],As)]; + true -> + analyze_report(Es, As) + end; +analyze_report([{NameE,_BinE}|Es], []) -> + [["Component ",NameE," missing.\n\n"] | analyze_report(Es,[])]; +analyze_report([], [{NameA,_BinA}|As]) -> + [["Component ",NameA," not expected.\n\n"] | analyze_report([],As)]; +analyze_report([], []) -> + "". + +tar_size(TarFileName, Config) -> + {ChPid,_} = ?config(sftp,Config), + {ok,Data} = ssh_sftp:read_file(ChPid, TarFileName), + io:format('Tar file ~p is~n ~p bytes.~n',[TarFileName, size(Data)]). + +expand_items(Items, Config) -> + lists:flatten( + [case Item of + {_Name,Bin} when is_binary(Bin) -> + Item; + {Name,FileName} when is_list(FileName) -> + read_item_contents(Name, fn(FileName,Config)); + FileName when is_list(FileName) -> + read_item_contents(FileName, fn(FileName,Config)) + end || Item <- Items]). + +read_item_contents(ItemName, FileName) -> + case file:read_file(FileName) of + {ok,Bin} -> + {ItemName, Bin}; + {error,eisdir} -> + {ok,FileNames} = file:list_dir(FileName), + [read_item_contents(filename:join(ItemName,Name), + filename:join(FileName,Name)) + || Name<-FileNames] + end. fn(Name, Config) -> Dir = ?config(data_dir, Config), -- cgit v1.2.3 From c5a526735109a27d919b340148db6a5a99f9ad09 Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Tue, 25 Nov 2014 10:52:24 +0100 Subject: ssh: Implements and tests erl_tar read from remote host. --- lib/ssh/doc/src/ssh_sftp.xml | 106 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 6 deletions(-) (limited to 'lib/ssh') diff --git a/lib/ssh/doc/src/ssh_sftp.xml b/lib/ssh/doc/src/ssh_sftp.xml index 251f5a4be3..ab111562f9 100644 --- a/lib/ssh/doc/src/ssh_sftp.xml +++ b/lib/ssh/doc/src/ssh_sftp.xml @@ -196,19 +196,113 @@ - open_tar(ChannelPid, Path) -> - open_tar(ChannelPid, Path, Timeout) -> {ok, Handle} | {error, Reason} - Open a tar file on the server to which ChannelPid is connected and return a handle + open_tar(ChannelPid, Path, Mode) -> + open_tar(ChannelPid, Path, Mode, Timeout) -> {ok, Handle} | {error, Reason} + Opens a tar file on the server to which ChannelPid is connected and returns a handle ChannelPid = pid() Path = string() + Mode = [read] | [write] | [read,EncryptOpt] | [write,DecryptOpt] + EncryptOpt = {crypto,{InitFun,EncryptFun,CloseFun}} + DecryptOpt = {crypto,{InitFun,DecryptFun}} + InitFun = (fun() -> {ok,CryptoState}) | (fun() -> {ok,CryptoState,ChunkSize}) + CryptoState = any() + ChunkSize = undefined | pos_integer() + EncryptFun = (fun(PlainBin,CryptoState) -> EncryptResult) + EncryptResult = {ok,EncryptedBin,CryptoState} | {ok,EncryptedBin,CryptoState,ChunkSize} + PlainBin = binary() + EncryptedBin = binary() + DecryptFun = (fun(EncryptedBin,CryptoState) -> DecryptResult) + DecryptResult = {ok,PlainBin,CryptoState} | {ok,PlainBin,CryptoState,ChunkSize} + CloseFun = (fun(PlainBin,CryptoState) -> {ok,EncryptedBin}) Timeout = timeout() Reason = term() -

Opens a handle to a tar file on the server, the handle - can be used for remote tar manipulation as defined by the - erl_tar:init/3 function.

+

Opens a handle to a tar file on the server associated with ChannelPid. The handle + can be used for remote tar creation and extraction as defined by the + erl_tar:init/3 function. +

+

An example of writing and then reading a tar file:

+ + {ok,HandleWrite} = ssh_sftp:open_tar(ChannelPid, ?tar_file_name, [write]), + ok = erl_tar:add(HandleWrite, .... ), + ok = erl_tar:add(HandleWrite, .... ), + ... + ok = erl_tar:add(HandleWrite, .... ), + ok = erl_tar:close(HandleWrite), + + %% And for reading + {ok,HandleRead} = ssh_sftp:open_tar(ChannelPid, ?tar_file_name, [read]), + {ok,NameValueList} = erl_tar:extract(HandleRead,[memory]), + ok = erl_tar:close(HandleRead), + + +

The crypto mode option is applied to the generated stream of bytes just prior to sending + them to the sftp server. This is intended for encryption but could of course be used for other + purposes. +

+

The InitFun is applied once + prior to any other crypto operation. The returned CryptoState is then folded into + repeated applications of the EncryptFun or DecryptFun. The binary returned + from those Funs are sent further to the remote sftp server. Finally - if doing encryption + - the CloseFun is applied to the last piece of data. The CloseFun is + responsible for padding (if needed) and encryption of that last piece. +

+

The ChunkSize defines the size of the PlainBins that EncodeFun is applied + to. If the ChunkSize is undefined the size of the PlainBins varies because + this is inteded for stream crypto while a fixed ChunkSize is intended for block crypto. It + is possible to change the ChunkSizes in the return from the EncryptFun or + DecryptFun. It is in fact possible to change the value between pos_integer() and + undefined. +

+

The write and read example above can be extended with encryption and decryption:

+ + %% First three parameters depending on which crypto type we select: + Key = <<"This is a 256 bit key. abcdefghi">>, + Ivec0 = crypto:rand_bytes(16), + DataSize = 1024, % DataSize rem 16 = 0 for aes_cbc + + %% Initialization of the CryptoState, in this case it is the Ivector. + InitFun = fun() -> {ok, Ivec0, DataSize} end, + + %% How to encrypt: + EncryptFun = + fun(PlainBin,Ivec) -> + EncryptedBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, PlainBin), + {ok, EncryptedBin, crypto:next_iv(aes_cbc,EncryptedBin)} + end, + + %% What to do with the very last block: + CloseFun = + fun(PlainBin, Ivec) -> + EncryptedBin = crypto:block_encrypt(aes_cbc256, Key, Ivec, + pad(16,PlainBin) %% Last chunk + ), + {ok, EncryptedBin} + end, + + Cw = {InitFun,EncryptFun,CloseFun}, + {ok,HandleWrite} = ssh_sftp:open_tar(ChannelPid, ?tar_file_name, [write,{crypto,Cw}]), + ok = erl_tar:add(HandleWrite, .... ), + ok = erl_tar:add(HandleWrite, .... ), + ... + ok = erl_tar:add(HandleWrite, .... ), + ok = erl_tar:close(HandleWrite), + + %% And for decryption (in this crypto example we could use the same InitFun + %% as for encryption): + DecryptFun = + fun(EncryptedBin,Ivec) -> + PlainBin = crypto:block_decrypt(aes_cbc256, Key, Ivec, EncryptedBin), + {ok, PlainBin, crypto:next_iv(aes_cbc,EncryptedBin)} + end, + + Cr = {InitFun,DecryptFun}, + {ok,HandleRead} = ssh_sftp:open_tar(ChannelPid, ?tar_file_name, [read,{crypto,Cw}]), + {ok,NameValueList} = erl_tar:extract(HandleRead,[memory]), + ok = erl_tar:close(HandleRead), + -- cgit v1.2.3