From f536f86ebe3face7164c63d7455389968be94203 Mon Sep 17 00:00:00 2001 From: Thomas Arts Date: Thu, 11 Sep 2014 10:12:11 +0200 Subject: Passing global var to QuickCheck statemachine The data_dir used by the tests is given at runtime. This has as a disadvantage that the generate test has a hardcoded data_dir in it (ssh_eqc_client_server_dirs below): [{set,{var,1}, {call,ssh_eqc_client_server,initial_state, [{state,false,[],[],[],[],"ssh_eqc_client_server_dirs"}]}}, {set,{var,2}, {call,ssh_eqc_client_server,ssh_server, [{{127,1,1,1}, {call,ssh_eqc_client_server,inet_port,[{127,1,1,1}]}}, "ssh_eqc_client_server_dirs", [{parallel_login,true}]]}}, Re-running this tests on another machine works, since the path is relative, but if it were absolute, it would have been hard. Instead, we may use a symbolic representation of the data_dir and fill it in each time one runs the property, thus even when one does a check or recheck. The key to this is to use a variable in the test and bind the variable in the place where one runs the commands by using the environment variable feature of run_commands. Conflicts: lib/ssh/test/property_test/ssh_eqc_client_server.erl --- .../test/property_test/ssh_eqc_client_server.erl | 25 +++++++++++----------- 1 file changed, 12 insertions(+), 13 deletions(-) (limited to 'lib/ssh') diff --git a/lib/ssh/test/property_test/ssh_eqc_client_server.erl b/lib/ssh/test/property_test/ssh_eqc_client_server.erl index cf895ae85e..40782cd034 100644 --- a/lib/ssh/test/property_test/ssh_eqc_client_server.erl +++ b/lib/ssh/test/property_test/ssh_eqc_client_server.erl @@ -105,9 +105,9 @@ prop_seq(CT_Config) -> do_prop_seq(DataDir) -> - ?FORALL(Cmds,commands(?MODULE, #state{data_dir=DataDir}), + ?FORALL(Cmds,commands(?MODULE), begin - {H,Sf,Result} = run_commands(?MODULE,Cmds), + {H,Sf,Result} = run_commands(?MODULE,Cmds,[{data_dir,DataDir}]), present_result(?MODULE, Cmds, {H,Sf,Result}, Result==ok) end). @@ -123,9 +123,9 @@ prop_parallel(CT_Config) -> do_prop_parallel(full_path(?SSH_DIR, CT_Config)). do_prop_parallel(DataDir) -> - ?FORALL(Cmds,parallel_commands(?MODULE, #state{data_dir=DataDir}), + ?FORALL(Cmds,parallel_commands(?MODULE), begin - {H,Sf,Result} = run_parallel_commands(?MODULE,Cmds), + {H,Sf,Result} = run_parallel_commands(?MODULE,Cmds,[{data_dir,DataDir}]), present_result(?MODULE, Cmds, {H,Sf,Result}, Result==ok) end). @@ -139,10 +139,10 @@ prop_parallel_multi(CT_Config) -> do_prop_parallel_multi(DataDir) -> ?FORALL(Repetitions,?SHRINK(1,[10]), - ?FORALL(Cmds,parallel_commands(?MODULE, #state{data_dir=DataDir}), + ?FORALL(Cmds,parallel_commands(?MODULE), ?ALWAYS(Repetitions, begin - {H,Sf,Result} = run_parallel_commands(?MODULE,Cmds), + {H,Sf,Result} = run_parallel_commands(?MODULE,Cmds,[{data_dir,DataDir}]), present_result(?MODULE, Cmds, {H,Sf,Result}, Result==ok) end))). @@ -151,14 +151,13 @@ do_prop_parallel_multi(DataDir) -> %%% called when using commands/1 initial_state() -> - S = initial_state(#state{}), - S#state{initialized=true}. + #state{}. %%% called when using commands/2 -initial_state(S) -> +initial_state(DataDir) -> application:stop(ssh), ssh:start(), - setup_rsa(S#state.data_dir). + setup_rsa(DataDir). %%%---------------- weight(S, ssh_send) -> 5*length([C || C<-S#state.channels, has_subsyst(C)]); @@ -172,7 +171,7 @@ weight(_S, _) -> 1. initial_state_pre(S) -> not S#state.initialized. -initial_state_args(S) -> [S]. +initial_state_args(S) -> [{var,data_dir}]. initial_state_next(S, _, _) -> S#state{initialized=true}. @@ -183,7 +182,7 @@ initial_state_next(S, _, _) -> S#state{initialized=true}. ssh_server_pre(S) -> S#state.initialized andalso length(S#state.servers) < ?MAX_NUM_SERVERS. -ssh_server_args(S) -> [?SERVER_ADDRESS, S#state.data_dir, ?SERVER_EXTRA_OPTIONS]. +ssh_server_args(S) -> [?SERVER_ADDRESS, {var,data_dir}, ?SERVER_EXTRA_OPTIONS]. ssh_server({IP,Port}, DataDir, ExtraOptions) -> ok(ssh:daemon(IP, Port, @@ -241,7 +240,7 @@ do(Pid, Fun, Timeout) when is_function(Fun,0) -> ssh_open_connection_pre(S) -> S#state.servers /= []. -ssh_open_connection_args(S) -> [oneof(S#state.servers), S#state.data_dir]. +ssh_open_connection_args(S) -> [oneof(S#state.servers), {var,data_dir}]. ssh_open_connection(#srvr{address=Ip, port=Port}, DataDir) -> ok(ssh:connect(ensure_string(Ip), Port, -- cgit v1.2.3 From fccdd825e41006b36137cebf4b2059682f82c2eb Mon Sep 17 00:00:00 2001 From: Thomas Arts Date: Thu, 11 Sep 2014 19:52:21 +0200 Subject: No side effects in function arguments We want to be able to see tests with eqc_gen:sample, we also want to be able to re-run tests. Side effects are no good idea if these goals need to be met. Therefore, we replace the side effect to ask for a port by making a symbolic call of this. Nicer solutions are possible, but at least we can now re-run the test case, and therefore shrink. --- lib/ssh/test/property_test/ssh_eqc_client_server.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/ssh') diff --git a/lib/ssh/test/property_test/ssh_eqc_client_server.erl b/lib/ssh/test/property_test/ssh_eqc_client_server.erl index 40782cd034..3fc7a15dd0 100644 --- a/lib/ssh/test/property_test/ssh_eqc_client_server.erl +++ b/lib/ssh/test/property_test/ssh_eqc_client_server.erl @@ -75,7 +75,7 @@ -define(SUBSYSTEMS, ["echo1", "echo2", "echo3", "echo4"]). --define(SERVER_ADDRESS, { {127,1,1,1}, inet_port({127,1,1,1}) }). +-define(SERVER_ADDRESS, { {127,1,1,1}, {call, ?MODULE, inet_port, [{127,1,1,1}]} }). -define(SERVER_EXTRA_OPTIONS, [{parallel_login,bool()}] ). -- cgit v1.2.3 From 14418409c971cf20f5db35f8ff33713b1924c20a Mon Sep 17 00:00:00 2001 From: Thomas Arts Date: Thu, 11 Sep 2014 20:28:23 +0200 Subject: Reduce the total testing time for the properties to 20 seconds We need to make a global budget for testing on CI server and then define for each property how much it may use of that time. Probably in auxiliary file, such that it can be computed from what has changed in the repo. --- lib/ssh/test/property_test/ssh_eqc_client_server.erl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'lib/ssh') diff --git a/lib/ssh/test/property_test/ssh_eqc_client_server.erl b/lib/ssh/test/property_test/ssh_eqc_client_server.erl index 3fc7a15dd0..669aa5b5e0 100644 --- a/lib/ssh/test/property_test/ssh_eqc_client_server.erl +++ b/lib/ssh/test/property_test/ssh_eqc_client_server.erl @@ -32,6 +32,10 @@ -else. +%% Limit the testing time on CI server... this needs to be improved in % from total budget. +-define(TESTINGTIME(Prop), eqc:testing_time(30,Prop)). + + -include_lib("eqc/include/eqc.hrl"). -include_lib("eqc/include/eqc_statem.hrl"). -eqc_group_commands(true). @@ -97,7 +101,7 @@ %% To be called as eqc:quickcheck( ssh_eqc_client_server:prop_seq() ). prop_seq() -> - do_prop_seq(?SSH_DIR). + ?TESTINGTIME(do_prop_seq(?SSH_DIR)). %% To be called from a common_test test suite prop_seq(CT_Config) -> @@ -116,7 +120,7 @@ full_path(SSHdir, CT_Config) -> SSHdir). %%%---- prop_parallel() -> - do_prop_parallel(?SSH_DIR). + ?TESTINGTIME(do_prop_parallel(?SSH_DIR)). %% To be called from a common_test test suite prop_parallel(CT_Config) -> @@ -131,7 +135,7 @@ do_prop_parallel(DataDir) -> %%%---- prop_parallel_multi() -> - do_prop_parallel_multi(?SSH_DIR). + ?TESTINGTIME(do_prop_parallel_multi(?SSH_DIR)). %% To be called from a common_test test suite prop_parallel_multi(CT_Config) -> @@ -171,7 +175,7 @@ weight(_S, _) -> 1. initial_state_pre(S) -> not S#state.initialized. -initial_state_args(S) -> [{var,data_dir}]. +initial_state_args(_) -> [{var,data_dir}]. initial_state_next(S, _, _) -> S#state{initialized=true}. @@ -182,7 +186,7 @@ initial_state_next(S, _, _) -> S#state{initialized=true}. ssh_server_pre(S) -> S#state.initialized andalso length(S#state.servers) < ?MAX_NUM_SERVERS. -ssh_server_args(S) -> [?SERVER_ADDRESS, {var,data_dir}, ?SERVER_EXTRA_OPTIONS]. +ssh_server_args(_) -> [?SERVER_ADDRESS, {var,data_dir}, ?SERVER_EXTRA_OPTIONS]. ssh_server({IP,Port}, DataDir, ExtraOptions) -> ok(ssh:daemon(IP, Port, -- cgit v1.2.3 From 822ed67fc53746ffe3b76ca03f88cee72977f5d0 Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Wed, 17 Sep 2014 17:05:13 +0200 Subject: Add timeout to a property test (Thanks John & Tobias @ QuviQ) --- lib/ssh/test/property_test/ssh_eqc_client_server.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib/ssh') diff --git a/lib/ssh/test/property_test/ssh_eqc_client_server.erl b/lib/ssh/test/property_test/ssh_eqc_client_server.erl index 669aa5b5e0..3030fff38b 100644 --- a/lib/ssh/test/property_test/ssh_eqc_client_server.erl +++ b/lib/ssh/test/property_test/ssh_eqc_client_server.erl @@ -251,8 +251,9 @@ ssh_open_connection(#srvr{address=Ip, port=Port}, DataDir) -> [ {silently_accept_hosts, true}, {user_dir, user_dir(DataDir)}, - {user_interaction, false} - ])). + {user_interaction, false}, + {connect_timeout, 2000} + ], 2000)). ssh_open_connection_post(_S, _Args, Result) -> is_ok(Result). -- cgit v1.2.3 From 6a7fad099d29929e4b9d311dfeb8ff1d79ec3b5d Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Thu, 18 Sep 2014 15:54:22 +0200 Subject: ssh_eqc_encode_decode.erl compiles --- lib/ssh/test/property_test/ssh_eqc_encode_decode.erl | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib/ssh') diff --git a/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl b/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl index 34630bdc91..57ea2012c1 100644 --- a/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl +++ b/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl @@ -25,8 +25,6 @@ -proptest(eqc). -proptest([triq,proper]). --include_lib("ct_property_test.hrl"). - -ifndef(EQC). -ifndef(PROPER). -ifndef(TRIQ). -- cgit v1.2.3 From df4f9a5d10f920dfa16c8f2c75bc40182296d33e Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Thu, 18 Sep 2014 18:35:38 +0200 Subject: Optimizations --- .../test/property_test/ssh_eqc_client_server.erl | 25 ++++++++++++++-------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'lib/ssh') diff --git a/lib/ssh/test/property_test/ssh_eqc_client_server.erl b/lib/ssh/test/property_test/ssh_eqc_client_server.erl index 3030fff38b..123b48412b 100644 --- a/lib/ssh/test/property_test/ssh_eqc_client_server.erl +++ b/lib/ssh/test/property_test/ssh_eqc_client_server.erl @@ -79,7 +79,9 @@ -define(SUBSYSTEMS, ["echo1", "echo2", "echo3", "echo4"]). --define(SERVER_ADDRESS, { {127,1,1,1}, {call, ?MODULE, inet_port, [{127,1,1,1}]} }). +-define(SERVER_ADDRESS, { {127,1,0,choose(1,254)}, % IP + choose(1024,65535) % Port + }). -define(SERVER_EXTRA_OPTIONS, [{parallel_login,bool()}] ). @@ -109,6 +111,7 @@ prop_seq(CT_Config) -> do_prop_seq(DataDir) -> + setup_rsa(DataDir), ?FORALL(Cmds,commands(?MODULE), begin {H,Sf,Result} = run_commands(?MODULE,Cmds,[{data_dir,DataDir}]), @@ -127,6 +130,7 @@ prop_parallel(CT_Config) -> do_prop_parallel(full_path(?SSH_DIR, CT_Config)). do_prop_parallel(DataDir) -> + setup_rsa(DataDir), ?FORALL(Cmds,parallel_commands(?MODULE), begin {H,Sf,Result} = run_parallel_commands(?MODULE,Cmds,[{data_dir,DataDir}]), @@ -142,6 +146,7 @@ prop_parallel_multi(CT_Config) -> do_prop_parallel_multi(full_path(?SSH_DIR, CT_Config)). do_prop_parallel_multi(DataDir) -> + setup_rsa(DataDir), ?FORALL(Repetitions,?SHRINK(1,[10]), ?FORALL(Cmds,parallel_commands(?MODULE), ?ALWAYS(Repetitions, @@ -160,8 +165,7 @@ initial_state() -> %%% called when using commands/2 initial_state(DataDir) -> application:stop(ssh), - ssh:start(), - setup_rsa(DataDir). + ssh:start(). %%%---------------- weight(S, ssh_send) -> 5*length([C || C<-S#state.channels, has_subsyst(C)]); @@ -183,6 +187,13 @@ initial_state_next(S, _, _) -> S#state{initialized=true}. %%% Start a new daemon %%% Precondition: not more than ?MAX_NUM_SERVERS started +%%% This is a bit funny because we need to pick an IP address and Port to +%%% run the server on, but there is no way to atomically select a free Port! +%%% +%%% Therefore we just grab one IP-Port pair randomly and try to start the ssh server +%%% on that pair. If it fails, we just forget about it and goes on. Yes, it +%%% is a waste of cpu cycles, but at least it works! + ssh_server_pre(S) -> S#state.initialized andalso length(S#state.servers) < ?MAX_NUM_SERVERS. @@ -197,8 +208,10 @@ ssh_server({IP,Port}, DataDir, ExtraOptions) -> | ExtraOptions ])). +ssh_server_post(_S, _Args, {error,eaddrinuse}) -> true; ssh_server_post(_S, _Args, Result) -> is_ok(Result). +ssh_server_next(S, {error,eaddrinuse}, _) -> S; ssh_server_next(S, Result, [{IP,Port},_,_]) -> S#state{servers=[#srvr{ref = Result, address = IP, @@ -573,12 +586,6 @@ median(_) -> %%%================================================================ %%% The rest is taken and modified from ssh_test_lib.erl -inet_port(IpAddress)-> - {ok, Socket} = gen_tcp:listen(0, [{ip,IpAddress},{reuseaddr,true}]), - {ok, Port} = inet:port(Socket), - gen_tcp:close(Socket), - Port. - setup_rsa(Dir) -> erase_dir(system_dir(Dir)), erase_dir(user_dir(Dir)), -- cgit v1.2.3