Merge branch 'siri/ct/no_prompt_check/OTP-11095' into maint

* siri/ct/no_prompt_check/OTP-11095:
  [common_test] Refactor ct_telnet_SUITE.erl
  [common_test] Add new option 'no_prompt_check' to ct_telnet:expect/3.
  [common_test] Add debug printouts in ct_telnet_client
  [common_test] Backed out wait_for_linebreak option from ct_telnet
  [common_test] Add test for ct_telnet using own telnet server
  [common_test] Add telnet server to use for testing

4 files changed, 467 insertions, 15 deletions

diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index 94569fa87f..31ab28c41d 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -28,6 +28,7 @@ MODULES= \
 	ct_test_support \
 	ct_test_support_eh \
 	ct_userconfig_callback \
+	telnet_server \
 	ct_smoke_test_SUITE \
 	ct_priv_dir_SUITE \
 	ct_event_handler_SUITE \
diff --git a/lib/common_test/test/ct_telnet_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE.erl
index b4f24baa0c..17617f59eb 100644
--- a/lib/common_test/test/ct_telnet_SUITE.erl
+++ b/lib/common_test/test/ct_telnet_SUITE.erl
@@ -33,6 +33,10 @@
 
 -define(eh, ct_test_support_eh).
 
+-define(erl_telnet_server_port,1234).
+-define(erl_telnet_server_user,"telnuser").
+-define(erl_telnet_server_pwd,"telnpwd").
+
 %%--------------------------------------------------------------------
 %% TEST SERVER CALLBACK FUNCTIONS
 %%--------------------------------------------------------------------
@@ -48,17 +52,27 @@ init_per_suite(Config) ->
 end_per_suite(Config) ->
     ct_test_support:end_per_suite(Config).
 
+init_per_testcase(TestCase, Config) when TestCase=/=unix_telnet->
+    TS = telnet_server:start([{port,?erl_telnet_server_port},
+			      {users,[{?erl_telnet_server_user,
+				       ?erl_telnet_server_pwd}]}]),
+    ct_test_support:init_per_testcase(TestCase, [{telnet_server,TS}|Config]);
 init_per_testcase(TestCase, Config) ->
     ct_test_support:init_per_testcase(TestCase, Config).
 
 end_per_testcase(TestCase, Config) ->
+    case ?config(telnet_server,Config) of
+	undefined -> ok;
+	TS -> telnet_server:stop(TS)
+    end,
     ct_test_support:end_per_testcase(TestCase, Config).
 
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() ->
     [
-     default
+     unix_telnet,
+     own_server
     ].
 
 %%--------------------------------------------------------------------
@@ -67,19 +81,29 @@ all() ->
 
 %%%-----------------------------------------------------------------
 %%%
-default(Config) when is_list(Config) ->
-    DataDir = ?config(data_dir, Config),
-    Suite = filename:join(DataDir, "ct_telnet_basic_SUITE"),
-    Cfg = {unix, ct:get_config(unix)},
-    ok = file:write_file(filename:join(DataDir, "telnet.cfg"), io_lib:write(Cfg) ++ "."),
-    CfgFile = filename:join(DataDir, "telnet.cfg"),
-    {Opts,ERPid} = setup([{suite,Suite},{label,default}, {config, CfgFile}], Config),
-    ok = execute(default, Opts, ERPid, Config).
+unix_telnet(Config) when is_list(Config) ->
+    all_tests_in_suite(unix_telnet,"ct_telnet_basic_SUITE","telnet.cfg",Config).
+
+own_server(Config) ->
+    all_tests_in_suite(own_server,"ct_telnet_own_server_SUITE",
+		       "telnet2.cfg",Config).
 
 %%%-----------------------------------------------------------------
 %%% HELP FUNCTIONS
 %%%-----------------------------------------------------------------
 
+all_tests_in_suite(TestCase, SuiteName, CfgFileName, Config) ->
+    DataDir = ?config(data_dir, Config),
+    Suite = filename:join(DataDir, SuiteName),
+    CfgFile = filename:join(DataDir, CfgFileName),
+    Cfg = telnet_config(TestCase),
+    ok = file:write_file(CfgFile, io_lib:write(Cfg) ++ "."),
+    {Opts,ERPid} = setup([{suite,Suite},
+			  {label,TestCase},
+			  {config,CfgFile}],
+			 Config),
+    ok = execute(TestCase, Opts, ERPid, Config).
+
 setup(Test, Config) ->
     Opts0 = ct_test_support:get_opts(Config),
     Level = ?config(trace_level, Config),
@@ -103,19 +127,34 @@ execute(Name, Opts, ERPid, Config) ->
 reformat(Events, EH) ->
     ct_test_support:reformat(Events, EH).
 
+
+telnet_config(unix_telnet) ->
+    {unix, ct:get_config(unix)};
+telnet_config(_) ->
+    {unix,[{telnet,"localhost"},
+	   {port, ?erl_telnet_server_port},
+	   {username,?erl_telnet_server_user},
+	   {password,?erl_telnet_server_pwd},
+	   {keep_alive,true}]}.
+
 %%%-----------------------------------------------------------------
 %%% TEST EVENTS
 %%%-----------------------------------------------------------------
-events_to_check(default,Config) ->
+events_to_check(unix_telnet,Config) ->
+    all_cases(ct_telnet_basic_SUITE,Config);
+events_to_check(own_server,Config) ->
+    all_cases(ct_telnet_own_server_SUITE,Config).
+
+all_cases(Suite,Config) ->
     {module,_} = code:load_abs(filename:join(?config(data_dir,Config),
-					     ct_telnet_basic_SUITE)),
-    TCs = ct_telnet_basic_SUITE:all(),
-    code:purge(ct_telnet_basic_SUITE),
-    code:delete(ct_telnet_basic_SUITE),
+					     Suite)),
+    TCs = Suite:all(),
+    code:purge(Suite),
+    code:delete(Suite),
 
     OneTest =
 	[{?eh,start_logging,{'DEF','RUNDIR'}}] ++
-	[{?eh,tc_done,{ct_telnet_basic_SUITE,TC,ok}} || TC <- TCs] ++
+	[{?eh,tc_done,{Suite,TC,ok}} || TC <- TCs] ++
 	[{?eh,stop_logging,[]}],
 
     %% 2 tests (ct:run_test + script_start) is default
diff --git a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl
new file mode 100644
index 0000000000..3f7c0d68bf
--- /dev/null
+++ b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl
@@ -0,0 +1,184 @@
+-module(ct_telnet_own_server_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+
+suite() -> [{require,erl_telnet_server,{unix,[telnet]}}].
+
+all() ->
+    [expect,
+     expect_repeat,
+     expect_sequence,
+     expect_error_prompt,
+     expect_error_timeout,
+     no_prompt_check,
+     no_prompt_check_repeat,
+     no_prompt_check_sequence,
+     no_prompt_check_timeout,
+     ignore_prompt,
+     ignore_prompt_repeat,
+     ignore_prompt_sequence,
+     ignore_prompt_timeout].
+
+groups() ->
+    [].
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+%% Simple expect
+expect(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo ayt"),
+    {ok,["ayt"]} = ct_telnet:expect(Handle, ["ayt"]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% Expect with repeat option
+expect_repeat(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo_ml xy xy"),
+    {ok,[["xy"],["xy"]],done} = ct_telnet:expect(Handle, ["xy"],[{repeat,2}]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% Expect with sequence option
+expect_sequence(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo_ml ab cd ef"),
+    {ok,[["ab"],["cd"],["ef"]]} = ct_telnet:expect(Handle,
+						   [["ab"],["cd"],["ef"]],
+						   [sequence]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% Check that expect returns when a prompt is found, even if pattern
+%% is not matched.
+expect_error_prompt(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo xxx> yyy"),
+    {error,{prompt,"> "}} = ct_telnet:expect(Handle, ["yyy"]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% Check that expect returns after idle timeout, and even if the
+%% expected pattern is received - as long as not newline or prompt is
+%% received it will not match.
+expect_error_timeout(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo_no_prompt xxx"),
+    {error,timeout} = ct_telnet:expect(Handle, ["xxx"], [{timeout,1000}]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% expect with ignore_prompt option should not return even if a prompt
+%% is found. The pattern after the prompt (here "> ") can be matched.
+ignore_prompt(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo xxx> yyy"),
+    {ok,["yyy"]} = ct_telnet:expect(Handle, ["yyy"], [ignore_prompt]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% expect with ignore_prompt and repeat options.
+ignore_prompt_repeat(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo_ml yyy> yyy>"),
+    {ok,[["yyy"],["yyy"]],done} = ct_telnet:expect(Handle, ["yyy"],
+						   [{repeat,2},
+						    ignore_prompt]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% expect with ignore_prompt and sequence options.
+ignore_prompt_sequence(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo_ml xxx> yyy> zzz> "),
+    {ok,[["xxx"],["yyy"],["zzz"]]} = ct_telnet:expect(Handle,
+						      [["xxx"],["yyy"],["zzz"]],
+						      [sequence,
+						       ignore_prompt]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% Check that expect returns after idle timeout when ignore_prompt
+%% option is used.
+%% As for expect without the ignore_prompt option, it a newline or a
+%% prompt is required in order for the pattern to match.
+ignore_prompt_timeout(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo xxx"),
+    {error,timeout} = ct_telnet:expect(Handle, ["yyy"], [ignore_prompt,
+							 {timeout,1000}]),
+    ok = ct_telnet:send(Handle, "echo xxx"), % sends prompt and newline
+    {ok,["xxx"]} = ct_telnet:expect(Handle, ["xxx"], [ignore_prompt,
+						      {timeout,1000}]),
+    ok = ct_telnet:send(Handle, "echo_no_prompt xxx\n"), % no prompt, but newline
+    {ok,["xxx"]} = ct_telnet:expect(Handle, ["xxx"], [ignore_prompt,
+						      {timeout,1000}]),
+    ok = ct_telnet:send(Handle, "echo_no_prompt xxx"), % no prompt, no newline
+    {error,timeout} = ct_telnet:expect(Handle, ["xxx"], [ignore_prompt,
+							 {timeout,1000}]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% no_prompt_check option shall match pattern both when prompt is sent
+%% and when it is not.
+no_prompt_check(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo xxx"),
+    {ok,["xxx"]} = ct_telnet:expect(Handle, ["xxx"], [no_prompt_check]),
+    ok = ct_telnet:send(Handle, "echo_no_prompt yyy"),
+    {ok,["yyy"]} = ct_telnet:expect(Handle, ["yyy"], [no_prompt_check]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% no_prompt_check and repeat options
+no_prompt_check_repeat(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo_ml xxx xxx"),
+    {ok,[["xxx"],["xxx"]],done} = ct_telnet:expect(Handle,["xxx"],
+						   [{repeat,2},
+						    no_prompt_check]),
+    ok = ct_telnet:send(Handle, "echo_ml_no_prompt yyy yyy"),
+    {ok,[["yyy"],["yyy"]],done} = ct_telnet:expect(Handle,["yyy"],
+						   [{repeat,2},
+						    no_prompt_check]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% no_prompt_check and sequence options
+no_prompt_check_sequence(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo_ml_no_prompt ab cd ef"),
+    {ok,[["ab"],["cd"],["ef"]]} = ct_telnet:expect(Handle,
+						   [["ab"],["cd"],["ef"]],
+						   [sequence,
+						    no_prompt_check]),
+    ok = ct_telnet:close(Handle),
+    ok.
+
+%% Check that expect returns after idle timeout when no_prompt_check
+%% option is used.
+no_prompt_check_timeout(_) ->
+    {ok, Handle} = ct_telnet:open(erl_telnet_server),
+    ok = ct_telnet:send(Handle, "echo xxx"),
+    {error,timeout} = ct_telnet:expect(Handle, ["yyy"], [no_prompt_check,
+							 {timeout,1000}]),
+    ok = ct_telnet:close(Handle),
+    ok.
diff --git a/lib/common_test/test/telnet_server.erl b/lib/common_test/test/telnet_server.erl
new file mode 100644
index 0000000000..31884aa182
--- /dev/null
+++ b/lib/common_test/test/telnet_server.erl
@@ -0,0 +1,228 @@
+-module(telnet_server).
+-compile(export_all).
+
+%% telnet control characters
+-define(SE,	240).
+-define(NOP,	241).
+-define(DM,	242).
+-define(BRK,	243).
+-define(IP,	244).
+-define(AO,	245).
+-define(AYT,	246).
+-define(EC,	247).
+-define(EL,	248).
+-define(GA,	249).
+-define(SB,	250).
+-define(WILL,	251).
+-define(WONT,	252).
+-define(DO,	253).
+-define(DONT,	254).
+-define(IAC,	255).
+
+%% telnet options
+-define(BINARY,            0).
+-define(ECHO,	           1).
+-define(SUPPRESS_GO_AHEAD, 3).
+-define(TERMINAL_TYPE,     24).  
+
+-record(state,
+	{listen,
+	 client,
+	 users,
+	 authorized=false,
+	 suppress_go_ahead=false,
+	 buffer=[]}).
+
+-type options() :: [{port,pos_integer()} | {users,users()}].
+-type users() :: [{user(),password()}].
+-type user() :: string().
+-type password() :: string(). 
+
+-spec start(Opts::options()) -> Pid::pid().
+start(Opts) when is_list(Opts) ->
+    spawn_link(fun() -> init(Opts) end).
+
+-spec stop(Pid::pid()) -> ok.
+stop(Pid) ->
+    Ref = erlang:monitor(process,Pid),
+    Pid ! stop,
+    receive {'DOWN',Ref,_,_,_} -> ok end.
+
+init(Opts) ->
+    Port = proplists:get_value(port,Opts),
+    Users = proplists:get_value(users,Opts,[]),
+    {ok, LSock} = gen_tcp:listen(Port, [list, {packet, 0}, 
+                                        {active, true}]),
+    State = #state{listen=LSock,users=Users},
+    accept(State),
+    ok = gen_tcp:close(LSock).
+
+accept(#state{listen=LSock}=State) ->
+    Server = self(),
+    Acceptor = spawn_link(fun() -> do_accept(LSock,Server) end),
+    receive 
+	{Acceptor,Sock} when is_port(Sock) ->
+	    case init_client(State#state{client=Sock}) of
+		stopped ->
+		    io:format("telnet_server stopped\n"),
+		    ok;
+		R ->
+		    io:format("connection to client closed with reason ~p~n",[R]),
+		    accept(State)
+	    end;
+	{Acceptor,closed} ->
+	    io:format("listen socket closed unexpectedly, "
+		      "terminating telnet_server\n"),
+	    ok;
+	stop  ->
+	    io:format("telnet_server stopped\n"),
+	    ok
+    end.
+
+do_accept(LSock,Server) ->
+    case gen_tcp:accept(LSock) of
+	{ok, Sock} ->
+	    ok = gen_tcp:controlling_process(Sock,Server),
+	    Server ! {self(),Sock};
+	{error,closed} ->
+	    %% This will happen when stop/1 is called, since listen
+	    %% socket is closed. Then the server is probably already
+	    %% dead by now, but to be 100% sure not to hang, we send a
+	    %% message to say what happened.
+	    Server ! {self(),closed}
+    end.
+
+init_client(#state{client=Sock}=State) ->
+    dbg("Server sending: ~p~n",["login: "]),
+    R = case gen_tcp:send(Sock,"login: ") of
+	    ok ->
+		loop(State);
+	    Error ->
+		Error
+	end,
+    _ = gen_tcp:close(Sock),
+    R.
+
+loop(State) ->
+    receive
+	{tcp,_,Data} ->
+	    try handle_data(Data,State) of
+		{ok,State1} ->
+		    loop(State1)
+	    catch 
+		throw:Error ->
+		    Error
+	    end;
+        {tcp_closed, _} ->
+            closed;
+	{tcp_error,_,Error} ->
+	    {error,tcp,Error};
+	stop ->
+	    stopped
+    end.
+
+handle_data([?IAC|Cmd],State) ->
+    dbg("Server got cmd: ~p~n",[Cmd]),
+    handle_cmd(Cmd,State);
+handle_data(Data,State) ->
+    dbg("Server got data: ~p~n",[Data]),
+    case get_line(Data,[]) of
+	{Line,Rest} ->
+	    WholeLine = lists:flatten(lists:reverse(State#state.buffer,Line)),
+	    {ok,State1} = do_handle_data(WholeLine,State),
+	    case Rest of
+		[] -> {ok,State1};
+		_ -> handle_data(Rest,State1)
+	    end;
+	false ->
+	    {ok,State#state{buffer=[Data|State#state.buffer]}}
+    end.
+
+%% Add function clause below to handle new telnet commands (sent with
+%% ?IAC from client - this is not possible to do from ct_telnet API,
+%% but ct_telnet sends DONT SUPPRESS_GO_AHEAD)
+handle_cmd([?DO,?SUPPRESS_GO_AHEAD|T],State) ->
+    send([?IAC,?WILL,?SUPPRESS_GO_AHEAD],State),
+    handle_cmd(T,State#state{suppress_go_ahead=true});
+handle_cmd([?DONT,?SUPPRESS_GO_AHEAD|T],State) ->
+    send([?IAC,?WONT,?SUPPRESS_GO_AHEAD],State),
+    handle_cmd(T,State#state{suppress_go_ahead=false});
+handle_cmd([?IAC|T],State) ->
+    %% Multiple commands in one packet
+    handle_cmd(T,State);
+handle_cmd([_H|T],State) ->
+    %% Not responding to this command
+    handle_cmd(T,State);
+handle_cmd([],State) ->
+    {ok,State}.
+
+%% Add function clause below to handle new text command (text entered
+%% from the telnet prompt)
+do_handle_data(Data,#state{authorized=false}=State) ->
+    check_user(Data,State);
+do_handle_data(Data,#state{authorized={user,_}}=State) ->
+    check_pwd(Data,State);
+do_handle_data("echo "++ Data,State) ->
+    send(Data++"\r\n> ",State),
+    {ok,State};
+do_handle_data("echo_no_prompt "++ Data,State) ->
+    send(Data,State),
+    {ok,State};
+do_handle_data("echo_ml "++ Data,State) ->
+    Lines = string:tokens(Data," "),
+    ReturnData = string:join(Lines,"\n"),
+    send(ReturnData++"\r\n> ",State),
+    {ok,State};
+do_handle_data("echo_ml_no_prompt "++ Data,State) ->
+    Lines = string:tokens(Data," "),
+    ReturnData = string:join(Lines,"\n"),
+    send(ReturnData,State),
+    {ok,State};
+do_handle_data([],State) ->
+    send("> ",State),
+    {ok,State};
+do_handle_data(_Data,State) ->
+    send("error: unknown command\r\n> ",State),
+    {ok,State}.
+
+check_user(User,State) ->
+    case lists:keyfind(User,1,State#state.users) of
+	{User,Pwd} ->
+	   dbg("user ok\n"),
+	    send("Password: ",State),
+	    {ok,State#state{authorized={user,Pwd}}};
+	false ->
+	    throw({error,unknown_user})
+    end.
+
+check_pwd(Pwd,#state{authorized={user,Pwd}}=State) ->
+    dbg("password ok\n"),
+    send("Welcome to the ultimate telnet server!\r\n> ",State),
+    {ok,State#state{authorized=true}};
+check_pwd(_,_State) ->
+    throw({error,authentication}).
+
+send(Data,State) ->
+    dbg("Server sending: ~p~n",[Data]),
+    case gen_tcp:send(State#state.client,Data) of
+	ok ->
+	    ok;
+	{error,Error} ->
+	    throw({error,send,Error})
+    end.
+    
+get_line([$\r,$\n|Rest],Acc) ->
+    {lists:reverse(Acc),Rest};
+get_line([$\r,0|Rest],Acc) ->
+    {lists:reverse(Acc),Rest};
+get_line([$\n|Rest],Acc) ->
+    {lists:reverse(Acc),Rest};
+get_line([H|T],Acc) ->
+    get_line(T,[H|Acc]);
+get_line([],_) ->
+    false.
+
+dbg(_F) ->
+    io:format(_F).
+dbg(_F,_A) ->
+    io:format(_F,_A).