Merge branch 'peppe/ct_misc_18_patches' into maint-18

* peppe/ct_misc_18_patches:
  Add flag/option for disabling the character escaping functionality
  Fix bug using the wrong lists search function
  Fix bug with clashing timestamp values
  Fix problem with stylesheet tags getting escaped
  Skip pre/post test IO suite if cover or debug is running
  Tweak pre_post_io test case to run without failing
  Fix various log related problems

10 files changed, 148 insertions, 41 deletions

diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index 538be514d6..22941668f2 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -161,9 +161,9 @@ run(TestDirs) ->
 %%%               {repeat,N} | {duration,DurTime} | {until,StopTime} |
 %%%               {force_stop,ForceStop} | {decrypt,DecryptKeyOrFile} |
 %%%               {refresh_logs,LogDir} | {logopts,LogOpts} | 
-%%%               {verbosity,VLevels} | {basic_html,Bool} | 
-%%%               {ct_hooks, CTHs} | {enable_builtin_hooks,Bool} |
-%%%               {release_shell,Bool}
+%%%               {verbosity,VLevels} | {basic_html,Bool} |
+%%%               {esc_chars,Bool} | {ct_hooks, CTHs} |
+%%%               {enable_builtin_hooks,Bool} | {release_shell,Bool}
 %%%   TestDirs = [string()] | string()
 %%%   Suites = [string()] | [atom()] | string() | atom()
 %%%   Cases = [atom()] | atom()
diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl
index 92640cf323..899d2bfb5a 100644
--- a/lib/common_test/src/ct_groups.erl
+++ b/lib/common_test/src/ct_groups.erl
@@ -325,7 +325,7 @@ modify_tc_list1(GrSpecTs, TSCs) ->
 		      true ->
 			  {[TC|TSCs1],lists:delete(TC,GrSpecTs2)};
 		      false ->
-			  case lists:keymember(TC, 2, GrSpecTs) of
+			  case lists:keysearch(TC, 2, GrSpecTs) of
 			      {value,Test} ->
 				  {[Test|TSCs1],
 				   lists:keydelete(TC, 2, GrSpecTs2)};
diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl
index 83ad33fdd8..90f36cbdc2 100644
--- a/lib/common_test/src/ct_hooks.erl
+++ b/lib/common_test/src/ct_hooks.erl
@@ -67,6 +67,8 @@ terminate(Hooks) ->
 %% tests.
 -spec init_tc(Mod :: atom(),
 	      FuncSpec :: atom() | 
+			  {ConfigFunc :: init_per_testcase | end_per_testcase,
+			   TestCase :: atom()} |
 			  {ConfigFunc :: init_per_group | end_per_group,
 			   GroupName :: atom(),
 			   Properties :: list()},
@@ -103,7 +105,9 @@ init_tc(_Mod, TC = error_in_suite, Config) ->
 %% @doc Called as each test case is completed. This includes all configuration
 %% tests.
 -spec end_tc(Mod :: atom(),
-	     FuncSpec :: atom() | 
+	     FuncSpec :: atom() |  
+			 {ConfigFunc :: init_per_testcase | end_per_testcase,
+			  TestCase :: atom()} |
 			 {ConfigFunc :: init_per_group | end_per_group,
 			  GroupName :: atom(),
 			  Properties :: list()},
diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl
index 4920383f39..a9ad571bfc 100644
--- a/lib/common_test/src/ct_logs.erl
+++ b/lib/common_test/src/ct_logs.erl
@@ -609,7 +609,8 @@ log_timestamp({MS,S,US}) ->
 		      ct_log_fd,
 		      tc_groupleaders,
 		      stylesheet,
-		      async_print_jobs}).
+		      async_print_jobs,
+		      tc_esc_chars}).
 
 logger(Parent, Mode, Verbosity) ->
     register(?MODULE,self()),
@@ -728,14 +729,18 @@ logger(Parent, Mode, Verbosity) ->
 	   end
      end || {Cat,VLvl} <- Verbosity],
     io:nl(CtLogFd),
-
+    TcEscChars = case application:get_env(common_test, esc_chars) of
+		   {ok,ECBool} -> ECBool;
+		   _           -> true
+	       end,
     logger_loop(#logger_state{parent=Parent,
 			      log_dir=AbsDir,
 			      start_time=Time,
 			      orig_GL=group_leader(),
 			      ct_log_fd=CtLogFd,
 			      tc_groupleaders=[],
-			      async_print_jobs=[]}).
+			      async_print_jobs=[],
+			      tc_esc_chars=TcEscChars}).
 
 copy_priv_files([SrcF | SrcFs], [DestF | DestFs]) ->
     case file:copy(SrcF, DestF) of
@@ -761,20 +766,21 @@ logger_loop(State) ->
 		end,
 	    if Importance >= (100-VLvl) ->
 		    CtLogFd = State#logger_state.ct_log_fd,
+		    DoEscChars = State#logger_state.tc_esc_chars and EscChars,
 		    case get_groupleader(Pid, GL, State) of
 			{tc_log,TCGL,TCGLs} ->
 			    case erlang:is_process_alive(TCGL) of
 				true ->
 				    State1 = print_to_log(SyncOrAsync, Pid,
 							  Category, TCGL, Content,
-							  EscChars, State),
+							  DoEscChars, State),
 				    logger_loop(State1#logger_state{
 						  tc_groupleaders = TCGLs});
 				false ->
 				    %% Group leader is dead, so write to the
 				    %% CtLog or unexpected_io log instead
 				    unexpected_io(Pid, Category, Importance,
-						  Content, CtLogFd, EscChars),
+						  Content, CtLogFd, DoEscChars),
 
 				    logger_loop(State)			    
 			    end;
@@ -783,7 +789,7 @@ logger_loop(State) ->
 			    %% to ct_log, else write to unexpected_io
 			    %% log
 			    unexpected_io(Pid, Category, Importance, Content,
-					  CtLogFd, EscChars),
+					  CtLogFd, DoEscChars),
 			    logger_loop(State#logger_state{
 					  tc_groupleaders = TCGLs})
 		    end;
@@ -794,7 +800,7 @@ logger_loop(State) ->
 	    %% make sure no IO for this test case from the
 	    %% CT logger gets rejected
 	    test_server:permit_io(GL, self()),
-	    print_style(GL, State#logger_state.stylesheet),
+	    print_style(GL,GL,State#logger_state.stylesheet),
 	    set_evmgr_gl(GL),
 	    TCGLs = add_tc_gl(TCPid,GL,State),
 	    if not RefreshLog ->
@@ -882,7 +888,7 @@ create_io_fun(FromPid, CtLogFd, EscChars) ->
 		    {_HdOrFt,S,A} -> {false,S,A};
 		    {S,A}         -> {true,S,A}
 		end,
-	    try io_lib:format(Str,Args) of
+	    try io_lib:format(Str, Args) of
 		IoStr when Escapable, EscChars, IoList == [] ->
 		    escape_chars(IoStr);
 		IoStr when Escapable, EscChars ->
@@ -925,7 +931,12 @@ print_to_log(sync, FromPid, Category, TCGL, Content, EscChars, State) ->
     if FromPid /= TCGL ->
 	    IoFun = create_io_fun(FromPid, CtLogFd, EscChars),
 	    IoList = lists:foldl(IoFun, [], Content),
-	    io:format(TCGL,["$tc_html","~ts"], [IoList]);
+	    try io:format(TCGL,["$tc_html","~ts"], [IoList]) of
+		ok -> ok
+	    catch
+		_:_ ->
+		    io:format(TCGL,"~ts", [IoList])
+	    end;
        true ->
 	    unexpected_io(FromPid, Category, ?MAX_IMPORTANCE, Content,
 			  CtLogFd, EscChars)
@@ -958,7 +969,10 @@ print_to_log(async, FromPid, Category, TCGL, Content, EscChars, State) ->
 				    _:terminated ->
 					unexpected_io(FromPid, Category,
 						      ?MAX_IMPORTANCE,
-						      Content, CtLogFd, EscChars)
+						      Content, CtLogFd, EscChars);
+				    _:_ ->
+					io:format(TCGL, "~ts",
+						  [lists:foldl(IoFun,[],Content)])
 				end;
 			    false ->
 				unexpected_io(FromPid, Category,
@@ -1099,26 +1113,27 @@ open_ctlog(MiscIoName) ->
 	      "View I/O logged after the test run</a></li>\n</ul>\n",
 	      [MiscIoName,MiscIoName]),
 
-    print_style(Fd,undefined),
+    print_style(Fd,group_leader(),undefined),
     io:format(Fd, 
 	      xhtml("<br><h2>Progress Log</h2>\n<pre>\n",
 		    "<br />\n<h4>PROGRESS LOG</h4>\n<pre>\n"), []),
     Fd.
 
-print_style(Fd,undefined) ->
+print_style(Fd,GL,undefined) ->
     case basic_html() of
 	true ->
-	    io:format(Fd,
-		      "<style>\n"
-		      "div.ct_internal { background:lightgrey; color:black; }\n"
-		      "div.default     { background:lightgreen; color:black; }\n"
-		      "</style>\n",
-		      []);
+	    Style = "<style>\n
+		div.ct_internal { background:lightgrey; color:black; }\n
+		div.default     { background:lightgreen; color:black; }\n
+		</style>\n",
+	    if Fd == GL -> io:format(["$tc_html",Style], []);
+	       true     -> io:format(Fd, Style, [])
+	    end;
 	_ ->
 	    ok
     end;
 
-print_style(Fd,StyleSheet) ->
+print_style(Fd,GL,StyleSheet) ->
     case file:read_file(StyleSheet) of
 	{ok,Bin} ->
 	    Str = b2s(Bin,encoding(StyleSheet)),
@@ -1131,23 +1146,30 @@ print_style(Fd,StyleSheet) ->
 		       N1 -> N1
 		   end,
 	    if (Pos0 == 0) and (Pos1 /= 0) ->
-		    print_style_error(Fd,StyleSheet,missing_style_start_tag);
+		    print_style_error(Fd,GL,StyleSheet,missing_style_start_tag);
 	       (Pos0 /= 0) and (Pos1 == 0) ->
-		    print_style_error(Fd,StyleSheet,missing_style_end_tag);
+		    print_style_error(Fd,GL,StyleSheet,missing_style_end_tag);
 	       Pos0 /= 0 ->
 		    Style = string:sub_string(Str,Pos0,Pos1+7),
-		    io:format(Fd,"~ts\n",[Style]);
+		    if Fd == GL -> io:format(Fd,["$tc_html","~ts\n"],[Style]);
+		       true     -> io:format(Fd,"~ts\n",[Style])
+		    end;
 	       Pos0 == 0 ->
-		    io:format(Fd,"<style>~ts</style>\n",[Str])
+		    if Fd == GL -> io:format(Fd,["$tc_html","<style>\n~ts</style>\n"],[Str]);
+		       true     -> io:format(Fd,"<style>\n~ts</style>\n",[Str])
+		    end
 	    end;
 	{error,Reason} ->
-	    print_style_error(Fd,StyleSheet,Reason)  
+	    print_style_error(Fd,GL,StyleSheet,Reason)  
     end.
 
-print_style_error(Fd,StyleSheet,Reason) ->
-    io:format(Fd,"\n<!-- Failed to load stylesheet ~ts: ~p -->\n",
-	      [StyleSheet,Reason]),
-    print_style(Fd,undefined).    
+print_style_error(Fd,GL,StyleSheet,Reason) ->
+    IO = io_lib:format("\n<!-- Failed to load stylesheet ~ts: ~p -->\n",
+		       [StyleSheet,Reason]),
+    if Fd == GL -> io:format(Fd,["$tc_html",IO],[]);
+       true     -> io:format(Fd,IO,[])
+    end,
+    print_style(Fd,GL,undefined).    
 
 close_ctlog(Fd) ->
     io:format(Fd, "\n</pre>\n", []),
diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl
index 228daf459b..d24edad2eb 100644
--- a/lib/common_test/src/ct_master.erl
+++ b/lib/common_test/src/ct_master.erl
@@ -27,7 +27,7 @@
 -export([run_on_node/2,run_on_node/3]).
 -export([run_test/1,run_test/2]).
 -export([get_event_mgr_ref/0]).
--export([basic_html/1]).
+-export([basic_html/1,esc_chars/1]).
 
 -export([abort/0,abort/1,progress/0]).
 
@@ -317,6 +317,16 @@ basic_html(Bool) ->
     ok.
 
 %%%-----------------------------------------------------------------
+%%% @spec esc_chars(Bool) -> ok
+%%%       Bool = true | false
+%%%
+%%% @doc If set to false, the ct_master logs will be written without
+%%%      special characters being escaped in the HTML logs.
+esc_chars(Bool) ->
+    application:set_env(common_test_master, esc_chars, Bool),
+    ok.
+
+%%%-----------------------------------------------------------------
 %%% MASTER, runs on central controlling node.
 %%%-----------------------------------------------------------------
 start_master(NodeOptsList) ->
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index ceb94ceee5..a0f9f47b41 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -65,6 +65,7 @@
 	       logdir,
 	       logopts = [],
 	       basic_html,
+	       esc_chars = true,
 	       verbosity = [],
 	       config = [],
 	       event_handlers = [],
@@ -346,6 +347,15 @@ script_start1(Parent, Args) ->
 			application:set_env(common_test, basic_html, true),
 			true
 		end,
+    %% esc_chars - used by ct_logs
+    EscChars = case proplists:get_value(no_esc_chars, Args) of
+		   undefined ->
+		       application:set_env(common_test, esc_chars, true),
+		       undefined;
+		   _ ->
+		       application:set_env(common_test, esc_chars, false),
+		       false
+	       end,
     %% disable_log_cache - used by ct_logs
     case proplists:get_value(disable_log_cache, Args) of
 	undefined ->
@@ -359,6 +369,7 @@ script_start1(Parent, Args) ->
 		 cover = Cover, cover_stop = CoverStop,
 		 logdir = LogDir, logopts = LogOpts,
 		 basic_html = BasicHtml,
+		 esc_chars = EscChars,
 		 verbosity = Verbosity,
 		 event_handlers = EvHandlers,
 		 ct_hooks = CTHooks,
@@ -587,6 +598,17 @@ combine_test_opts(TS, Specs, Opts) ->
 		BHBool
 	end,
 
+    EscChars =
+	case choose_val(Opts#opts.esc_chars,
+			TSOpts#opts.esc_chars) of
+	    undefined ->
+		true;
+	    ECBool ->
+		application:set_env(common_test, esc_chars,
+				    ECBool),
+		ECBool
+	end,
+
     Opts#opts{label = Label,
 	      profile = Profile,
 	      testspec_files = Specs,
@@ -595,6 +617,7 @@ combine_test_opts(TS, Specs, Opts) ->
 	      logdir = which(logdir, LogDir),
 	      logopts = AllLogOpts,
 	      basic_html = BasicHtml,
+	      esc_chars = EscChars,
 	      verbosity = AllVerbosity,
 	      silent_connections = AllSilentConns,
 	      config = TSOpts#opts.config,
@@ -795,6 +818,7 @@ script_usage() ->
 	      "\n\t [-scale_timetraps]"
 	      "\n\t [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]"
 	      "\n\t [-basic_html]"
+	      "\n\t [-no_esc_chars]"
 	      "\n\t [-repeat N] |"
 	      "\n\t [-duration HHMMSS [-force_stop [skip_rest]]] |"
 	      "\n\t [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]"
@@ -822,6 +846,7 @@ script_usage() ->
 	      "\n\t [-scale_timetraps]"
 	      "\n\t [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]"
 	      "\n\t [-basic_html]"
+	      "\n\t [-no_esc_chars]"
 	      "\n\t [-repeat N] |"
 	      "\n\t [-duration HHMMSS [-force_stop [skip_rest]]] |"
 	      "\n\t [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]\n\n"),
@@ -847,7 +872,8 @@ script_usage() ->
 	      "\n\t [-multiply_timetraps N]"
 	      "\n\t [-scale_timetraps]"
 	      "\n\t [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]"
-	      "\n\t [-basic_html]\n\n").
+	      "\n\t [-basic_html]"
+	      "\n\t [-no_esc_chars]\n\n").
 
 %%%-----------------------------------------------------------------
 %%% @hidden
@@ -1089,7 +1115,17 @@ run_test2(StartOpts) ->
 		application:set_env(common_test, basic_html, BasicHtmlBool),
 		BasicHtmlBool		
     end,
-
+    %% esc_chars - used by ct_logs
+    EscChars =
+	case proplists:get_value(esc_chars, StartOpts) of
+	    undefined ->
+		application:set_env(common_test, esc_chars, true),
+		undefined;
+	    EscCharsBool ->
+		application:set_env(common_test, esc_chars, EscCharsBool),
+		EscCharsBool		
+    end,
+    %% disable_log_cache - used by ct_logs
     case proplists:get_value(disable_log_cache, StartOpts) of
 	undefined ->
 	    application:set_env(common_test, disable_log_cache, false);
@@ -1104,6 +1140,7 @@ run_test2(StartOpts) ->
 		 cover = Cover, cover_stop = CoverStop,
 		 step = Step, logdir = LogDir,
 		 logopts = LogOpts, basic_html = BasicHtml,
+		 esc_chars = EscChars,
 		 config = CfgFiles,
 		 verbosity = Verbosity,
 		 event_handlers = EvHandlers,
@@ -1445,6 +1482,7 @@ get_data_for_node(#testspec{label = Labels,
 			    logdir = LogDirs,
 			    logopts = LogOptsList,
 			    basic_html = BHs,
+			    esc_chars = EscChs,
 			    stylesheet = SSs,
 			    verbosity = VLvls,
 			    silent_connections = SilentConnsList,
@@ -1472,6 +1510,7 @@ get_data_for_node(#testspec{label = Labels,
 		  LOs -> LOs
 	      end,
     BasicHtml = proplists:get_value(Node, BHs),
+    EscChars = proplists:get_value(Node, EscChs),
     Stylesheet = proplists:get_value(Node, SSs),
     Verbosity = case proplists:get_value(Node, VLvls) of
 		    undefined -> [];
@@ -1498,6 +1537,7 @@ get_data_for_node(#testspec{label = Labels,
 	  logdir = LogDir,
 	  logopts = LogOpts,
 	  basic_html = BasicHtml,
+	  esc_chars = EscChars,
 	  stylesheet = Stylesheet,
 	  verbosity = Verbosity,
 	  silent_connections = SilentConns,
@@ -2182,10 +2222,18 @@ do_run_test(Tests, Skip, Opts0) ->
 	    %% test_server needs to know the include path too
 	    InclPath = case application:get_env(common_test, include) of
 			   {ok,Incls} -> Incls;
-			   _ -> []
+			   _          -> []
 		       end,
 	    application:set_env(test_server, include, InclPath),
 
+	    %% copy the escape characters setting to test_server
+	    EscChars =
+		case application:get_env(common_test, esc_chars) of
+		    {ok,ECBool} -> ECBool;
+		    _           -> true
+		end,
+	    application:set_env(test_server, esc_chars, EscChars),
+
 	    test_server_ctrl:start_link(local),
 
 	    %% let test_server expand the test tuples and count no of cases
@@ -3071,6 +3119,10 @@ opts2args(EnvStartOpts) ->
 			  [{basic_html,[]}];
 		     ({basic_html,false}) ->
 			  [];
+		     ({esc_chars,false}) ->
+			  [{no_esc_chars,[]}];
+		     ({esc_chars,true}) ->
+			  [];
 		     ({event_handler,EH}) when is_atom(EH) ->
 			  [{event_handler,[atom_to_list(EH)]}];
 		     ({event_handler,EHs}) when is_list(EHs) ->
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 5cd52bd042..61d8f49dcc 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -1146,8 +1146,9 @@ should_be_added(Tag,Node,_Data,Spec) ->
     if 
 	%% list terms *without* possible duplicates here
 	Tag == logdir;       Tag == logopts;
-	Tag == basic_html;   Tag == label;
-	Tag == auto_compile; Tag == abort_if_missing_suites;
+	Tag == basic_html;   Tag == esc_chars;
+	Tag == label;        Tag == auto_compile;
+	Tag == abort_if_missing_suites;
 	Tag == stylesheet;   Tag == verbosity;
 	Tag == silent_connections ->
 	    lists:keymember(ref2node(Node,Spec#testspec.nodes),1,
@@ -1544,6 +1545,8 @@ valid_terms() ->
      {logopts,3},
      {basic_html,2},
      {basic_html,3},
+     {esc_chars,2},
+     {esc_chars,3},
      {verbosity,2},
      {verbosity,3},
      {silent_connections,2},
diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl
index b7fa7947e2..3561a0a2d3 100644
--- a/lib/common_test/src/ct_util.erl
+++ b/lib/common_test/src/ct_util.erl
@@ -459,6 +459,7 @@ loop(Mode,TestData,StartDir) ->
 		    error:badarg -> []
 		end,
 	    ct_hooks:terminate(Callbacks),
+
 	    close_connections(ets:tab2list(?conn_table)),
 	    ets:delete(?conn_table),
 	    ets:delete(?board_table),
diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl
index 2c1954c2b3..bdfe2041a5 100644
--- a/lib/common_test/src/ct_util.hrl
+++ b/lib/common_test/src/ct_util.hrl
@@ -37,6 +37,7 @@
 		   logdir=["."],
 		   logopts=[],
 		   basic_html=[],
+		   esc_chars=[],
 		   verbosity=[],
 		   silent_connections=[],
 		   cover=[],
diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl
index a8c4a455e1..780fbea79a 100644
--- a/lib/common_test/src/cth_log_redirect.erl
+++ b/lib/common_test/src/cth_log_redirect.erl
@@ -130,7 +130,14 @@ handle_event(Event, #eh_state{log_func = LogFunc} = State) ->
 						tag_event(Event)),
 	    if is_list(SReport) ->
 		    SaslHeader = format_header(State),
-		    ct_logs:LogFunc(sasl, ?STD_IMPORTANCE, SaslHeader, SReport, []);
+		    case LogFunc of
+			tc_log ->
+			    ct_logs:tc_log(sasl, ?STD_IMPORTANCE,
+					   SaslHeader, SReport, [], []);
+			tc_log_async ->
+			    ct_logs:tc_log_async(sasl, ?STD_IMPORTANCE,
+						 SaslHeader, SReport, [])
+		    end;
 	       true -> %% Report is an atom if no logging is to be done
 		    ignore
 	    end
@@ -139,7 +146,14 @@ handle_event(Event, #eh_state{log_func = LogFunc} = State) ->
 		tag_event(Event),io_lib),
     if is_list(EReport) ->
 	    ErrHeader = format_header(State),
-	    ct_logs:LogFunc(error_logger, ?STD_IMPORTANCE, ErrHeader, EReport, []);
+	    case LogFunc of
+		tc_log ->
+		    ct_logs:tc_log(error_logger, ?STD_IMPORTANCE,
+				   ErrHeader, EReport, [], []);
+		tc_log_async ->
+		    ct_logs:tc_log_async(error_logger, ?STD_IMPORTANCE,
+					 ErrHeader, EReport, [])
+	    end;
        true -> %% Report is an atom if no logging is to be done
 	    ignore
     end,