From a908160088b654e436bdcdd32369c29603ca5fa7 Mon Sep 17 00:00:00 2001
From: Siri Hansen <siri@erlang.org>
Date: Mon, 19 Nov 2012 15:13:47 +0100
Subject: [test_server] Fix erl2html2.erl to handle badly indented files

Line numbering of erlang files that were not correctly indented could
be wrong after coverting to html with erl2html2:convert/[2,3]. This has
been corrected.

This commit also fixes the following:

* There are now link targets for each line and not only for each 10th
  line - meaning that links from test logs are now to the exact line,
  and not to the last number for which N rem 10 == 0.
* there will only be one link target per function, i.e. the faulty
  link targets for function clauses are removed.
* link targets for function now includes the arity (e.g. func/1 has a
  link target "func-1")

And some tests are added.
---
 lib/test_server/src/erl2html2.erl        | 228 ++++++++++++++-----------------
 lib/test_server/src/test_server_ctrl.erl |   2 +-
 lib/test_server/src/test_server_sup.erl  |   7 +-
 3 files changed, 104 insertions(+), 133 deletions(-)

(limited to 'lib/test_server/src')

diff --git a/lib/test_server/src/erl2html2.erl b/lib/test_server/src/erl2html2.erl
index 6891e87e48..290acc5040 100644
--- a/lib/test_server/src/erl2html2.erl
+++ b/lib/test_server/src/erl2html2.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2012. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,19 +18,9 @@
 %%
 
 %%%------------------------------------------------------------------
-%%% Purpose:Convert Erlang files to html. (Pretty faaast... :-)
+%%% Purpose:Convert Erlang files to html.
 %%%------------------------------------------------------------------
 
-%--------------------------------------------------------------------
-% Some stats (Sparc5@110Mhz):
-% 4109 lines (erl_parse.erl):              3.00 secs
-% 1847 lines (application_controller.erl): 0.57 secs
-% 3160 lines (test_server.erl):            1.00 secs
-% 1199 lines (ts_estone.erl):              0.35 secs
-%
-% Avg: ~4.5e-4s/line, or ~0.45s/1000 lines, or ~2200 lines/sec.
-%--------------------------------------------------------------------
-
 -module(erl2html2).
 -export([convert/2, convert/3]).
 
@@ -54,132 +44,118 @@ convert(File, Dest) ->
     convert(File, Dest, Header).
     
 convert(File, Dest, Header) ->
-    case file:read_file(File) of
-	{ok, Bin} ->
-	    Code=binary_to_list(Bin),
-	    statistics(runtime),
-	    {Html1, Lines} = root(Code, [], 1),
-	    Html = [Header,
-		    "<pre>\n", Html1, "</pre>\n", 
-		    footer(Lines),"</body>\n</html>\n"],
-	    file:write_file(Dest, Html);
-	{error, Reason}  ->
-	    {error, Reason}
+    %% statistics(runtime),
+    case parse_file(File) of
+	{ok,Functions} ->
+	    %% io:format("~p~n",[Functions]),
+	    case file:read_file(File) of
+		{ok,Bin} ->
+		    {Html,Lines} = build_html(Bin,Functions),
+		    Html1 = [Header,"<pre>\n",Html,"</pre>\n",
+			     footer(Lines),"</body>\n</html>\n"],
+		    file:write_file(Dest,Html1);
+		Error ->
+		    Error
+	    end;
+	Error ->
+	    Error
     end.
 
-root([], Res, Line) ->
-    {Res, Line};
-root([Char0|Code], Res, Line0) ->
-    Char = [Char0],
-    case Char of
-	"-" ->
-	    {Match, Line1, NewCode0, AttName} = 
-		read_to_char(Line0+1, Code, [], [$(, $.]),
-	    {_, Line2, NewCode, Stuff} = read_to_char(Line1, NewCode0, [], $\n),
-	    NewRes = [Res,linenum(Line0),"-<b>",AttName,
-			 "</b>",Match, Stuff, "\n"],
-	    root(NewCode, NewRes, Line2);
-	"%" ->
-	    {_, Line, NewCode, Stuff} = read_to_char(Line0+1, Code, [], $\n),
-	    NewRes = [Res,linenum(Line0),"<i>%",Stuff,"</i>\n"],
-	    root(NewCode, NewRes, Line);
-	"\n" ->
-	    root(Code, [Res,linenum(Line0), "\n"], Line0+1);
-	" " ->
-	    {_, Line, NewCode, Stuff} = read_to_char(Line0+1, Code, [], $\n),
-	    root(NewCode, [Res,linenum(Line0)," ",Stuff, "\n"],
-		 Line);
-	"\t" ->
-	    {_, Line, NewCode, Stuff} = read_to_char(Line0+1, Code, [], $\n),
-	    root(NewCode, [Res,linenum(Line0),"\t",Stuff, "\n"],
-		 Line);
-	[Chr|_] when Chr>96, Chr<123 ->
-	    %% Assumed to be function/clause start.
-	    %% FIXME: This will trivially generate non-unique anchors
-	    %% (one for each clause) --- which is illegal HTML.
-	    {_, Line1, NewCode0, FName0} = read_to_char(Line0+1, Code, [], $(),
-	    {_, Line2, NewCode, Stuff} = 
-                read_to_char(Line1,NewCode0, [], $\n),
-	    FuncName = [[Chr],FName0],
-	    NewRes=[Res,"<a name=",FuncName,">",
-		    linenum(Line0),"<b>",FuncName,"</b></a>",
-		    "(",Stuff, "\n"],
-	    root(NewCode, NewRes, Line2);
-	Chr ->
-	    {_, Line, NewCode, Stuff} = read_to_char(Line0+1, Code, [], $\n),
-	    root(NewCode, [Res,linenum(Line0),Chr,Stuff, "\n"],
-		 Line)
+%%%-----------------------------------------------------------------
+%%% Parse the input file to get the line numbers for all function
+%%% definitions. This will be used when creating link targets for each
+%%% function in build_html/5.
+%%%
+%%% All function clauses are also marked in order to allow
+%%% possibly_enhance/2 to write these in bold.
+parse_file(File) ->
+    case epp:parse_file(File,[],[]) of
+	{ok,Forms} ->
+	    {ok,get_functions(Forms,File,false)};
+	Error ->
+	    Error
     end.
 
-read_to_char(Line0, [], Res, _Chr) ->
-    {nomatch, Line0, [], Res};
-read_to_char(Line0, [Char|Code], Res, Chr) ->
-    case Char of
-	Chr -> {Char, Line0, Code, Res};
-	_ when is_list(Chr) ->
-	    case lists:member(Char,Chr) of
-		true -> 
-		    {Char, Line0, Code, Res};
-		false -> 
-		    {Line,NewCode,NewRes} = maybe_convert(Line0,Code,Res,Char),
-		    read_to_char(Line, NewCode, NewRes, Chr)
-	    end;
-	_ ->
-	    {Line,NewCode,NewRes} = maybe_convert(Line0,Code,Res,Char),
-	    read_to_char(Line,NewCode, NewRes, Chr)
-    end.
+get_functions([{attribute,_,file,{File,_}}|Forms],File,_) ->
+    get_functions(Forms,File,true);
+get_functions([{attribute,_,file,{_OtherFile,_}}|Forms],File,_) ->
+    get_functions(Forms,File,false);
+get_functions([{function,L,F,A,[_|C]}|Forms],File,true) ->
+    Clauses = [{clause,CL} || {clause,CL,_,_,_} <- C],
+    [{atom_to_list(F),A,L} | Clauses] ++ get_functions(Forms,File,true);
+get_functions([_|Forms],File,InCorrectFile) ->
+    get_functions(Forms,File,InCorrectFile);
+get_functions([],_,_) ->
+    [].
 
-maybe_convert(Line0,Code,Res,Chr) ->
-    case Chr of
-	%% Quoted stuff should not have the highlighting like normal code
-	%% FIXME: unbalanced quotes (e.g. in comments) will cause trouble with
-	%% highlighting and line numbering in the rest of the module.
-	$" ->  
-	    {_, Line1, NewCode, Stuff0} = read_to_char(Line0, Code, [], $"),
-            {Line2,Stuff} = add_linenumbers(Line1,lists:flatten(Stuff0),[]),
-	    {Line2,NewCode,[Res,$",Stuff,$"]};
-	%% These chars have meaning in HTML, and *must* *not* be
-	%% written as themselves.
-	$& ->
-	    {Line0, Code, [Res,"&amp;"]};
-	$< ->
-	    {Line0, Code, [Res,"&lt;"]};
-	$> ->
-	    {Line0, Code, [Res,"&gt;"]};
-	%% Everything else is simply copied.
-	OtherChr ->
-	    {Line0, Code, [Res,OtherChr]}
-    end.
+%%%-----------------------------------------------------------------
+%%% Add a link target for each line and one for each function definition.
+build_html(Bin,Functions) ->
+    build_html(binary:split(Bin,<<"\n">>),1,Functions,false,[]).
 
-add_linenumbers(Line,[Chr|Chrs],Res) ->
-    case Chr of
-	$\n -> add_linenumbers(Line+1,Chrs,[Res,$\n,linenum(Line)]);
-	_  -> add_linenumbers(Line,Chrs,[Res,Chr])
-    end;
-add_linenumbers(Line,[],Res) ->
-    {Line,Res}.
+build_html(Split,L,[{F,A,L}|Functions],_IsFuncDef,Acc) ->
+    FALink = http_uri:encode(F++"-"++integer_to_list(A)),
+    NewAcc = [Acc,"<a name=\"",FALink,"\"/>"],
+    build_html(Split,L,Functions,true,NewAcc);
+build_html(Split,L,[{clause,L}|Functions],_IsFuncDef,Acc) ->
+    build_html(Split,L,Functions,true,Acc);
+build_html([Line,Bin],L,Functions,IsFuncDef,Acc) ->
+    NewAcc = [Acc,line_number(L),line(binary_to_list(Line),IsFuncDef),$\n],
+    build_html(binary:split(Bin,<<"\n">>),L+1,Functions,false,NewAcc);
+build_html([Line],L,Functions,IsFuncDef,Acc) ->
+    {[Acc,line_number(L),line(binary_to_list(Line),IsFuncDef)],L}.
 
-%% Make nicely indented line numbers.
-linenum(Line) ->
-    Num = integer_to_list(Line),
-    A = case Line rem 10 of
-	    0 -> "<a name=\"" ++ Num ++"\"></a>";
-	    _ -> []
-	end,
+line_number(L) ->
+    LStr = integer_to_list(L),
     Pred =
-	case length(Num) of
+	case length(LStr) of
 	    Length when Length < 5 ->
 		lists:duplicate(5-Length,$\s);
 	    _ -> 
 		[]
 	end, 
-    [A,Pred,integer_to_list(Line),":"].
+    ["<a name=\"",LStr,"\"/>",Pred,LStr,": "].
+
+line(Str,IsFuncDef) ->
+    Str1 = htmlize(Str),
+    possibly_enhance(Str1,IsFuncDef).
+
+%%%-----------------------------------------------------------------
+%%% Substitute special characters that should not appear in HTML
+htmlize([$<|Str]) ->
+    [$&,$l,$t,$;|htmlize(Str)];
+htmlize([$>|Str]) ->
+    [$&,$g,$t,$;|htmlize(Str)];
+htmlize([$&|Str]) ->
+    [$&,$a,$m,$p,$;|htmlize(Str)];
+htmlize([$"|Str]) ->
+    [$&,$q,$u,$o,$t,$;|htmlize(Str)];
+htmlize([Ch|Str]) ->
+    [Ch|htmlize(Str)];
+htmlize([]) ->
+    [].
+
+%%%-----------------------------------------------------------------
+%%% Write comments in italic and function definitions in bold.
+possibly_enhance(Str,true) ->
+    case lists:splitwith(fun($() -> false; (_) -> true end, Str) of
+	{_,[]} -> Str;
+	{F,A} -> ["<b>",F,"</b>",A]
+    end;
+possibly_enhance([$%|_]=Str,_) ->
+    ["<i>",Str,"</i>"];
+possibly_enhance([$-|_]=Str,_) ->
+    possibly_enhance(Str,true);
+possibly_enhance(Str,false) ->
+    Str.
 
+%%%-----------------------------------------------------------------
+%%% End of the file
 footer(_Lines) ->
-    "".
-%%    {_, Time} = statistics(runtime),
-%%    io:format("Converted ~p lines in ~.2f Seconds.~n",
-%%	      [Lines, Time/1000]),
-%%    S = "<i>The transformation of this file (~p lines) took ~.2f seconds</i>",
-%%    F = lists:flatten(io_lib:format(S, [Lines, Time/1000])),
-%%    ["<hr size=1>",F,"<br>\n"].
+   "".
+   %% {_, Time} = statistics(runtime),
+   %% io:format("Converted ~p lines in ~.2f Seconds.~n",
+   %% 	      [_Lines, Time/1000]),
+   %% S = "<i>The transformation of this file (~p lines) took ~.2f seconds</i>",
+   %% F = lists:flatten(io_lib:format(S, [_Lines, Time/1000])),
+   %% ["<hr size=1>",F,"<br>\n"].
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index 7b2ebcefc0..bc08c12089 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -1796,7 +1796,7 @@ start_minor_log_file1(Mod, Func, LogDir, AbsName, MFA) ->
 	  lists:member(no_src, get(test_server_logopts))} of
 	{true,false} ->
 	    print(Lev, "<a href=\"~s#~s\">source code for ~p:~p/1</a>\n",
-		  [SrcListing,Func,Mod,Func]);
+		  [SrcListing,atom_to_list(Func)++"-1",Mod,Func]);
 	_ -> ok
     end,
 
diff --git a/lib/test_server/src/test_server_sup.erl b/lib/test_server/src/test_server_sup.erl
index ba5bb9f5d2..c7553cccb5 100644
--- a/lib/test_server/src/test_server_sup.erl
+++ b/lib/test_server/src/test_server_sup.erl
@@ -568,16 +568,11 @@ format_loc1({Mod,Func,Line}) ->
 	{false,[$E,$T,$I,$U,$S,$_|_]}  ->
 	    io_lib:format("{~s,~w,<a href=\"~s~s#~w\">~w</a>}",
 			  [ModStr,Func,downcase(ModStr),?src_listing_ext,
-			   round_to_10(Line),Line]);
+			   Line,Line]);
 	_ ->
 	    io_lib:format("{~s,~w,~w}",[ModStr,Func,Line])
     end.
 
-round_to_10(N) when (N rem 10) == 0 ->
-    N;
-round_to_10(N) ->
-    trunc(N/10)*10.
-
 downcase(S) -> downcase(S, []).
 downcase([Uc|Rest], Result) when $A =< Uc, Uc =< $Z ->
     downcase(Rest, [Uc-$A+$a|Result]);
-- 
cgit v1.2.3


From ee65e005b0e18d92d786b14e2dd3cf7b8584ca27 Mon Sep 17 00:00:00 2001
From: Siri Hansen <siri@erlang.org>
Date: Tue, 20 Nov 2012 12:30:45 +0100
Subject: [test_server] Minimize memory usage in erl2html2:convert/[2,3]

Reading form by form and line by line, instead of reading the complete
file in one go.
---
 lib/test_server/src/erl2html2.erl | 111 ++++++++++++++++++++++----------------
 1 file changed, 66 insertions(+), 45 deletions(-)

(limited to 'lib/test_server/src')

diff --git a/lib/test_server/src/erl2html2.erl b/lib/test_server/src/erl2html2.erl
index 290acc5040..9c459c05d4 100644
--- a/lib/test_server/src/erl2html2.erl
+++ b/lib/test_server/src/erl2html2.erl
@@ -42,18 +42,31 @@ convert(File, Dest) ->
 	      "<body bgcolor=\"white\" text=\"black\""
 	      " link=\"blue\" vlink=\"purple\" alink=\"red\">\n"],
     convert(File, Dest, Header).
-    
+
+
 convert(File, Dest, Header) ->
     %% statistics(runtime),
     case parse_file(File) of
 	{ok,Functions} ->
-	    %% io:format("~p~n",[Functions]),
-	    case file:read_file(File) of
-		{ok,Bin} ->
-		    {Html,Lines} = build_html(Bin,Functions),
-		    Html1 = [Header,"<pre>\n",Html,"</pre>\n",
-			     footer(Lines),"</body>\n</html>\n"],
-		    file:write_file(Dest,Html1);
+	    %% {_, Time1} = statistics(runtime),
+	    %% io:format("Parsed file in ~.2f Seconds.~n",[Time1/1000]),
+	    case file:open(File,[raw,{read_ahead,10000}]) of
+		{ok,SFd} ->
+		    case file:open(Dest,[write,raw]) of
+			{ok,DFd} ->
+			    file:write(DFd,[Header,"<pre>\n"]),
+			    Lines = build_html(SFd,DFd,Functions),
+			    file:write(DFd,["</pre>\n",footer(),
+					    "</body>\n</html>\n"]),
+			    %% {_, Time2} = statistics(runtime),
+			    %% io:format("Converted ~p lines in ~.2f Seconds.~n",
+			    %% 	      [Lines, Time2/1000]),
+			    file:close(SFd),
+			    file:close(DFd),
+			    ok;
+			Error ->
+			    Error
+		    end;
 		Error ->
 		    Error
 	    end;
@@ -69,41 +82,55 @@ convert(File, Dest, Header) ->
 %%% All function clauses are also marked in order to allow
 %%% possibly_enhance/2 to write these in bold.
 parse_file(File) ->
-    case epp:parse_file(File,[],[]) of
-	{ok,Forms} ->
-	    {ok,get_functions(Forms,File,false)};
-	Error ->
-	    Error
+    case epp:open(File, [], []) of
+	{ok,Epp} ->
+	    Forms = parse_file(Epp,File,false),
+	    epp:close(Epp),
+	    {ok,Forms};
+	{error,E} ->
+	    {error,E}
     end.
 
-get_functions([{attribute,_,file,{File,_}}|Forms],File,_) ->
-    get_functions(Forms,File,true);
-get_functions([{attribute,_,file,{_OtherFile,_}}|Forms],File,_) ->
-    get_functions(Forms,File,false);
-get_functions([{function,L,F,A,[_|C]}|Forms],File,true) ->
-    Clauses = [{clause,CL} || {clause,CL,_,_,_} <- C],
-    [{atom_to_list(F),A,L} | Clauses] ++ get_functions(Forms,File,true);
-get_functions([_|Forms],File,InCorrectFile) ->
-    get_functions(Forms,File,InCorrectFile);
-get_functions([],_,_) ->
-    [].
+
+parse_file(Epp,File,InCorrectFile) ->
+    case epp:parse_erl_form(Epp) of
+	{ok,Form} ->
+	    case Form of
+		{attribute,_,file,{File,_}} ->
+		    parse_file(Epp,File,true);
+		{attribute,_,file,{_OtherFile,_}} ->
+		    parse_file(Epp,File,false);
+		{function,L,F,A,[_|C]} when InCorrectFile ->
+		    Clauses = [{clause,CL} || {clause,CL,_,_,_} <- C],
+		    [{atom_to_list(F),A,L} | Clauses] ++
+			parse_file(Epp,File,true);
+		_ ->
+		    parse_file(Epp,File,InCorrectFile)
+	    end;
+	{error,_E} ->
+	    parse_file(Epp,File,InCorrectFile);
+	{eof,_Location} ->
+	    []
+    end.
 
 %%%-----------------------------------------------------------------
 %%% Add a link target for each line and one for each function definition.
-build_html(Bin,Functions) ->
-    build_html(binary:split(Bin,<<"\n">>),1,Functions,false,[]).
+build_html(SFd,DFd,Functions) ->
+    build_html(SFd,DFd,file:read_line(SFd),1,Functions,false).
 
-build_html(Split,L,[{F,A,L}|Functions],_IsFuncDef,Acc) ->
+build_html(SFd,DFd,{ok,Str},L,[{F,A,L}|Functions],_IsFuncDef) ->
     FALink = http_uri:encode(F++"-"++integer_to_list(A)),
-    NewAcc = [Acc,"<a name=\"",FALink,"\"/>"],
-    build_html(Split,L,Functions,true,NewAcc);
-build_html(Split,L,[{clause,L}|Functions],_IsFuncDef,Acc) ->
-    build_html(Split,L,Functions,true,Acc);
-build_html([Line,Bin],L,Functions,IsFuncDef,Acc) ->
-    NewAcc = [Acc,line_number(L),line(binary_to_list(Line),IsFuncDef),$\n],
-    build_html(binary:split(Bin,<<"\n">>),L+1,Functions,false,NewAcc);
-build_html([Line],L,Functions,IsFuncDef,Acc) ->
-    {[Acc,line_number(L),line(binary_to_list(Line),IsFuncDef)],L}.
+    file:write(DFd,["<a name=\"",FALink,"\"/>"]),
+    build_html(SFd,DFd,{ok,Str},L,Functions,true);
+build_html(SFd,DFd,{ok,Str},L,[{clause,L}|Functions],_IsFuncDef) ->
+    build_html(SFd,DFd,{ok,Str},L,Functions,true);
+build_html(SFd,DFd,{ok,Str},L,Functions,IsFuncDef) ->
+    LStr = line_number(L),
+    Str1 = line(Str,IsFuncDef),
+    file:write(DFd,[LStr,Str1]),
+    build_html(SFd,DFd,file:read_line(SFd),L+1,Functions,false);
+build_html(_SFd,_DFd,eof,L,_Functions,_IsFuncDef) ->
+    L.
 
 line_number(L) ->
     LStr = integer_to_list(L),
@@ -143,7 +170,7 @@ possibly_enhance(Str,true) ->
 	{F,A} -> ["<b>",F,"</b>",A]
     end;
 possibly_enhance([$%|_]=Str,_) ->
-    ["<i>",Str,"</i>"];
+    ["<i>",Str--"\n","</i>","\n"];
 possibly_enhance([$-|_]=Str,_) ->
     possibly_enhance(Str,true);
 possibly_enhance(Str,false) ->
@@ -151,11 +178,5 @@ possibly_enhance(Str,false) ->
 
 %%%-----------------------------------------------------------------
 %%% End of the file
-footer(_Lines) ->
-   "".
-   %% {_, Time} = statistics(runtime),
-   %% io:format("Converted ~p lines in ~.2f Seconds.~n",
-   %% 	      [_Lines, Time/1000]),
-   %% S = "<i>The transformation of this file (~p lines) took ~.2f seconds</i>",
-   %% F = lists:flatten(io_lib:format(S, [_Lines, Time/1000])),
-   %% ["<hr size=1>",F,"<br>\n"].
+footer() ->
+    "".
-- 
cgit v1.2.3