%% ``The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved via the world wide web at http://www.erlang.org/. %% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. %% %% The Initial Developer of the Original Code is Ericsson Utvecklings AB. %% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings %% AB. All Rights Reserved.'' %% %% $Id$ %% -module(bench). %% User interface -export([run/0]). %% Exported to be used in spawn -export([measure/4]). %% Internal constants -define(MAX, 999999999999999). -define(RANGE_MAX, 16#7ffffff). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Interface %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%--------------------------------------------------------------------------- %% run() -> _ %% %% Compiles and runs all benchmarks in the current directory, %% and creates a report %%--------------------------------------------------------------------------- run() -> run(compiler_options()). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Generic Benchmark functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%--------------------------------------------------------------------------- %% compiler_options() -> OptionsList %% OptionsList = list() - See Erlang/OTP module compile %%--------------------------------------------------------------------------- compiler_options() -> [report_errors, report_warnings]. %%--------------------------------------------------------------------------- %% run(OptionsList) -> %% OptionsList = list() - See Erlang/OTP module compile %% %% Help function to run/0. %%--------------------------------------------------------------------------- run(OptionsList) -> Bms = compile_benchmarks(OptionsList), run_benchmarks(Bms), report(). %%--------------------------------------------------------------------------- %% compile_benchmarks(OptionsList) -> [BmInfo| _] %% OptionsList = list() - See Erlang/OTP module compile %% BmInfo = {Module, Iterations, [BmFunctionName| _]} %% Module = atom() %% Iterations = integer() %% BmFunctionName = atom() %% %% Compiles all benchmark modules in the current directory and %% returns info about the benchmarks. %%--------------------------------------------------------------------------- compile_benchmarks(OptionsList) -> {ok, FilesInCurrentDir} = file:list_dir("."), ErlFiles = [ErlFile || ErlFile <- lists:sort(FilesInCurrentDir), lists:suffix(".erl", ErlFile)], lists:foldr(fun(File, BmInfoAcc) -> case lists:suffix("_bm.erl", File) of true -> BmInfo = bm_compile(File, OptionsList), [BmInfo | BmInfoAcc]; false -> just_compile(File, OptionsList), BmInfoAcc end end, [], ErlFiles). %%--------------------------------------------------------------------------- %% just_compile(FileName, OptionsList) -> ok %% FileName = string() %% OptionsList = list() - See Erlang/OTP module compile %% %% Compiles a support module. %%--------------------------------------------------------------------------- just_compile(FileName, OptionsList) -> io:format("Compiling ~s...\n", [FileName]), % Progress info to user case c:c(FileName, OptionsList) of {ok, _Mod} -> ok; %% If compilation fails there is no point in trying to continue error -> Reason = lists:flatten( io_lib:format("Could not compile file ~s", [FileName])), exit(self(), Reason) end. %%--------------------------------------------------------------------------- %% bm_compile(FileName, OptionsList) -> BmInfo %% FileName = string() %% OptionsList = list() - See Erlang/OTP module compile %% BmInfo = {Module, Iterations, [BmFunctionName| _]} %% Iterations = integer() %% Module = atom() %% BmFunctionName = atom() %% %% Compiles the benchmark module implemented in <FileName> and returns %% information about the benchmark tests. %%--------------------------------------------------------------------------- bm_compile(FileName, OptionsList) -> io:format("Compiling ~s...\n", [FileName]), % Progress info to user case c:c(FileName, OptionsList) of {ok, Mod} -> bm_cases(Mod); %% If compilation fails there is no point in trying to continue error -> Reason = lists:flatten( io_lib:format("Could not compile file ~s", [FileName])), exit(self(), Reason) end. %%--------------------------------------------------------------------------- %% bm_cases(Module) -> {Module, Iter, [BmFunctionName |_]} %% Module = atom() %% Iter = integer() %% BmFunctionName = atom() %% %% Fetches the number of iterations and the names of the benchmark %% functions for the module <Module>. %%--------------------------------------------------------------------------- bm_cases(Module) -> case catch Module:benchmarks() of {Iter, BmList} when integer(Iter), list(BmList) -> {Module, Iter, BmList}; %% The benchmark is incorrect implemented there is no point in %% trying to continue Other -> Reason = lists:flatten( io_lib:format("Incorrect return value: ~p " "from ~p:benchmarks()", [Other, Module])), exit(self(), Reason) end. %%--------------------------------------------------------------------------- %% run_benchmarks(Bms) -> %% Bms = [{Module, Iter, [BmFunctionName |_]} | _] %% Module = atom() %% Iter = integer() %% BmFunctionName = atom() %% %% Runs all the benchmark tests described in <Bms>. %%--------------------------------------------------------------------------- run_benchmarks(Bms) -> Ver = erlang:system_info(version), Machine = erlang:system_info(machine), SysInfo = {Ver,Machine}, Res = [bms_run(Mod, Tests, Iter, SysInfo) || {Mod,Iter,Tests} <- Bms], %% Create an intermediate file that is later used to generate a bench %% mark report. Name = Ver ++ [$.|Machine] ++ ".bmres", {ok, IntermediatFile} = file:open(Name, [write]), %% Create mark that identifies version of the benchmark modules io:format(IntermediatFile, "~p.\n", [erlang:phash(Bms, ?RANGE_MAX)]), io:format(IntermediatFile, "~p.\n", [Res]), file:close(IntermediatFile). %%--------------------------------------------------------------------------- %% bms_run(Module, BmTests, Iter, Info) -> %% Module = atom(), %% BmTests = [BmFunctionName|_], %% BmFunctionName = atom() %% Iter = integer(), %% SysInfo = {Ver, Machine} %% Ver = string() %% Machine = string() %% %% Runs all benchmark tests in module <Module>. %%--------------------------------------------------------------------------- bms_run(Module, BmTests, Iter, SysInfo) -> io:format("Running ~s:", [Module]), % Progress info to user Res = {Module,{SysInfo,[{Bm, bm_run(Module, Bm, Iter)} || Bm <- BmTests]}}, io:nl(), Res. %%--------------------------------------------------------------------------- %% bm_run(Module, BmTest, Iter) -> Elapsed %% Module = atom(), %% BmTest = atom(), %% Iter = integer() %% Elapsed = integer() - elapsed time in milliseconds. %% %% Runs the benchmark Module:BmTest(Iter) %%--------------------------------------------------------------------------- bm_run(Module, BmTest, Iter) -> io:format(" ~s", [BmTest]), % Progress info to user spawn_link(?MODULE, measure, [self(), Module, BmTest, Iter]), receive {Elapsed, ok} -> Elapsed; {_Elapsed, Fault} -> io:nl(), Reason = lists:flatten( io_lib:format("~w", [Fault])), exit(self(), Reason) end. %%--------------------------------------------------------------------------- %% measure(Parent, Module, BmTest, Iter) -> _ %% Parent = pid(), %% Module = atom(), %% BmTest = atom(), %% Iter = integer() %% %% Measures the time it take to execute Module:Bm(Iter) %% and send the result to <Parent>. %%--------------------------------------------------------------------------- measure(Parent, Module, BmTest, Iter) -> statistics(runtime), Res = (catch apply(Module, BmTest, [Iter])), {_TotalRunTime, TimeSinceLastCall} = statistics(runtime), Parent ! {TimeSinceLastCall, Res}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Report functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%--------------------------------------------------------------------------- %% report() -> _ %% %% Creates a report of the bench marking test that appeals to a human. %% Currently this means creating a html-file. (Other formats could be added) %%--------------------------------------------------------------------------- report() -> {ok, AllFiles} = file:list_dir("."), BmResultFiles = [File || File <- AllFiles, lists:suffix(".bmres", File)], Results = fetch_bmres_data(BmResultFiles), create_report(Results). %%--------------------------------------------------------------------------- %% fetch_bmres_data(BmResultFiles) -> Results %% BmResultFiles = [FileName | _] %% FileName = string() %% Results = [[{Bm, Res} | _]] %% Bm = atom() - Name of benchmark module %% Res = [{VersionInfo, [{Test, Time} | _]}] %% VersionInfo = {Ver, Machine} %% Ver = string() %% Machine = string() %% Test = atom() %% Time = integer() %% %% Reads result data from intermediate files %%--------------------------------------------------------------------------- fetch_bmres_data(BmResultFiles) -> fetch_bmres_data(BmResultFiles, [], undefined). %%--------------------------------------------------------------------------- %% fetch_bmres_data(BmResultFiles, AccResData, Check) -> Results %% BmResultFiles = [FileName | _] %% FileName = string() %% AccResData = see Results fetch_bmres_data/1 %% Check = integer() | undefined (first time) %% %% Help function to fetch_bmres_data/1 %%--------------------------------------------------------------------------- fetch_bmres_data([], AccResData, _Check) -> AccResData; fetch_bmres_data([Name | BmResultFiles], AccResData, Check) -> {DataList, NewCheck} = read_bmres_file(Name, Check), fetch_bmres_data(BmResultFiles, [DataList| AccResData], NewCheck). %%--------------------------------------------------------------------------- %% read_bmres_file(Name, Check) -> %% Name = string() %% Check = integer() | undefined %% %% Reads the data from the result files. Checks that all result %% files where created with the same set of tests. %%--------------------------------------------------------------------------- read_bmres_file(Name, Check) -> case file:consult(Name) of {ok, [Check1, List]} when Check =:= undefined, integer(Check1) -> {List, Check1}; {ok, [Check, List]} when integer(Check) -> {List, Check}; {ok, [Check1, _List]} when integer(Check1) -> Reason = lists:flatten( io_lib:format("Different test setup, remove old setup " "result by removing *.bmres files and " "try again", [])), exit(self(), Reason); {error, Reason} when atom(Reason) -> exit(self(), Reason); {error, Reason} -> exit(self(), file:format(Reason)) end. %%--------------------------------------------------------------------------- %% create_report(Results) -> %% Results = see Results fetch_bmres_data/1 %% %% Organizes <Result> so it will be right for create_html_report/1 %% i.e. group results for the same benchmark test, run on different versions %% of erlang. %%--------------------------------------------------------------------------- create_report(Results) -> Dictionary = lists:foldl(fun(BmResultList, Dict0) -> lists:foldl(fun({Bm, VerResult}, Dict1) -> dict:append(Bm, VerResult, Dict1) end,Dict0, BmResultList) end, dict:new(), Results), create_html_report(dict:to_list(Dictionary)). %%--------------------------------------------------------------------------- %% create_html_report(ResultList) -> _ %% ResultList = [{Bm, Res} | _] %% Bm = atom() - Name of benchmark module %% Res = [{VersionInfo, [{Test, Time} | _]} | _] %% VersionInfo = {Ver, Machine} %% Ver = string() %% Machine = string() %% Test = atom() %% Time = integer() %% %% Writes the result to an html-file %%--------------------------------------------------------------------------- create_html_report(ResultList) -> {ok, OutputFile} = file:open("index.html", [write]), %% Create the begining of the result html-file. Head = Title = "Benchmark Results", io:put_chars(OutputFile, "<html>\n"), io:put_chars(OutputFile, "<head>\n"), io:format(OutputFile, "<title>~s</title>\n", [Title]), io:put_chars(OutputFile, "</head>\n"), io:put_chars(OutputFile, "<body bgcolor=\"#FFFFFF\" text=\"#000000\"" ++ " link=\"#0000FF\" vlink=\"#800080\" alink=\"#FF0000\">\n"), io:format(OutputFile, "<h1>~s</h1>\n", [Head]), %% Add the result tables lists:foreach(fun(Element) -> create_html_table(OutputFile, Element) end, ResultList), %% Put in the end-html tags io:put_chars(OutputFile, "</body>\n"), io:put_chars(OutputFile, "</html>\n"), file:close(OutputFile). %%--------------------------------------------------------------------------- %% create_html_table(File, {Bm, Res}) -> _ %% File = file() - html file to write data to. %% Bm = atom() - Name of benchmark module %% Res = [{VersionInfo, [{Test, Time} | _]}] %% VersionInfo = {Ver, Machine} %% Ver = string() %% Machine = string() %% Test = atom() %% Time = integer() %% %% Creates a html table that displays the result of the benchmark <Bm>. %%--------------------------------------------------------------------------- create_html_table(File, {Bm, Res}) -> {MinTime, Order} = min_time_and_sort(Res), io:format(File, "<h2>~s</h2>\n" , [Bm]), %% Fun that calculates relative measure values and puts them in %% a dictionary RelativeMesureFun = fun({TestName, Time}, Dict1) -> dict:append(TestName, Time/MinTime, Dict1) end, %% For all erlang versions that the benchmark tests has been run, %% calculate the relative measure values and put them in a dictionary. ResultDict = lists:foldl(fun({_VerInfo, Bms}, Dict0) -> lists:foldl(RelativeMesureFun, Dict0, Bms) end, dict:new(), Res), %% Create the table and its headings io:put_chars(File, "<table border=0 cellpadding=1><tr>" "<td bgcolor=\"#000000\">\n"), io:put_chars(File, "<table cellpadding=3 border=0 cellspacing=1>\n"), io:put_chars(File, "<tr bgcolor=white>"), io:put_chars(File, "<td>Test</td>"), Heads = table_headers(Res), lists:foreach(fun({Ver,Machine}) -> io:format(File, "<td>~s<br>~s</td>", [Ver,Machine]) end, Heads), io:put_chars(File, "</tr>\n"), %% Create table rows lists:foreach(fun(Name) -> create_html_row(File, Name, ResultDict) end, Order), %% Tabel end-tags io:put_chars(File, "</table></td></tr></table>\n"), %% Create link to benchmark source code io:format(File, "<p><a href=\"~s.erl\">Source for ~s.erl</a>\n", [Bm,Bm]). %%--------------------------------------------------------------------------- %% create_html_row(File, Name, Dict) -> _ %% File = file() - html file to write data to. %% Name = atom() - Name of benchmark test %% Dict = dict() - Dictonary where the relative time measures for %% the test can be found. %% %% Creates an actual html table-row. %%--------------------------------------------------------------------------- create_html_row(File, Name, Dict) -> ReletiveTimes = dict:fetch(Name, Dict), io:put_chars(File, "<tr bgcolor=white>\n"), io:format(File, "<td>~s</td>", [Name]), lists:foreach(fun(Time) -> io:format(File, "<td>~-8.2f</td>", [Time]) end, ReletiveTimes), io:put_chars(File, "</tr>\n"). %%--------------------------------------------------------------------------- %% min_time_and_sort(ResultList) -> {MinTime, Order} %% ResultList = [{VersionInfo, [{Test, Time} | _]}] %% MinTime = integer() - The execution time of the fastes test %% Order = [BmFunctionName|_] - the order of the testcases in %% increasing execution time. %% BmFunctionName = atom() %%--------------------------------------------------------------------------- min_time_and_sort(ResultList) -> %% Use the results from the run on the highest version %% of Erlang as norm. {_, TestRes} = lists:foldl(fun ({{Ver, _}, ResList}, {CurrentVer, _}) when Ver > CurrentVer -> {Ver, ResList}; (_, VerAndRes) -> VerAndRes end, {"0", []}, ResultList), {lists:foldl(fun ({_, Time0}, Min1) when Time0 < Min1 -> Time0; (_, Min1) -> Min1 end, ?MAX, TestRes), [Name || {Name, _} <- lists:keysort(2, TestRes)]}. %%--------------------------------------------------------------------------- %% table_headers(VerResultList) -> SysInfo %% VerResultList = [{{Ver, Machine},[{BmFunctionName, Time}]} | _] %% Ver = string() %% Machine = string() %% BmFunctionName = atom() %% Time = integer() %% SysInfo = {Ver, Machine} %%--------------------------------------------------------------------------- table_headers(VerResultList) -> [SysInfo || {SysInfo, _} <- VerResultList].