diff options
Diffstat (limited to 'system/doc/efficiency_guide/bench.erl')
| -rw-r--r-- | system/doc/efficiency_guide/bench.erl | 488 |
1 files changed, 488 insertions, 0 deletions
diff --git a/system/doc/efficiency_guide/bench.erl b/system/doc/efficiency_guide/bench.erl new file mode 100644 index 0000000000..8f26b8d0af --- /dev/null +++ b/system/doc/efficiency_guide/bench.erl @@ -0,0 +1,488 @@ +%% ``The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved via the world wide web at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% The Initial Developer of the Original Code is Ericsson Utvecklings AB. +%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings +%% AB. All Rights Reserved.'' +%% +%% $Id$ +%% + +-module(bench). + +%% User interface +-export([run/0]). + +%% Exported to be used in spawn +-export([measure/4]). + +%% Internal constants +-define(MAX, 999999999999999). +-define(RANGE_MAX, 16#7ffffff). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Interface +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%%--------------------------------------------------------------------------- +%% run() -> _ +%% +%% Compiles and runs all benchmarks in the current directory, +%% and creates a report +%%--------------------------------------------------------------------------- +run() -> + run(compiler_options()). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Generic Benchmark functions +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%%--------------------------------------------------------------------------- +%% compiler_options() -> OptionsList +%% OptionsList = list() - See Erlang/OTP module compile +%%--------------------------------------------------------------------------- +compiler_options() -> + [report_errors, report_warnings]. + +%%--------------------------------------------------------------------------- +%% run(OptionsList) -> +%% OptionsList = list() - See Erlang/OTP module compile +%% +%% Help function to run/0. +%%--------------------------------------------------------------------------- +run(OptionsList) -> + Bms = compile_benchmarks(OptionsList), + run_benchmarks(Bms), + report(). + +%%--------------------------------------------------------------------------- +%% compile_benchmarks(OptionsList) -> [BmInfo| _] +%% OptionsList = list() - See Erlang/OTP module compile +%% BmInfo = {Module, Iterations, [BmFunctionName| _]} +%% Module = atom() +%% Iterations = integer() +%% BmFunctionName = atom() +%% +%% Compiles all benchmark modules in the current directory and +%% returns info about the benchmarks. +%%--------------------------------------------------------------------------- +compile_benchmarks(OptionsList) -> + {ok, FilesInCurrentDir} = file:list_dir("."), + ErlFiles = [ErlFile || ErlFile <- lists:sort(FilesInCurrentDir), + lists:suffix(".erl", ErlFile)], + lists:foldr(fun(File, BmInfoAcc) -> + case lists:suffix("_bm.erl", File) of + true -> + BmInfo = bm_compile(File, OptionsList), + [BmInfo | BmInfoAcc]; + false -> + just_compile(File, OptionsList), + BmInfoAcc + end + end, [], ErlFiles). + +%%--------------------------------------------------------------------------- +%% just_compile(FileName, OptionsList) -> ok +%% FileName = string() +%% OptionsList = list() - See Erlang/OTP module compile +%% +%% Compiles a support module. +%%--------------------------------------------------------------------------- +just_compile(FileName, OptionsList) -> + io:format("Compiling ~s...\n", [FileName]), % Progress info to user + case c:c(FileName, OptionsList) of + {ok, _Mod} -> + ok; + %% If compilation fails there is no point in trying to continue + error -> + Reason = + lists:flatten( + io_lib:format("Could not compile file ~s", [FileName])), + exit(self(), Reason) + end. +%%--------------------------------------------------------------------------- +%% bm_compile(FileName, OptionsList) -> BmInfo +%% FileName = string() +%% OptionsList = list() - See Erlang/OTP module compile +%% BmInfo = {Module, Iterations, [BmFunctionName| _]} +%% Iterations = integer() +%% Module = atom() +%% BmFunctionName = atom() +%% +%% Compiles the benchmark module implemented in <FileName> and returns +%% information about the benchmark tests. +%%--------------------------------------------------------------------------- +bm_compile(FileName, OptionsList) -> + io:format("Compiling ~s...\n", [FileName]), % Progress info to user + case c:c(FileName, OptionsList) of + {ok, Mod} -> + bm_cases(Mod); + %% If compilation fails there is no point in trying to continue + error -> + Reason = + lists:flatten( + io_lib:format("Could not compile file ~s", [FileName])), + exit(self(), Reason) + end. +%%--------------------------------------------------------------------------- +%% bm_cases(Module) -> {Module, Iter, [BmFunctionName |_]} +%% Module = atom() +%% Iter = integer() +%% BmFunctionName = atom() +%% +%% Fetches the number of iterations and the names of the benchmark +%% functions for the module <Module>. +%%--------------------------------------------------------------------------- +bm_cases(Module) -> + case catch Module:benchmarks() of + {Iter, BmList} when integer(Iter), list(BmList) -> + {Module, Iter, BmList}; + %% The benchmark is incorrect implemented there is no point in + %% trying to continue + Other -> + Reason = + lists:flatten( + io_lib:format("Incorrect return value: ~p " + "from ~p:benchmarks()", + [Other, Module])), + exit(self(), Reason) + end. +%%--------------------------------------------------------------------------- +%% run_benchmarks(Bms) -> +%% Bms = [{Module, Iter, [BmFunctionName |_]} | _] +%% Module = atom() +%% Iter = integer() +%% BmFunctionName = atom() +%% +%% Runs all the benchmark tests described in <Bms>. +%%--------------------------------------------------------------------------- +run_benchmarks(Bms) -> + Ver = erlang:system_info(version), + Machine = erlang:system_info(machine), + SysInfo = {Ver,Machine}, + + Res = [bms_run(Mod, Tests, Iter, SysInfo) || {Mod,Iter,Tests} <- Bms], + + %% Create an intermediate file that is later used to generate a bench + %% mark report. + Name = Ver ++ [$.|Machine] ++ ".bmres", + {ok, IntermediatFile} = file:open(Name, [write]), + + %% Create mark that identifies version of the benchmark modules + io:format(IntermediatFile, "~p.\n", [erlang:phash(Bms, ?RANGE_MAX)]), + + io:format(IntermediatFile, "~p.\n", [Res]), + file:close(IntermediatFile). + +%%--------------------------------------------------------------------------- +%% bms_run(Module, BmTests, Iter, Info) -> +%% Module = atom(), +%% BmTests = [BmFunctionName|_], +%% BmFunctionName = atom() +%% Iter = integer(), +%% SysInfo = {Ver, Machine} +%% Ver = string() +%% Machine = string() +%% +%% Runs all benchmark tests in module <Module>. +%%--------------------------------------------------------------------------- +bms_run(Module, BmTests, Iter, SysInfo) -> + io:format("Running ~s:", [Module]), % Progress info to user + Res = + {Module,{SysInfo,[{Bm, bm_run(Module, Bm, Iter)} || Bm <- BmTests]}}, + io:nl(), + Res. +%%--------------------------------------------------------------------------- +%% bm_run(Module, BmTest, Iter) -> Elapsed +%% Module = atom(), +%% BmTest = atom(), +%% Iter = integer() +%% Elapsed = integer() - elapsed time in milliseconds. +%% +%% Runs the benchmark Module:BmTest(Iter) +%%--------------------------------------------------------------------------- +bm_run(Module, BmTest, Iter) -> + io:format(" ~s", [BmTest]), % Progress info to user + spawn_link(?MODULE, measure, [self(), Module, BmTest, Iter]), + receive + {Elapsed, ok} -> + Elapsed; + {_Elapsed, Fault} -> + io:nl(), + Reason = + lists:flatten( + io_lib:format("~w", [Fault])), + exit(self(), Reason) + end. +%%--------------------------------------------------------------------------- +%% measure(Parent, Module, BmTest, Iter) -> _ +%% Parent = pid(), +%% Module = atom(), +%% BmTest = atom(), +%% Iter = integer() +%% +%% Measures the time it take to execute Module:Bm(Iter) +%% and send the result to <Parent>. +%%--------------------------------------------------------------------------- +measure(Parent, Module, BmTest, Iter) -> + statistics(runtime), + Res = (catch apply(Module, BmTest, [Iter])), + {_TotalRunTime, TimeSinceLastCall} = statistics(runtime), + Parent ! {TimeSinceLastCall, Res}. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Report functions +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%%--------------------------------------------------------------------------- +%% report() -> _ +%% +%% Creates a report of the bench marking test that appeals to a human. +%% Currently this means creating a html-file. (Other formats could be added) +%%--------------------------------------------------------------------------- +report() -> + {ok, AllFiles} = file:list_dir("."), + BmResultFiles = [File || File <- AllFiles, lists:suffix(".bmres", File)], + + Results = fetch_bmres_data(BmResultFiles), + create_report(Results). + +%%--------------------------------------------------------------------------- +%% fetch_bmres_data(BmResultFiles) -> Results +%% BmResultFiles = [FileName | _] +%% FileName = string() +%% Results = [[{Bm, Res} | _]] +%% Bm = atom() - Name of benchmark module +%% Res = [{VersionInfo, [{Test, Time} | _]}] +%% VersionInfo = {Ver, Machine} +%% Ver = string() +%% Machine = string() +%% Test = atom() +%% Time = integer() +%% +%% Reads result data from intermediate files +%%--------------------------------------------------------------------------- +fetch_bmres_data(BmResultFiles) -> + fetch_bmres_data(BmResultFiles, [], undefined). + +%%--------------------------------------------------------------------------- +%% fetch_bmres_data(BmResultFiles, AccResData, Check) -> Results +%% BmResultFiles = [FileName | _] +%% FileName = string() +%% AccResData = see Results fetch_bmres_data/1 +%% Check = integer() | undefined (first time) +%% +%% Help function to fetch_bmres_data/1 +%%--------------------------------------------------------------------------- +fetch_bmres_data([], AccResData, _Check) -> + AccResData; + +fetch_bmres_data([Name | BmResultFiles], AccResData, Check) -> + {DataList, NewCheck} = read_bmres_file(Name, Check), + fetch_bmres_data(BmResultFiles, [DataList| AccResData], NewCheck). + +%%--------------------------------------------------------------------------- +%% read_bmres_file(Name, Check) -> +%% Name = string() +%% Check = integer() | undefined +%% +%% Reads the data from the result files. Checks that all result +%% files where created with the same set of tests. +%%--------------------------------------------------------------------------- +read_bmres_file(Name, Check) -> + case file:consult(Name) of + {ok, [Check1, List]} when Check =:= undefined, integer(Check1) -> + {List, Check1}; + {ok, [Check, List]} when integer(Check) -> + {List, Check}; + {ok, [Check1, _List]} when integer(Check1) -> + Reason = + lists:flatten( + io_lib:format("Different test setup, remove old setup " + "result by removing *.bmres files and " + "try again", [])), + exit(self(), Reason); + {error, Reason} when atom(Reason) -> + exit(self(), Reason); + {error, Reason} -> + exit(self(), file:format(Reason)) + end. + +%%--------------------------------------------------------------------------- +%% create_report(Results) -> +%% Results = see Results fetch_bmres_data/1 +%% +%% Organizes <Result> so it will be right for create_html_report/1 +%% i.e. group results for the same benchmark test, run on different versions +%% of erlang. +%%--------------------------------------------------------------------------- +create_report(Results) -> + Dictionary = + lists:foldl(fun(BmResultList, Dict0) -> + lists:foldl(fun({Bm, VerResult}, Dict1) -> + dict:append(Bm, VerResult, + Dict1) + end,Dict0, BmResultList) + end, + dict:new(), Results), + + create_html_report(dict:to_list(Dictionary)). +%%--------------------------------------------------------------------------- +%% create_html_report(ResultList) -> _ +%% ResultList = [{Bm, Res} | _] +%% Bm = atom() - Name of benchmark module +%% Res = [{VersionInfo, [{Test, Time} | _]} | _] +%% VersionInfo = {Ver, Machine} +%% Ver = string() +%% Machine = string() +%% Test = atom() +%% Time = integer() +%% +%% Writes the result to an html-file +%%--------------------------------------------------------------------------- +create_html_report(ResultList) -> + + {ok, OutputFile} = file:open("index.html", [write]), + + %% Create the begining of the result html-file. + Head = Title = "Benchmark Results", + io:put_chars(OutputFile, "<html>\n"), + io:put_chars(OutputFile, "<head>\n"), + io:format(OutputFile, "<title>~s</title>\n", [Title]), + io:put_chars(OutputFile, "</head>\n"), + io:put_chars(OutputFile, "<body bgcolor=\"#FFFFFF\" text=\"#000000\"" ++ + " link=\"#0000FF\" vlink=\"#800080\" alink=\"#FF0000\">\n"), + io:format(OutputFile, "<h1>~s</h1>\n", [Head]), + + %% Add the result tables + lists:foreach(fun(Element) -> + create_html_table(OutputFile, Element) end, + ResultList), + + %% Put in the end-html tags + io:put_chars(OutputFile, "</body>\n"), + io:put_chars(OutputFile, "</html>\n"), + + file:close(OutputFile). + +%%--------------------------------------------------------------------------- +%% create_html_table(File, {Bm, Res}) -> _ +%% File = file() - html file to write data to. +%% Bm = atom() - Name of benchmark module +%% Res = [{VersionInfo, [{Test, Time} | _]}] +%% VersionInfo = {Ver, Machine} +%% Ver = string() +%% Machine = string() +%% Test = atom() +%% Time = integer() +%% +%% Creates a html table that displays the result of the benchmark <Bm>. +%%--------------------------------------------------------------------------- +create_html_table(File, {Bm, Res}) -> + + {MinTime, Order} = min_time_and_sort(Res), + + io:format(File, "<h2>~s</h2>\n" , [Bm]), + + %% Fun that calculates relative measure values and puts them in + %% a dictionary + RelativeMesureFun = fun({TestName, Time}, Dict1) -> + dict:append(TestName, Time/MinTime, Dict1) + end, + + %% For all erlang versions that the benchmark tests has been run, + %% calculate the relative measure values and put them in a dictionary. + ResultDict = + lists:foldl(fun({_VerInfo, Bms}, Dict0) -> + lists:foldl(RelativeMesureFun, Dict0, Bms) end, + dict:new(), Res), + + %% Create the table and its headings + io:put_chars(File, "<table border=0 cellpadding=1><tr>" + "<td bgcolor=\"#000000\">\n"), + io:put_chars(File, "<table cellpadding=3 border=0 cellspacing=1>\n"), + io:put_chars(File, "<tr bgcolor=white>"), + io:put_chars(File, "<td>Test</td>"), + Heads = table_headers(Res), + lists:foreach(fun({Ver,Machine}) -> io:format(File, "<td>~s<br>~s</td>", + [Ver,Machine]) end, Heads), + io:put_chars(File, "</tr>\n"), + + %% Create table rows + lists:foreach(fun(Name) -> + create_html_row(File, Name, ResultDict) + end, Order), + + %% Tabel end-tags + io:put_chars(File, "</table></td></tr></table>\n"), + + %% Create link to benchmark source code + io:format(File, "<p><a href=\"~s.erl\">Source for ~s.erl</a>\n", + [Bm,Bm]). + +%%--------------------------------------------------------------------------- +%% create_html_row(File, Name, Dict) -> _ +%% File = file() - html file to write data to. +%% Name = atom() - Name of benchmark test +%% Dict = dict() - Dictonary where the relative time measures for +%% the test can be found. +%% +%% Creates an actual html table-row. +%%--------------------------------------------------------------------------- +create_html_row(File, Name, Dict) -> + ReletiveTimes = dict:fetch(Name, Dict), + io:put_chars(File, "<tr bgcolor=white>\n"), + io:format(File, "<td>~s</td>", [Name]), + lists:foreach(fun(Time) -> + io:format(File, "<td>~-8.2f</td>", [Time]) end, + ReletiveTimes), + io:put_chars(File, "</tr>\n"). + +%%--------------------------------------------------------------------------- +%% min_time_and_sort(ResultList) -> {MinTime, Order} +%% ResultList = [{VersionInfo, [{Test, Time} | _]}] +%% MinTime = integer() - The execution time of the fastes test +%% Order = [BmFunctionName|_] - the order of the testcases in +%% increasing execution time. +%% BmFunctionName = atom() +%%--------------------------------------------------------------------------- +min_time_and_sort(ResultList) -> + + %% Use the results from the run on the highest version + %% of Erlang as norm. + {_, TestRes} = + lists:foldl(fun ({{Ver, _}, ResList}, + {CurrentVer, _}) when Ver > CurrentVer -> + {Ver, ResList}; + (_, VerAndRes) -> + VerAndRes + end, {"0", []}, ResultList), + + {lists:foldl(fun ({_, Time0}, Min1) when Time0 < Min1 -> + Time0; + (_, Min1) -> + Min1 + end, ?MAX, TestRes), + [Name || {Name, _} <- lists:keysort(2, TestRes)]}. + +%%--------------------------------------------------------------------------- +%% table_headers(VerResultList) -> SysInfo +%% VerResultList = [{{Ver, Machine},[{BmFunctionName, Time}]} | _] +%% Ver = string() +%% Machine = string() +%% BmFunctionName = atom() +%% Time = integer() +%% SysInfo = {Ver, Machine} +%%--------------------------------------------------------------------------- +table_headers(VerResultList) -> + [SysInfo || {SysInfo, _} <- VerResultList]. |