From 46aed30f8afbddd31f82e9b7a50a536a295f5eaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Wed, 20 Jan 2016 10:29:26 +0100 Subject: compile: Pre-load compiler modules when invoked from 'erlc' Slightly speed up 'erlc' by pre-loading the modules used by the compiler. Write a test case to ensure that the correct set of modules are loaded. --- lib/compiler/src/compile.erl | 44 ++++++++++++++ lib/compiler/test/compile_SUITE.erl | 114 +++++++++++++++++++++++++++++++++++- 2 files changed, 156 insertions(+), 2 deletions(-) diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 46917905de..332bc0bdf9 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -1681,6 +1681,7 @@ help(_) -> %% Compile entry point for erl_compile. compile(File0, _OutFile, Options) -> + pre_load(), File = shorten_filename(File0), case file(File, make_erl_options(Options)) of {ok,_Mod} -> ok; @@ -1745,3 +1746,46 @@ make_erl_options(Opts) -> end, Options ++ [report_errors, {cwd, Cwd}, {outdir, Outdir}| [{i, Dir} || Dir <- Includes]] ++ Specific. + +pre_load() -> + L = [beam_a, + beam_asm, + beam_block, + beam_bool, + beam_bs, + beam_bsm, + beam_clean, + beam_dead, + beam_dict, + beam_except, + beam_flatten, + beam_jump, + beam_opcodes, + beam_peep, + beam_receive, + beam_reorder, + beam_split, + beam_trim, + beam_type, + beam_utils, + beam_validator, + beam_z, + cerl, + cerl_clauses, + cerl_sets, + cerl_trees, + core_lib, + epp, + erl_bifs, + erl_expand_records, + erl_lint, + erl_parse, + erl_scan, + sys_core_dsetel, + sys_core_fold, + sys_pre_expand, + v3_codegen, + v3_core, + v3_kernel, + v3_life], + code:ensure_modules_loaded(L). diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index b2e9558cba..fc04c9a60a 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -22,6 +22,7 @@ %% Tests compile:file/1 and compile:file/2 with various options. -include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/erl_compile.hrl"). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, @@ -32,7 +33,7 @@ strict_record/1, missing_testheap/1, cover/1, env/1, core/1, asm/1, sys_pre_attributes/1, dialyzer/1, - warnings/1 + warnings/1, pre_load_check/1 ]). -export([init/3]). @@ -50,7 +51,7 @@ all() -> other_output, encrypted_abstr, strict_record, missing_testheap, cover, env, core, asm, - sys_pre_attributes, dialyzer, warnings]. + sys_pre_attributes, dialyzer, warnings, pre_load_check]. groups() -> []. @@ -881,6 +882,115 @@ do_warnings_2([], Next, F) -> do_warnings_1(Next, F). +%% Test that the compile:pre_load/0 function (used by 'erlc') +%% pre-loads the modules that are used by a typical compilation. + +pre_load_check(Config) -> + case test_server:is_cover() of + true -> + {skip,"Cover is running"}; + false -> + try + do_pre_load_check(Config) + after + dbg:stop_clear() + end + end. + +do_pre_load_check(Config) -> + DataDir = ?config(data_dir, Config), + Simple = filename:join(DataDir, "simple.erl"), + Big = filename:join(DataDir, "big.erl"), + {ok,_} = dbg:tracer(process, {fun pre_load_trace/2,[]}), + dbg:p(self(), call), + dbg:p(new, call), + {ok,_} = dbg:tpl({?MODULE,get_trace_data,0}, []), + {ok,_} = dbg:tp({code,ensure_modules_loaded,1}, []), + + %% Compile a simple module using the erl_compile interface + %% to find out the modules that are pre-loaded by + %% compile:pre_load/0. + Opts = #options{specific=[binary]}, + {ok,simple,_} = compile:compile(Simple, "", Opts), + [{code,ensure_modules_loaded,[PreLoaded0]}] = get_trace_data(), + PreLoaded1 = ordsets:from_list(PreLoaded0), + + %% Since 'compile' is the function doing the pre-loaded, + %% it is useless to include it in the list. + case ordsets:is_element(compile, PreLoaded1) of + true -> + io:put_chars("The 'compile' module should not be included " + "in the list of modules to be pre-loaded."), + ?t:fail(compile); + false -> + [] + end, + PreLoaded = ordsets:add_element(compile, PreLoaded1), + + %% Now unload all pre-loaded modules and all modules in + %% compiler application. Then compile a module to find + %% which modules that get loaded. + CompilerMods = compiler_modules(), + Unload = ordsets:union(ordsets:from_list(CompilerMods), PreLoaded), + _ = [begin + code:delete(M), + code:purge(M) + end || M <- Unload], + + {ok,_} = dbg:ctp({code,ensure_modules_loaded,1}), + {ok,_} = dbg:tp({code,ensure_loaded,1}, []), + {ok,big,_} = compile:file(Big, [binary]), + WasLoaded0 = get_trace_data(), + WasLoaded1 = [M || {code,ensure_loaded,[M]} <- WasLoaded0], + WasLoaded = ordsets:from_list(WasLoaded1), + + %% Check for modules that should have been pre-loaded. + case ordsets:subtract(WasLoaded, PreLoaded) of + [] -> + ok; + [_|_]=NotPreLoaded -> + io:format("The following modules were used " + "but not pre-loaded:\n~p\n", + [NotPreLoaded]), + ?t:fail({not_preload,NotPreLoaded}) + end, + + %% Check for modules that should not be pre-loaded. + case ordsets:subtract(PreLoaded, WasLoaded) of + [] -> + ok; + [_|_]=NotUsed -> + io:format("The following modules were pre-loaded" + " but not used:\n~p\n", + [NotUsed]), + ?t:fail({not_used,NotUsed}) + end, + + ok. + +get_trace_data() -> + %% Apparantely, doing a receive at the beginning of + %% a traced function can cause extra trace messages. + %% To avoid that, don't do the receive in this function. + do_get_trace_data(). + +do_get_trace_data() -> + receive + {trace_data,Data} -> Data + end. + +pre_load_trace({trace,Pid,call,{?MODULE,get_trace_data,[]}}, Acc) -> + Pid ! {trace_data,Acc}, + []; +pre_load_trace({trace,_,call,MFA}, Acc) -> + [MFA|Acc]. + +compiler_modules() -> + Wc = filename:join([code:lib_dir(compiler),"ebin","*.beam"]), + Ms = filelib:wildcard(Wc), + FN = filename, + [list_to_atom(FN:rootname(FN:basename(M), ".beam")) || M <- Ms]. + %%% %%% Utilities. %%% -- cgit v1.2.3 From c3c61822727939b8cad8f9a132602ac015820b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Thu, 10 Mar 2016 15:36:04 +0100 Subject: systools_make: Add commonly used modules to mandatory_modules/0 Add commonly used modules to the list of modules in mandatory_modules/0. Having those modules listed will potentially help the 'init' module to load them in parallel during start-up. --- lib/sasl/src/systools_make.erl | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index d207dc15bb..154374cd8c 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -1453,16 +1453,37 @@ behave([H|T]) -> behave([]) -> []. -%%______________________________________________________________________ -%% mandatory modules; this modules must be loaded before processes -%% can be started. These are a collection of modules from the kernel -%% and stdlib applications. -%% Nowadays, error_handler dynamically loads almost every module. -%% The error_handler self must still be there though. - mandatory_modules() -> - %% Sorted - [error_handler]. + [error_handler, %Truly mandatory. + + %% Modules that are almost always needed. Listing them here + %% helps the init module to load them faster. Modules not + %% listed here will be loaded by the error_handler module. + %% + %% Keep this list sorted. + application, + application_controller, + application_master, + code, + code_server, + erl_eval, + erl_lint, + erl_parse, + error_logger, + ets, + file, + filename, + file_server, + file_io_server, + gen, + gen_event, + gen_server, + heart, + kernel, + lists, + proc_lib, + supervisor + ]. %%______________________________________________________________________ %% This is the modules that are preloaded into the Erlang system. -- cgit v1.2.3 From c54c6243423b55602dd7c204f2351852132d4a98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Thu, 10 Mar 2016 15:42:33 +0100 Subject: init: Load modules in parallel using the new loader BIFs Use erlang:prepare_loading/1 and erlang:finish_loading/1 to load modules in parallel to potentially decrease start-up times. --- erts/preloaded/src/erl_prim_loader.erl | 11 +++- erts/preloaded/src/init.erl | 116 +++++++++++++++++++-------------- 2 files changed, 77 insertions(+), 50 deletions(-) diff --git a/erts/preloaded/src/erl_prim_loader.erl b/erts/preloaded/src/erl_prim_loader.erl index cbcced5512..641abae7b1 100644 --- a/erts/preloaded/src/erl_prim_loader.erl +++ b/erts/preloaded/src/erl_prim_loader.erl @@ -56,7 +56,7 @@ -export([purge_archive_cache/0]). %% Used by init and the code server. --export([get_modules/3]). +-export([get_modules/2,get_modules/3]). -include_lib("kernel/include/file.hrl"). @@ -239,6 +239,13 @@ set_primary_archive(File, ArchiveBin, FileInfo, ParserFun) purge_archive_cache() -> request(purge_archive_cache). +-spec get_modules([module()], + fun((atom(), string(), binary()) -> + {'ok',any()} | {'error',any()})) -> + {'ok',{[any()],[any()]}}. + +get_modules(Modules, Fun) -> + request({get_modules,{Modules,Fun}}). -spec get_modules([module()], fun((atom(), string(), binary()) -> @@ -338,6 +345,8 @@ handle_request(Req, Paths, St0) -> {{ok,Paths},St0}; {get_file,File} -> handle_get_file(St0, Paths, File); + {get_modules,{Modules,Fun}} -> + handle_get_modules(St0, Modules, Fun, Paths); {get_modules,{Modules,Fun,ModPaths}} -> handle_get_modules(St0, Modules, Fun, ModPaths); {list_dir,Dir} -> diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 5de3732c20..915f1183d6 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -297,9 +297,9 @@ crash(String, List) -> -spec boot_loop(pid(), state()) -> no_return(). boot_loop(BootPid, State) -> receive - {BootPid,loaded,ModLoaded} -> - Loaded = State#state.loaded, - boot_loop(BootPid,State#state{loaded = [ModLoaded|Loaded]}); + {BootPid,loaded,NewlyLoaded} -> + Loaded = NewlyLoaded ++ State#state.loaded, + boot_loop(BootPid, State#state{loaded = Loaded}); {BootPid,started,KernelPid} -> boot_loop(BootPid, new_kernelpid(KernelPid, BootPid, State)); {BootPid,progress,started} -> @@ -338,12 +338,25 @@ boot_loop(BootPid, State) -> end. ensure_loaded(Module, Loaded) -> + case erlang:module_loaded(Module) of + true -> + {{module, Module}, Loaded}; + false -> + do_ensure_loaded(Module, Loaded) + end. + +do_ensure_loaded(Module, Loaded) -> File = atom_to_list(Module) ++ objfile_extension(), - case catch load_mod(Module,File) of - {ok, FullName} -> - {{module, Module}, [{Module, FullName}|Loaded]}; - Res -> - {Res, Loaded} + case erl_prim_loader:get_file(File) of + {ok,BinCode,FullName} -> + case do_load_module(Module, BinCode) of + ok -> + {{module, Module}, [{Module, FullName}|Loaded]}; + error -> + {error, [{Module, FullName}|Loaded]} + end; + Error -> + {Error, Loaded} end. %% Tell subscribed processes the system has started. @@ -842,13 +855,6 @@ eval_script([{kernel_load_completed}|T], #es{load_mode=Mode}=Es0) -> _ -> Es0#es{prim_load=false} end, eval_script(T, Es); -eval_script([{primLoad,[Mod]}|T], #es{prim_load=true}=Es) -> - %% Common special case (loading of error_handler). Nothing - %% to gain by parallel loading. - File = atom_to_list(Mod) ++ objfile_extension(), - {ok,Full} = load_mod(Mod, File), - init ! {self(),loaded,{Mod,Full}}, % Tell init about loaded module - eval_script(T, Es); eval_script([{primLoad,Mods}|T], #es{init=Init,prim_load=PrimLoad}=Es) when is_list(Mods) -> case PrimLoad of @@ -873,14 +879,44 @@ eval_script([], #es{}) -> eval_script(What, #es{}) -> exit({'unexpected command in bootfile',What}). -load_modules([Mod|Mods], Init) -> - File = atom_to_list(Mod) ++ objfile_extension(), - {ok,Full} = load_mod(Mod,File), - Init ! {self(),loaded,{Mod,Full}}, %Tell init about loaded module - load_modules(Mods, Init); -load_modules([], _) -> +load_modules(Mods0, Init) -> + Mods = [M || M <- Mods0, not erlang:module_loaded(M)], + F = prepare_loading_fun(), + case erl_prim_loader:get_modules(Mods, F) of + {ok,{Prep0,[]}} -> + Prep = [Code || {_,{prepared,Code,_}} <- Prep0], + ok = erlang:finish_loading(Prep), + Loaded = [{Mod,Full} || {Mod,{_,_,Full}} <- Prep0], + Init ! {self(),loaded,Loaded}, + Beams = [{M,Beam,Full} || {M,{on_load,Beam,Full}} <- Prep0], + load_rest(Beams, Init); + {ok,{_,[_|_]=Errors}} -> + Ms = [M || {M,_} <- Errors], + exit({load_failed,Ms}) + end. + +load_rest([{Mod,Beam,Full}|T], Init) -> + do_load_module(Mod, Beam), + Init ! {self(),loaded,[{Mod,Full}]}, + load_rest(T, Init); +load_rest([], _) -> ok. +prepare_loading_fun() -> + fun(Mod, FullName, Beam) -> + case erlang:prepare_loading(Mod, Beam) of + Prepared when is_binary(Prepared) -> + case erlang:has_prepared_code_on_load(Prepared) of + true -> + {ok,{on_load,Beam,FullName}}; + false -> + {ok,{prepared,Prepared,FullName}} + end; + {error,_}=Error -> + Error + end + end. + make_path(Pa, Pz, Path, Vars) -> append([Pa,append([fix_path(Path,Vars),Pz])]). @@ -1033,35 +1069,17 @@ start_it([_|_]=MFA) -> [M,F|Args] -> M:F(Args) % Args is a list end. -%% -%% Fetch a module and load it into the system. -%% -load_mod(Mod, File) -> - case erlang:module_loaded(Mod) of - false -> - case erl_prim_loader:get_file(File) of - {ok,BinCode,FullName} -> - load_mod_code(Mod, BinCode, FullName); - _ -> - exit({'cannot load',Mod,get_file}) - end; - _ -> % Already loaded. - {ok,File} - end. +%% Load a module. -load_mod_code(Mod, BinCode, FullName) -> - case erlang:module_loaded(Mod) of - false -> - case erlang:load_module(Mod, BinCode) of - {module,Mod} -> {ok,FullName}; - {error,on_load} -> - ?ON_LOAD_HANDLER ! {loaded,Mod}, - {ok,FullName}; - Other -> - exit({'cannot load',Mod,Other}) - end; - _ -> % Already loaded. - {ok,FullName} +do_load_module(Mod, BinCode) -> + case erlang:load_module(Mod, BinCode) of + {module,Mod} -> + ok; + {error,on_load} -> + ?ON_LOAD_HANDLER ! {loaded,Mod}, + ok; + _ -> + error end. %% -------------------------------------------------------- -- cgit v1.2.3 From 3c3c984aa9443a1ac1464473fd50e721e4d43b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Thu, 17 Mar 2016 10:25:13 +0100 Subject: Update preloaded modules --- erts/preloaded/ebin/erl_prim_loader.beam | Bin 55528 -> 55792 bytes erts/preloaded/ebin/init.beam | Bin 44608 -> 46200 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam index 6d777fa811..f224178c4f 100644 Binary files a/erts/preloaded/ebin/erl_prim_loader.beam and b/erts/preloaded/ebin/erl_prim_loader.beam differ diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam index 9d2dd38f3a..7d73ca2234 100644 Binary files a/erts/preloaded/ebin/init.beam and b/erts/preloaded/ebin/init.beam differ -- cgit v1.2.3