From 65c4401c1cd1094da5d8f35137532a22054a8517 Mon Sep 17 00:00:00 2001 From: Richard Carlsson Date: Mon, 21 Nov 2016 15:31:16 +0100 Subject: Remove remnants of module package support --- lib/kernel/src/code.erl | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl index 622b27080c..1b149bbb4e 100644 --- a/lib/kernel/src/code.erl +++ b/lib/kernel/src/code.erl @@ -719,37 +719,28 @@ start_get_mode() -> which(Module) when is_atom(Module) -> case is_loaded(Module) of false -> - which2(Module); + File = atom_to_list(Module) ++ objfile_extension(), + which(File, get_path()); {file, File} -> File end. -which2(Module) -> - Base = atom_to_list(Module), - File = filename:basename(Base) ++ objfile_extension(), - Path = get_path(), - which(File, filename:dirname(Base), Path). - --spec which(file:filename(), file:filename(), [file:filename()]) -> +-spec which(file:filename(), [file:filename()]) -> 'non_existing' | file:filename(). -which(_, _, []) -> +which(_, []) -> non_existing; -which(File, Base, [Directory|Tail]) -> - Path = if - Base =:= "." -> Directory; - true -> filename:join(Directory, Base) - end, +which(File, [Path|Tail]) -> case erl_prim_loader:list_dir(Path) of {ok,Files} -> case lists:member(File,Files) of true -> filename:append(Path, File); false -> - which(File, Base, Tail) + which(File, Tail) end; _Error -> - which(File, Base, Tail) + which(File, Tail) end. %% Search the code path for a specific file. Try to locate @@ -760,13 +751,13 @@ which(File, Base, [Directory|Tail]) -> Absname :: file:filename(). where_is_file(File) when is_list(File) -> Path = get_path(), - which(File, ".", Path). + which(File, Path). -spec where_is_file(Path :: file:filename(), Filename :: file:filename()) -> file:filename() | 'non_existing'. where_is_file(Path, File) when is_list(Path), is_list(File) -> - which(File, ".", Path). + which(File, Path). -spec set_primary_archive(ArchiveFile :: file:filename(), ArchiveBin :: binary(), -- cgit v1.2.3 From c7e5cf8e259956120c07206c4e7df235b129cb56 Mon Sep 17 00:00:00 2001 From: Richard Carlsson Date: Tue, 22 Nov 2016 15:17:04 +0100 Subject: Restructure code:which() and where_is_file() --- lib/kernel/src/code.erl | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl index 1b149bbb4e..bb19d6716d 100644 --- a/lib/kernel/src/code.erl +++ b/lib/kernel/src/code.erl @@ -719,29 +719,14 @@ start_get_mode() -> which(Module) when is_atom(Module) -> case is_loaded(Module) of false -> - File = atom_to_list(Module) ++ objfile_extension(), - which(File, get_path()); + which(Module, get_path()); {file, File} -> File end. --spec which(file:filename(), [file:filename()]) -> - 'non_existing' | file:filename(). - -which(_, []) -> - non_existing; -which(File, [Path|Tail]) -> - case erl_prim_loader:list_dir(Path) of - {ok,Files} -> - case lists:member(File,Files) of - true -> - filename:append(Path, File); - false -> - which(File, Tail) - end; - _Error -> - which(File, Tail) - end. +which(Module, Path) when is_atom(Module) -> + File = atom_to_list(Module) ++ objfile_extension(), + where_is_file(Path, File). %% Search the code path for a specific file. Try to locate %% it in the code path cache if possible. @@ -751,13 +736,28 @@ which(File, [Path|Tail]) -> Absname :: file:filename(). where_is_file(File) when is_list(File) -> Path = get_path(), - which(File, Path). + where_is_file(Path, File). -spec where_is_file(Path :: file:filename(), Filename :: file:filename()) -> file:filename() | 'non_existing'. -where_is_file(Path, File) when is_list(Path), is_list(File) -> - which(File, Path). +where_is_file([], _) -> + non_existing; +where_is_file([Path|Tail], File) -> + case erl_prim_loader:list_dir(Path) of + {ok,Files} -> + where_is_file(Tail, File, Path, Files); + _Error -> + where_is_file(Tail, File) + end. + +where_is_file(Tail, File, Path, Files) -> + case lists:member(File, Files) of + true -> + filename:append(Path, File); + false -> + where_is_file(Tail, File) + end. -spec set_primary_archive(ArchiveFile :: file:filename(), ArchiveBin :: binary(), -- cgit v1.2.3 From a834d721e5047c5c43a8ede249fdf6711234ff1b Mon Sep 17 00:00:00 2001 From: Richard Carlsson Date: Tue, 22 Nov 2016 15:33:14 +0100 Subject: Handle prefetched paths --- lib/kernel/src/code.erl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl index bb19d6716d..0ad0676f98 100644 --- a/lib/kernel/src/code.erl +++ b/lib/kernel/src/code.erl @@ -738,11 +738,16 @@ where_is_file(File) when is_list(File) -> Path = get_path(), where_is_file(Path, File). --spec where_is_file(Path :: file:filename(), Filename :: file:filename()) -> - file:filename() | 'non_existing'. +%% To avoid unnecessary work when looking at many modules, this also +%% accepts pairs of directories and pre-fetched contents in the path +-spec where_is_file(Path :: [Dir|{Dir,Files}], Filename :: file:filename()) -> + 'non_existing' | file:filename() when + Dir :: file:filename(), Files :: [file:filename()]. where_is_file([], _) -> non_existing; +where_is_file([{Path, Files}|Tail], File) -> + where_is_file(Tail, File, Path, Files); where_is_file([Path|Tail], File) -> case erl_prim_loader:list_dir(Path) of {ok,Files} -> -- cgit v1.2.3 From 86fa667f9731c790d6575f31efa156c02cb7984b Mon Sep 17 00:00:00 2001 From: Richard Carlsson Date: Thu, 19 May 2016 20:29:45 +0200 Subject: Add code:module_status/1 and modified_modules/0 These functions use the MD5 beam/native checksum to determine whether the code for a module has changed on disk and is a candidate for loading. --- lib/kernel/doc/src/code.xml | 42 ++++++++++ lib/kernel/src/code.erl | 98 +++++++++++++++++++++- lib/kernel/test/code_SUITE.erl | 180 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 319 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/kernel/doc/src/code.xml b/lib/kernel/doc/src/code.xml index 3143cdc825..f881fd76fd 100644 --- a/lib/kernel/doc/src/code.xml +++ b/lib/kernel/doc/src/code.xml @@ -898,6 +898,48 @@ rpc:call(Node, code, load_binary, [Module, Filename, Binary]), identical names and writes a report to stdout.

+ + + Return the status of the module in relation to object file on disk. + +

Returns:

+ + not_loaded +

If Module is not currently loaded.

 + loaded +

If Module is loaded and the object file + exists and contains the same code.

 + removed +

If Module is loaded but no + corresponding object file can be found in the code path.

 + modified +

If Module is loaded but the object file + contains code with a different MD5 checksum.

 + +

Preloaded modules are always reported as loaded, without + inspecting the contents on disk. Cover compiled modules will always + be reported as modified if an object file exists, or as + removed otherwise. Modules whose load path is an empty string + (which is the convention for auto-generated code) will only be + reported as loaded or not_loaded.

+

For modules that have native code loaded (see + is_module_native/1), + the MD5 sum of the native code in the object file is used for the + comparison, if it exists; the Beam code in the file is ignored. + Reversely, for modules that do not currently have native code + loaded, any native code in the file will be ignored.

+

See also modified_modules/0.

+ + + + + Return a list of all modules modified on disk. + +

Returns the list of all currently loaded modules for which + module_status/1 + returns modified. See also all_loaded/0.

+ + Test if a module has native code. diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl index 0ad0676f98..5a7ca493cc 100644 --- a/lib/kernel/src/code.erl +++ b/lib/kernel/src/code.erl @@ -70,7 +70,9 @@ where_is_file/2, set_primary_archive/4, clash/0, - get_mode/0]). + module_status/1, + modified_modules/0, + get_mode/0]). -deprecated({rehash,0,next_major_release}). @@ -895,3 +897,97 @@ load_all_native_1([{Mod,BeamFilename}|T], ChunkTag) -> load_all_native_1(T, ChunkTag); load_all_native_1([], _) -> ok. + +%% Returns the status of the module in relation to object file on disk. +-spec module_status(Module :: module()) -> not_loaded | loaded | modified | removed. +module_status(Module) -> + module_status(Module, code:get_path()). + +%% Note that we don't want to go via which/1, since it doesn't look at the +%% disk contents at all if the module is already loaded. +module_status(Module, PathFiles) -> + case code:is_loaded(Module) of + false -> not_loaded; + {file, preloaded} -> loaded; + {file, cover_compiled} -> + %% cover compilation loads directly to memory and does not + %% create a beam file, so report 'modified' if a file exists + case which(Module, PathFiles) of + non_existing -> removed; + _File -> modified + end; + {file, []} -> loaded; % no beam file - generated code + {file, OldFile} when is_list(OldFile) -> + %% we don't care whether or not the file is in the same location + %% as when last loaded, as long as it can be found in the path + case which(Module, PathFiles) of + non_existing -> removed; + Path -> + case module_changed_on_disk(Module, Path) of + true -> modified; + false -> loaded + end + end + end. + +%% Detects actual code changes only, e.g. to decide whether a module should +%% be reloaded; does not care about file timestamps or compilation time +module_changed_on_disk(Module, Path) -> + MD5 = erlang:get_module_info(Module, md5), + case erlang:system_info(hipe_architecture) of + undefined -> + %% straightforward, since native is not supported + MD5 =/= beam_file_md5(Path); + Architecture -> + case code:is_module_native(Module) of + true -> + %% MD5 is for native code, so we check only the native + %% code on disk, ignoring the beam code + MD5 =/= beam_file_native_md5(Path, Architecture); + _ -> + %% MD5 is for beam code, so check only the beam code on + %% disk, even if the file contains native code as well + MD5 =/= beam_file_md5(Path) + end + end. + +beam_file_md5(Path) -> + case beam_lib:md5(Path) of + {ok,{_Mod,MD5}} -> MD5; + _ -> undefined + end. + +beam_file_native_md5(Path, Architecture) -> + try + get_beam_chunk(Path, hipe_unified_loader:chunk_name(Architecture)) + of + NativeCode when is_binary(NativeCode) -> + erlang:md5(NativeCode) + catch + _:_ -> undefined + end. + +get_beam_chunk(Path, Chunk) -> + {ok, {_, [{_, Bin}]}} = beam_lib:chunks(Path, [Chunk]), + Bin. + +%% Returns a list of all modules modified on disk. +-spec modified_modules() -> [module()]. +modified_modules() -> + PathFiles = path_files(), + [M || {M, _} <- code:all_loaded(), + module_status(M, PathFiles) =:= modified]. + +%% prefetch the directory contents of code path directories +path_files() -> + path_files(code:get_path()). + +path_files([]) -> + []; +path_files([Path|Tail]) -> + case erl_prim_loader:list_dir(Path) of + {ok, Files} -> + [{Path,Files} | path_files(Tail)]; + _Error -> + path_files(Tail) + end. diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 29b3f7caaf..4914ce9e4c 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -38,6 +38,7 @@ on_load_purge/1, on_load_self_call/1, on_load_pending/1, on_load_deleted/1, big_boot_embedded/1, + module_status/1, native_early_modules/1, get_mode/1, normalized_paths/1]). @@ -68,6 +69,7 @@ all() -> on_load_binary, on_load_embedded, on_load_errors, on_load_update, on_load_purge, on_load_self_call, on_load_pending, on_load_deleted, + module_status, big_boot_embedded, native_early_modules, get_mode, normalized_paths]. groups() -> @@ -93,6 +95,11 @@ init_per_suite(Config) -> end_per_suite(Config) -> Config. +-define(TESTMOD, test_dummy). +-define(TESTMODSTR, "test_dummy"). +-define(TESTMODSRC, ?TESTMODSTR ".erl"). +-define(TESTMODOBJ, ?TESTMODSTR ".beam"). + init_per_testcase(big_boot_embedded, Config) -> case catch crypto:start() of ok -> @@ -104,6 +111,13 @@ init_per_testcase(_Func, Config) -> P = code:get_path(), [{code_path, P}|Config]. +end_per_testcase(module_status, Config) -> + code:purge(?TESTMOD), + code:delete(?TESTMOD), + code:purge(?TESTMOD), + file:delete(?TESTMODOBJ), + file:delete(?TESTMODSRC), + end_per_testcase(Config); end_per_testcase(TC, Config) when TC == mult_lib_roots; TC == big_boot_embedded -> {ok, HostName} = inet:gethostname(), @@ -1757,6 +1771,172 @@ do_normalized_paths([M|Ms]) -> do_normalized_paths([]) -> ok. +%% Test that module_status/1 behaves as expected +module_status(_Config) -> + %% basics + not_loaded = code:module_status(fubar), % nonexisting + {file, preloaded} = code:is_loaded(erlang), + loaded = code:module_status(erlang), % preloaded + loaded = code:module_status(?MODULE), % normal known loaded + + non_existing = code:which(?TESTMOD), % verify dummy name not in path + code:purge(?TESTMOD), % ensure no previous version in memory + code:delete(?TESTMOD), + code:purge(?TESTMOD), + + %% generated code is detected as such + {ok,?TESTMOD,Bin} = compile:forms(dummy_ast(), []), + {module,?TESTMOD} = code:load_binary(?TESTMOD,"",Bin), % no source file + ok = ?TESTMOD:f(), + "" = code:which(?TESTMOD), % verify empty string for source file + loaded = code:module_status(?TESTMOD), + + %% deleting generated code + true = code:delete(?TESTMOD), + non_existing = code:which(?TESTMOD), % verify still not in path + not_loaded = code:module_status(?TESTMOD), + + %% beam file exists but not loaded + make_source_file(<<"0">>), + compile_beam(0), + true = (non_existing =/= code:which(?TESTMOD)), % verify in path + not_loaded = code:module_status(?TESTMOD), + + %% loading code from disk makes it loaded + load_code(), + loaded = code:module_status(?TESTMOD), % loaded + + %% cover compiling a module + {ok,?TESTMOD} = cover:compile(?TESTMOD), + {file, cover_compiled} = code:is_loaded(?TESTMOD), % verify cover compiled + modified = code:module_status(?TESTMOD), % loaded cover code but file exists + remove_code(), + removed = code:module_status(?TESTMOD), % removed + compile_beam(0), + modified = code:module_status(?TESTMOD), % recreated + load_code(), + loaded = code:module_status(?TESTMOD), % loading removes cover status + code:purge(?TESTMOD), + true = code:delete(?TESTMOD), + not_loaded = code:module_status(?TESTMOD), % deleted + + %% recompilation ignores timestamps, only md5 matters + load_code(), + compile_beam(1100), + loaded = code:module_status(?TESTMOD), + + %% modifying module detects different md5 + make_source_file(<<"1">>), + compile_beam(0), + modified = code:module_status(?TESTMOD), + + %% loading the modified code from disk makes it loaded + load_code(), + loaded = code:module_status(?TESTMOD), + + %% removing and recreating a module with same md5 + remove_code(), + removed = code:module_status(?TESTMOD), + compile_beam(0), + loaded = code:module_status(?TESTMOD), + + case erlang:system_info(hipe_architecture) of + undefined -> + %% no native support + ok; + _ -> + %% native chunk is ignored if beam code is already loaded + load_code(), + loaded = code:module_status(?TESTMOD), + false = has_native(?TESTMOD), + compile_native(0), + BeamMD5 = erlang:get_module_info(?TESTMOD, md5), + {ok,{?TESTMOD,BeamMD5}} = beam_lib:md5(?TESTMODOBJ), % beam md5 unchanged + loaded = code:module_status(?TESTMOD), + + %% native code reported as loaded, though different md5 from beam + load_code(), + true = has_native(?TESTMOD), + NativeMD5 = erlang:get_module_info(?TESTMOD, md5), + true = (BeamMD5 =/= NativeMD5), + loaded = code:module_status(?TESTMOD), + + %% recompilation ignores timestamps, only md5 matters + compile_native(1100), % later timestamp + loaded = code:module_status(?TESTMOD), + + %% modifying native module detects different md5 + make_source_file(<<"2">>), + compile_native(0), + modified = code:module_status(?TESTMOD), + + %% loading the modified native code from disk makes it loaded + load_code(), + true = has_native(?TESTMOD), + NativeMD5_2 = erlang:get_module_info(?TESTMOD, md5), + true = (NativeMD5 =/= NativeMD5_2), % verify native md5 changed + {ok,{?TESTMOD,BeamMD5_2}} = beam_lib:md5(?TESTMODOBJ), + true = (BeamMD5_2 =/= NativeMD5_2), % verify md5 differs from beam + loaded = code:module_status(?TESTMOD), + + %% removing and recreating a native module with same md5 + remove_code(), + removed = code:module_status(?TESTMOD), + compile_native(0), + loaded = code:module_status(?TESTMOD), + + %% purging/deleting native module + code:purge(?TESTMOD), + true = code:delete(?TESTMOD), + not_loaded = code:module_status(?TESTMOD) + end, + ok. + +compile_beam(Sleep) -> + compile(Sleep, []). + +compile_native(Sleep) -> + compile(Sleep, [native]). + +compile(Sleep, Opts) -> + timer:sleep(Sleep), % increment compilation timestamp + {ok,?TESTMOD} = compile:file(?TESTMODSRC, Opts). + +load_code() -> + code:purge(?TESTMOD), + {module,?TESTMOD} = code:load_file(?TESTMOD). + +remove_code() -> + ok = file:delete(?TESTMODOBJ). + +has_native(Module) -> + case erlang:get_module_info(Module, native_addresses) of + [] -> false; + [_|_] -> true + end. + +make_source_file(Body) -> + ok = file:write_file(?TESTMODSRC, dummy_source(Body)). + +dummy_source(Body) -> + [<<"-module(" ?TESTMODSTR ").\n" + "-export([f/0]).\n" + "f() -> ">>, Body, <<".\n">>]. + +dummy_ast() -> + dummy_ast(?TESTMODSTR). + +dummy_ast(Mod) when is_atom(Mod) -> + dummy_ast(atom_to_list(Mod)); +dummy_ast(ModStr) -> + [scan_form("-module(" ++ ModStr ++ ")."), + scan_form("-export([f/0])."), + scan_form("f() -> ok.")]. + +scan_form(String) -> + {ok,Ts,_} = erl_scan:string(String), + {ok,F} = erl_parse:parse_form(Ts), + F. %%----------------------------------------------------------------- %% error_logger handler. -- cgit v1.2.3 From a4d665795bb744f516e6fe36b97c38e123f8b706 Mon Sep 17 00:00:00 2001 From: Richard Carlsson Date: Mon, 21 Nov 2016 21:17:21 +0100 Subject: Add shell mm() and lm() functions --- lib/stdlib/doc/src/c.xml | 18 ++++++++++++++++++ lib/stdlib/src/c.erl | 14 +++++++++++++- lib/stdlib/src/shell_default.erl | 4 +++- 3 files changed, 34 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/stdlib/doc/src/c.xml b/lib/stdlib/doc/src/c.xml index 92ab59c6b0..55a77d1bc5 100644 --- a/lib/stdlib/doc/src/c.xml +++ b/lib/stdlib/doc/src/c.xml @@ -147,6 +147,15 @@ compile:file(File, Options ++ [report_errors, report_w + + + Loads all modified modules. + +

Reloads all currently loaded modules that have changed on disk (see mm()). + Returns the list of results from calling l(M) for each such M.

+ + + List files in the current directory. @@ -181,6 +190,15 @@ compile:file(File, Options ++ [report_errors, report_w + + + Lists all modified modules. + +

Lists all modified modules. Shorthand for + code:modified_modules/0.

+ + + Memory allocation information. diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl index ad4915eabe..d36630214c 100644 --- a/lib/stdlib/src/c.erl +++ b/lib/stdlib/src/c.erl @@ -26,7 +26,7 @@ -export([help/0,lc/1,c/1,c/2,nc/1,nc/2, nl/1,l/1,i/0,i/1,ni/0, y/1, y/2, lc_batch/0, lc_batch/1, - i/3,pid/3,m/0,m/1, + i/3,pid/3,m/0,m/1,mm/0,lm/0, bt/1, q/0, erlangrc/0,erlangrc/1,bi/1, flush/0, regs/0, uptime/0, nregs/0,pwd/0,ls/0,ls/1,cd/1,memory/1,memory/0, xm/1]). @@ -52,11 +52,13 @@ help() -> "ni() -- information about the networked system\n" "i(X,Y,Z) -- information about pid \n" "l(Module) -- load or reload module\n" + "lm() -- load all modified modules\n" "lc([File]) -- compile a list of Erlang modules\n" "ls() -- list files in the current directory\n" "ls(Dir) -- list files in directory \n" "m() -- which modules are loaded\n" "m(Mod) -- information about module \n" + "mm() -- list all modified modules\n" "memory() -- memory allocation information\n" "memory(T) -- memory allocation information of type \n" "nc(File) -- compile and load code in on all nodes\n" @@ -459,6 +461,16 @@ m() -> mformat(A1, A2) -> format("~-20s ~ts\n", [A1,A2]). +-spec mm() -> [module()]. + +mm() -> + code:modified_modules(). + +-spec lm() -> [code:load_ret()]. + +lm() -> + [l(M) || M <- mm()]. + %% erlangrc(Home) %% Try to run a ".erlang" file, first in the current directory %% else in home directory. diff --git a/lib/stdlib/src/shell_default.erl b/lib/stdlib/src/shell_default.erl index 6947cf181b..cd63ab28b5 100644 --- a/lib/stdlib/src/shell_default.erl +++ b/lib/stdlib/src/shell_default.erl @@ -23,7 +23,7 @@ -module(shell_default). --export([help/0,lc/1,c/1,c/2,nc/1,nl/1,l/1,i/0,pid/3,i/3,m/0,m/1, +-export([help/0,lc/1,c/1,c/2,nc/1,nl/1,l/1,i/0,pid/3,i/3,m/0,m/1,lm/0,mm/0, memory/0,memory/1,uptime/0, erlangrc/1,bi/1, regs/0, flush/0,pwd/0,ls/0,ls/1,cd/1, y/1, y/2, @@ -83,6 +83,8 @@ ls() -> c:ls(). ls(S) -> c:ls(S). m() -> c:m(). m(Mod) -> c:m(Mod). +lm() -> c:lm(). +mm() -> c:mm(). memory() -> c:memory(). memory(Type) -> c:memory(Type). nc(X) -> c:nc(X). -- cgit v1.2.3