From f610a7efe07fe47bec1bbc86d82545b094514ad9 Mon Sep 17 00:00:00 2001 From: Konrad Kaplita Date: Tue, 3 Dec 2013 14:40:11 +0100 Subject: Accelerate `rlx_prv_discover` provider Introduce option `enable_shallow_app_discovery` which is disabled by default. When enabled searching for `*.app` files is done by using `filelib:wildcard/1` and doesn't recursively traverse all directories under `libs_dir` root dirs. Instead its assumed that each directory on `libs_dir` list contains a flat list of application dirs or is an app itself. --- src/rlx_app_discovery.erl | 67 +++++++++++++++++++++++++++++++++++++++++---- test/rlx_discover_SUITE.erl | 41 ++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 10 deletions(-) diff --git a/src/rlx_app_discovery.erl b/src/rlx_app_discovery.erl index aaa6e86..4a5620b 100644 --- a/src/rlx_app_discovery.erl +++ b/src/rlx_app_discovery.erl @@ -52,8 +52,56 @@ format_error(ErrorDetails) %%%=================================================================== %%% Internal Functions %%%=================================================================== + +-spec app_files(list(binary())) -> list(binary()). +app_files(LibDirs) -> + lists:foldl(fun(LibDir, Acc) -> + Files = app_files_paths(LibDir), + BinFiles = lists:map(fun(F) -> + list_to_binary(F) + end, Files), + Acc ++ BinFiles + end, [], LibDirs). + +-spec app_files_paths(binary()) -> list(string()). +app_files_paths(LibDir) -> + %% Search for Erlang apps in the lib dir itself + Path1 = filename:join([binary_to_list(LibDir), + "*.app"]), + %% Search for Erlang apps in subdirs of lib dir + Path2 = filename:join([binary_to_list(LibDir), + "*", + "ebin", + "*.app"]), + lists:foldl(fun(Path, Acc) -> + Files = filelib:wildcard(Path), + Files ++ Acc + end, [], [Path1, Path2]). + +-spec get_app_metadata(rlx_state:t(), list(binary())) -> list({ok, rlx_app_info:t()}). +get_app_metadata(State, LibDirs) -> + lists:foldl(fun(AppFile, Acc) -> + case is_valid_otp_app(AppFile) of + {ok, _} = AppMeta -> + [AppMeta|Acc]; + {warning, W} -> + ec_cmd_log:warn(rlx_state:log(State), format_detail(W)), + Acc; + {error, E} -> + ec_cmd_log:error(rlx_state:log(State), format_detail(E)), + Acc; + _ -> + Acc + end + end, [], app_files(LibDirs)). + resolve_app_metadata(State, LibDirs) -> - AppMeta0 = lists:flatten(rlx_dscv_util:do(fun discover_dir/2, LibDirs)), + AppMeta0 = case rlx_state:get(State, enable_shallow_app_discovery, false) of + true -> + get_app_metadata(State, LibDirs); + false -> + lists:flatten(rlx_dscv_util:do(fun discover_dir/2, LibDirs)) + end, case [case Err of {error, Ret} -> Ret @@ -139,9 +187,10 @@ discover_dir(_File, directory) -> discover_dir(File, file) -> is_valid_otp_app(File). --spec is_valid_otp_app(file:name()) -> {ok, rlx_app_info:t()} | {error, Reason::term()} | +-spec is_valid_otp_app(file:name()) -> {ok, rlx_app_info:t()} | + {warning, Reason::term()} | + {error, Reason::term()} | {noresult, false}. - is_valid_otp_app(File) -> %% Is this an ebin dir? EbinDir = filename:dirname(File), @@ -159,7 +208,9 @@ is_valid_otp_app(File) -> -spec gather_application_info(file:name(), file:filename()) -> - {ok, rlx_app_info:t()} | {error, Reason::term()}. + {ok, rlx_app_info:t()} | + {warning, Reason::term()} | + {error, Reason::term()}. gather_application_info(EbinDir, File) -> AppDir = filename:dirname(EbinDir), case file:consult(File) of @@ -175,7 +226,9 @@ gather_application_info(EbinDir, File) -> file:name(), atom(), proplists:proplist()) -> - {ok, list()} | {error, Reason::term()}. + {ok, list()} | + {warning, Reason::term()} | + {error, Reason::term()}. validate_application_info(EbinDir, AppFile, AppName, AppDetail) -> AppDir = filename:dirname(EbinDir), case get_modules_list(AppFile, AppDetail) of @@ -190,7 +243,9 @@ validate_application_info(EbinDir, AppFile, AppName, AppDetail) -> end. -spec get_modules_list(file:name(), proplists:proplist()) -> - {ok, list()} | {error, Reason::term()}. + {ok, list()} | + {warning, Reason::term()} | + {error, Reason::term()}. get_modules_list(AppFile, AppDetail) -> case proplists:get_value(modules, AppDetail) of undefined -> diff --git a/test/rlx_discover_SUITE.erl b/test/rlx_discover_SUITE.erl index e75ea36..846c62b 100644 --- a/test/rlx_discover_SUITE.erl +++ b/test/rlx_discover_SUITE.erl @@ -26,7 +26,9 @@ all/0, normal_case/1, no_beam_case/1, - bad_ebin_case/1]). + bad_ebin_case/1, + shallow_app_discovery/1 + ]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -53,7 +55,7 @@ init_per_testcase(_, Config) -> all() -> - [normal_case, no_beam_case, bad_ebin_case]. + [normal_case, no_beam_case, bad_ebin_case, shallow_app_discovery]. normal_case(Config) -> LibDir1 = proplists:get_value(lib1, Config), @@ -83,8 +85,7 @@ normal_case(Config) -> lists:foreach(fun(App) -> ?assertMatch(true, lists:member(App, rlx_state:available_apps(State2))) end, Apps2), - Length = erlang:length(Apps2) + - erlang:length(Apps2), + Length = erlang:length(Apps1) + erlang:length(Apps2), ?assertMatch(Length, erlang:length(rlx_state:available_apps(State2))). no_beam_case(Config) -> @@ -144,6 +145,38 @@ bad_ebin_case(Config) -> ?assertMatch([], [App || App <- rlx_state:available_apps(State2), BadName =:= rlx_app_info:name(App)]). +shallow_app_discovery(Config) -> + LibDir1 = proplists:get_value(lib1, Config), + Apps1 = [(fun({Name, Vsn}) -> + create_app(LibDir1, Name, Vsn) + end)(App) + || + App <- + [{create_random_name("lib_app1_"), create_random_vsn()} + || _ <- lists:seq(1, 100)]], + + LibDir2 = proplists:get_value(lib2, Config), + Apps2 = [(fun({Name, Vsn}) -> + create_app(LibDir2, Name, Vsn) + end)(App) + || App <- + [{create_random_name("lib_app2_"), create_random_vsn()} + || _ <- lists:seq(1, 100)]], + State0 = rlx_state:put(proplists:get_value(state, Config), + default_libs, false), + State1 = rlx_state:put(State0, enable_shallow_app_discovery, true), + {DiscoverProvider, {ok, State2}} = rlx_provider:new(rlx_prv_discover, State1), + {ok, State3} = rlx_provider:do(DiscoverProvider, State2), + lists:foreach(fun(App) -> + ?assertMatch(true, lists:member(App, rlx_state:available_apps(State3))) + end, Apps1), + + lists:foreach(fun(App) -> + ?assertMatch(true, lists:member(App, rlx_state:available_apps(State3))) + end, Apps2), + Length = erlang:length(Apps1) + erlang:length(Apps2), + ?assertMatch(Length, erlang:length(rlx_state:available_apps(State3))). + %%%=================================================================== %%% Helper functions %%%=================================================================== -- cgit v1.2.3