From ee477fb31ad9a7603aaea8922aaa9ecd3712ee2d Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 12 Jun 2018 19:07:22 -0600 Subject: support for OTP21's sys.config.src file in releases (#647) * support for OTP21's sys.config.src file in releases * always replace os vars in .src files if found * support vm_args_src to be consistent with sys_config_src * add newlines after warning logs * improve sys and vm src config tests --- priv/templates/extended_bin | 5 +- src/rlx_config.erl | 4 ++ src/rlx_prv_assembler.erl | 97 ++++++++++++++++++++++++-------- src/rlx_state.erl | 22 ++++++++ test/rlx_extended_bin_SUITE.erl | 119 +++++++++++++++++++++++++++++++++++++++- test/rlx_release_SUITE.erl | 73 +++++++++++++++++++++++- 6 files changed, 292 insertions(+), 28 deletions(-) diff --git a/priv/templates/extended_bin b/priv/templates/extended_bin index dc01d5f..d044a38 100755 --- a/priv/templates/extended_bin +++ b/priv/templates/extended_bin @@ -263,8 +263,11 @@ add_path() { check_replace_os_vars() { IN_FILE_PATH=$(add_path $1 $2) OUT_FILE_PATH="$IN_FILE_PATH" + SRC_FILE_PATH="$IN_FILE_PATH.src" ORIG_FILE_PATH="$IN_FILE_PATH.orig" - if [ $RELX_REPLACE_OS_VARS ]; then + if [ -f "$SRC_FILE_PATH" ]; then + replace_os_vars "$SRC_FILE_PATH" "$OUT_FILE_PATH" + elif [ $RELX_REPLACE_OS_VARS ]; then # Create a new file in the same location as original OUT_FILE_PATH=$(make_out_file_path $IN_FILE_PATH) # If vm.args.orig or sys.config.orig is present then use that diff --git a/src/rlx_config.erl b/src/rlx_config.erl index 57aac71..ee58db5 100644 --- a/src/rlx_config.erl +++ b/src/rlx_config.erl @@ -256,10 +256,14 @@ load_terms({vm_args, false}, {ok, State}) -> {ok, rlx_state:vm_args(State, false)}; load_terms({vm_args, VmArgs}, {ok, State}) -> {ok, rlx_state:vm_args(State, filename:absname(VmArgs))}; +load_terms({vm_args_src, VmArgs}, {ok, State}) -> + {ok, rlx_state:vm_args_src(State, filename:absname(VmArgs))}; load_terms({sys_config, false}, {ok, State}) -> {ok, rlx_state:sys_config(State, false)}; load_terms({sys_config, SysConfig}, {ok, State}) -> {ok, rlx_state:sys_config(State, filename:absname(SysConfig))}; +load_terms({sys_config_src, SysConfigSrc}, {ok, State}) -> + {ok, rlx_state:sys_config_src(State, filename:absname(SysConfigSrc))}; load_terms({root_dir, Root}, {ok, State}) -> {ok, rlx_state:root_dir(State, filename:absname(Root))}; load_terms({output_dir, OutputDir}, {ok, State}) -> diff --git a/src/rlx_prv_assembler.erl b/src/rlx_prv_assembler.erl index 65975e5..705e51a 100644 --- a/src/rlx_prv_assembler.erl +++ b/src/rlx_prv_assembler.erl @@ -86,8 +86,17 @@ format_error({unresolved_release, RelName, RelVsn}) -> format_error({ec_file_error, AppDir, TargetDir, E}) -> io_lib:format("Unable to copy OTP App from ~s to ~s due to ~p", [AppDir, TargetDir, E]); +format_error({vmargs_does_not_exist, Path}) -> + io_lib:format("The vm.args file specified for this release (~s) does not exist!", + [Path]); +format_error({vmargs_src_does_not_exist, Path}) -> + io_lib:format("The vm.args.src file specified for this release (~s) does not exist!", + [Path]); format_error({config_does_not_exist, Path}) -> - io_lib:format("The config file specified for this release (~s) does not exist!", + io_lib:format("The sys.config file specified for this release (~s) does not exist!", + [Path]); +format_error({config_src_does_not_exist, Path}) -> + io_lib:format("The sys.config.src file specified for this release (~s) does not exist!", [Path]); format_error({sys_config_parse_error, ConfigPath, Reason}) -> io_lib:format("The config file (~s) specified for this release could not be opened or parsed: ~s", @@ -557,18 +566,38 @@ generate_start_erl_data_file(Release, ReleasesDir) -> copy_or_generate_vmargs_file(State, Release, RelDir) -> RelVmargsPath = filename:join([RelDir, "vm.args"]), - case rlx_state:vm_args(State) of - false -> - ok; + RelVmargsSrcPath = filename:join([RelDir, "vm.args.src"]), + case rlx_state:vm_args_src(State) of undefined -> - RelName = erlang:atom_to_list(rlx_release:name(Release)), - unless_exists_write_default(RelVmargsPath, vm_args_file(RelName)); - ArgsPath -> - case filelib:is_regular(ArgsPath) of + case rlx_state:vm_args(State) of + false -> + ok; + undefined -> + RelName = erlang:atom_to_list(rlx_release:name(Release)), + unless_exists_write_default(RelVmargsPath, vm_args_file(RelName)); + ArgsPath -> + case filelib:is_regular(ArgsPath) of + false -> + ?RLX_ERROR({vmargs_does_not_exist, ArgsPath}); + true -> + copy_or_symlink_config_file(State, ArgsPath, RelVmargsPath) + end + end; + ArgsSrcPath -> + %% print a warning if vm_args is also set + case rlx_state:vm_args(State) of + undefined -> + ok; + _-> + ec_cmd_log:warn(rlx_state:log(State), + "Both vm_args_src and vm_args are set, vm_args will be ignored~n", []) + end, + + case filelib:is_regular(ArgsSrcPath) of false -> - ?RLX_ERROR({vmargs_does_not_exist, ArgsPath}); + ?RLX_ERROR({vmargs_src_does_not_exist, ArgsSrcPath}); true -> - copy_or_symlink_config_file(State, ArgsPath, RelVmargsPath) + copy_or_symlink_config_file(State, ArgsSrcPath, RelVmargsSrcPath) end end. @@ -577,23 +606,43 @@ copy_or_generate_vmargs_file(State, Release, RelDir) -> {ok, rlx_state:t()} | relx:error(). copy_or_generate_sys_config_file(State, RelDir) -> RelSysConfPath = filename:join([RelDir, "sys.config"]), - case rlx_state:sys_config(State) of - false -> - ok; + RelSysConfSrcPath = filename:join([RelDir, "sys.config.src"]), + case rlx_state:sys_config_src(State) of undefined -> - unless_exists_write_default(RelSysConfPath, sys_config_file()); - ConfigPath -> - case filelib:is_regular(ConfigPath) of + case rlx_state:sys_config(State) of false -> - ?RLX_ERROR({config_does_not_exist, ConfigPath}); - true -> - %% validate sys.config is valid Erlang terms - case file:consult(ConfigPath) of - {ok, _} -> - copy_or_symlink_config_file(State, ConfigPath, RelSysConfPath); - {error, Reason} -> - ?RLX_ERROR({sys_config_parse_error, ConfigPath, Reason}) + ok; + undefined -> + unless_exists_write_default(RelSysConfPath, sys_config_file()); + ConfigPath -> + case filelib:is_regular(ConfigPath) of + false -> + ?RLX_ERROR({config_does_not_exist, ConfigPath}); + true -> + %% validate sys.config is valid Erlang terms + case file:consult(ConfigPath) of + {ok, _} -> + copy_or_symlink_config_file(State, ConfigPath, RelSysConfPath); + {error, Reason} -> + ?RLX_ERROR({sys_config_parse_error, ConfigPath, Reason}) + end end + end; + ConfigSrcPath -> + %% print a warning if sys_config is also set + case rlx_state:sys_config(State) of + P when P =:= false orelse P =:= undefined -> + ok; + _-> + ec_cmd_log:warn(rlx_state:log(State), + "Both sys_config_src and sys_config are set, sys_config will be ignored~n", []) + end, + + case filelib:is_regular(ConfigSrcPath) of + false -> + ?RLX_ERROR({config_src_does_not_exist, ConfigSrcPath}); + true -> + copy_or_symlink_config_file(State, ConfigSrcPath, RelSysConfSrcPath) end end. diff --git a/src/rlx_state.erl b/src/rlx_state.erl index 5032628..5488a41 100644 --- a/src/rlx_state.erl +++ b/src/rlx_state.erl @@ -54,8 +54,12 @@ hooks/2, vm_args/1, vm_args/2, + vm_args_src/1, + vm_args_src/2, sys_config/1, sys_config/2, + sys_config_src/1, + sys_config_src/2, root_dir/1, root_dir/2, add_configured_release/2, @@ -106,7 +110,9 @@ available_apps=[] :: [rlx_app_info:t()], default_configured_release :: {rlx_release:name() | undefined, rlx_release:vsn() |undefined} | undefined, vm_args :: file:filename() | false | undefined, + vm_args_src :: file:filename() | undefined, sys_config :: file:filename() | false | undefined, + sys_config_src :: file:filename() | undefined, overrides=[] :: [{AppName::atom(), Directory::file:filename()}], skip_apps=[] :: [AppName::atom()], exclude_apps=[] :: [AppName::atom()], @@ -284,6 +290,14 @@ vm_args(#state_t{vm_args=VmArgs}) -> vm_args(State, VmArgs) -> State#state_t{vm_args=VmArgs}. +-spec vm_args_src(t()) -> file:filename() | undefined. +vm_args_src(#state_t{vm_args_src=VmArgs}) -> + VmArgs. + +-spec vm_args_src(t(), undefined | file:filename()) -> t(). +vm_args_src(State, VmArgs) -> + State#state_t{vm_args_src=VmArgs}. + -spec sys_config(t()) -> file:filename() | false | undefined. sys_config(#state_t{sys_config=SysConfig}) -> SysConfig. @@ -292,6 +306,14 @@ sys_config(#state_t{sys_config=SysConfig}) -> sys_config(State, SysConfig) -> State#state_t{sys_config=SysConfig}. +-spec sys_config_src(t()) -> file:filename() | undefined. +sys_config_src(#state_t{sys_config_src=SysConfigSrc}) -> + SysConfigSrc. + +-spec sys_config_src(t(), file:filename() | undefined) -> t(). +sys_config_src(State, SysConfigSrc) -> + State#state_t{sys_config_src=SysConfigSrc}. + -spec root_dir(t()) -> file:filename() | undefined. root_dir(#state_t{root_dir=RootDir}) -> RootDir. diff --git a/test/rlx_extended_bin_SUITE.erl b/test/rlx_extended_bin_SUITE.erl index 9fd47c0..86a9d23 100644 --- a/test/rlx_extended_bin_SUITE.erl +++ b/test/rlx_extended_bin_SUITE.erl @@ -42,6 +42,7 @@ escript/1, remote_console/1, shortname_remote_console/1, replace_os_vars/1, + replace_os_vars_sys_config_vm_args_src/1, replace_os_vars_multi_node/1, replace_os_vars_included_config/1, replace_os_vars_custom_location/1, @@ -85,7 +86,8 @@ all() -> start_fail_when_missing_argsfile, start_fail_when_nonreadable_argsfile, start_fail_when_relative_argsfile, start_fail_when_circular_argsfiles, ping, shortname_ping, longname_ping, attach, pid, restart, reboot, escript, - remote_console, shortname_remote_console, replace_os_vars, replace_os_vars_multi_node, replace_os_vars_included_config, + remote_console, shortname_remote_console, replace_os_vars, replace_os_vars_sys_config_vm_args_src, replace_os_vars_multi_node, + replace_os_vars_included_config, replace_os_vars_custom_location, replace_os_vars_dev_mode, replace_os_vars_twice, custom_start_script_hooks, builtin_wait_for_vm_start_script_hook, builtin_pid_start_script_hook, builtin_wait_for_process_start_script_hook, mixed_custom_and_builtin_start_script_hooks, @@ -579,6 +581,121 @@ replace_os_vars(Config) -> {"COOKIE", "cookie2"}]), ok. + +replace_os_vars_sys_config_vm_args_src(Config) -> + LibDir1 = proplists:get_value(lib1, Config), + + rlx_test_utils:create_app(LibDir1, "goal_app", "0.0.1", [stdlib,kernel], []), + + ConfigFile = filename:join([LibDir1, "relx.config"]), + SysConfigSrc = filename:join([LibDir1, "sys.config.src"]), + VmArgs = filename:join([LibDir1, "vm.args.src"]), + + rlx_test_utils:write_config(ConfigFile, + [{release, {foo, "0.0.1"}, + [goal_app]}, + {lib_dirs, [filename:join(LibDir1, "*")]}, + {sys_config_src, SysConfigSrc}, + {vm_args_src, VmArgs}, + {generate_start_script, true}, + {extended_start_script, true} + ]), + + %% new with sys.config.src it doesn't have to be valid Erlang + %% until after var replacemen at runtime. + ec_file:write(SysConfigSrc, "[{goal_app, [{var1, ${VAR1}}]}]."), + ec_file:write(VmArgs, "-sname ${NODENAME}\n\n" + "-setcookie ${COOKIE}\n"), + + OutputDir = filename:join([proplists:get_value(priv_dir, Config), + rlx_test_utils:create_random_name("relx-output")]), + + {ok, _State} = relx:do([{relname, foo}, + {relvsn, "0.0.1"}, + {goals, []}, + {lib_dirs, [LibDir1]}, + {log_level, 3}, + {output_dir, OutputDir}, + {config, ConfigFile}], ["release"]), + + {ok, _} = sh(filename:join([OutputDir, "foo", "bin", "foo start"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node1"}, + {"COOKIE", "cookie1"}, + {"VAR1", "101"}]), + timer:sleep(2000), + {ok, "pong"} = sh(filename:join([OutputDir, "foo", "bin", "foo ping"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node1"}, + {"COOKIE", "cookie1"}]), + {ok, "101"} = sh(filename:join([OutputDir, "foo", "bin", + "foo eval '{ok, V} = application:get_env(goal_app, var1), V.'"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node1"}, + {"COOKIE", "cookie1"}]), + {ok, "\"node1\""} = sh(filename:join([OutputDir, "foo", "bin", + "foo eval '[Node,_] = re:split(atom_to_list(node()), \"@\"),binary_to_list(Node).'"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node1"}, + {"COOKIE", "cookie1"}]), + {ok, "cookie1"} = sh(filename:join([OutputDir, "foo", "bin", + "foo eval 'erlang:get_cookie().'"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node1"}, + {"COOKIE", "cookie1"}]), + {ok, _Node1} = sh(filename:join([OutputDir, "foo", "bin", + "foo eval 'atom_to_list(node()).'"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node1"}, + {"COOKIE", "cookie1"}]), + %% check that the replaced files have been created in the right place + ?assert(ec_file:exists(filename:join([OutputDir, "foo", "releases", "0.0.1", + "sys.config"]))), + {ok, _} = sh(filename:join([OutputDir, "foo", "bin", "foo stop"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node1"}, + {"COOKIE", "cookie1"}]), + + %% start the node again but this time with different env variables to replace + {ok, _} = sh(filename:join([OutputDir, "foo", "bin", "foo start"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node2"}, + {"COOKIE", "cookie2"}, + {"VAR1", "201"}]), + timer:sleep(2000), + {ok, "pong"} = sh(filename:join([OutputDir, "foo", "bin", "foo ping"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node2"}, + {"COOKIE", "cookie2"}]), + {ok, "201"} = sh(filename:join([OutputDir, "foo", "bin", + "foo eval '{ok, V} = application:get_env(goal_app, var1), V.'"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node2"}, + {"COOKIE", "cookie2"}]), + {ok, "\"node2\""} = sh(filename:join([OutputDir, "foo", "bin", + "foo eval '[Node,_] = re:split(atom_to_list(node()), \"@\"),binary_to_list(Node).'"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node2"}, + {"COOKIE", "cookie2"}]), + {ok, "cookie2"} = sh(filename:join([OutputDir, "foo", "bin", + "foo eval 'erlang:get_cookie().'"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node2"}, + {"COOKIE", "cookie2"}]), + {ok, _Node2} = sh(filename:join([OutputDir, "foo", "bin", + "foo eval 'atom_to_list(node()).'"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node2"}, + {"COOKIE", "cookie2"}]), + %% check that the replaced files have been created in the right place + ?assert(ec_file:exists(filename:join([OutputDir, "foo", "releases", "0.0.1", + "sys.config"]))), + {ok, _} = sh(filename:join([OutputDir, "foo", "bin", "foo stop"]), + [{"RELX_REPLACE_OS_VARS", "1"}, + {"NODENAME", "node2"}, + {"COOKIE", "cookie2"}]), + ok. + replace_os_vars_multi_node(Config) -> LibDir1 = proplists:get_value(lib1, Config), diff --git a/test/rlx_release_SUITE.erl b/test/rlx_release_SUITE.erl index cfabc24..affc0ad 100644 --- a/test/rlx_release_SUITE.erl +++ b/test/rlx_release_SUITE.erl @@ -56,7 +56,8 @@ make_not_included_nodetool_release/1, make_src_release/1, make_excluded_src_release/1, - make_exclude_modules_release/1]). + make_exclude_modules_release/1, + make_release_with_sys_config_vm_args_src/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -92,7 +93,8 @@ all() -> make_config_script_release, make_release_twice, make_release_twice_dev_mode, make_erts_release, make_erts_config_release, make_included_nodetool_release, make_not_included_nodetool_release, - make_src_release, make_excluded_src_release, make_exclude_modules_release]. + make_src_release, make_excluded_src_release, make_exclude_modules_release, + make_release_with_sys_config_vm_args_src]. add_providers(Config) -> LibDir1 = proplists:get_value(lib1, Config), @@ -1443,6 +1445,73 @@ make_exclude_modules_release(Config) -> "non_goal_1-0.0.1", "ebin", "non_goal_1.app"]))). +make_release_with_sys_config_vm_args_src(Config) -> + LibDir1 = proplists:get_value(lib1, Config), + + rlx_test_utils:create_app(LibDir1, "goal_app_1", "0.0.1", [stdlib,kernel,non_goal_1], []), + rlx_test_utils:create_app(LibDir1, "lib_dep_1", "0.0.1", [stdlib,kernel], []), + rlx_test_utils:create_app(LibDir1, "goal_app_2", "0.0.1", [stdlib,kernel,goal_app_1,non_goal_2], []), + rlx_test_utils:create_app(LibDir1, "non_goal_1", "0.0.1", [stdlib,kernel], [lib_dep_1]), + rlx_test_utils:create_app(LibDir1, "non_goal_2", "0.0.1", [stdlib,kernel], []), + + %% the .src versions should take precedence and the others are not copied + SysConfig = filename:join([LibDir1, "config", "sys.config"]), + rlx_test_utils:write_config(SysConfig, [{this_is_a_test, "yup it is"}]), + + SysConfigSrc = filename:join([LibDir1, "config", "sys.config.src"]), + rlx_test_utils:write_config(SysConfigSrc, [{this_is_a_test, "yup it is"}]), + + VmArgs = filename:join([LibDir1, "config", "vm.args"]), + ec_file:write(VmArgs, ""), + + VmArgsSrc = filename:join([LibDir1, "config", "vm.args.src"]), + ec_file:write(VmArgsSrc, ""), + + ConfigFile = filename:join([LibDir1, "relx.config"]), + rlx_test_utils:write_config(ConfigFile, + [{dev_mode, true}, + {sys_config_src, SysConfigSrc}, + {vm_args_src, VmArgsSrc}, + {release, {foo, "0.0.1"}, + [goal_app_1, + goal_app_2]}]), + OutputDir = filename:join([proplists:get_value(priv_dir, Config), + rlx_test_utils:create_random_name("relx-output")]), + {ok, State} = relx:do(undefined, undefined, [], [LibDir1], 3, + OutputDir, ConfigFile), + [{{foo, "0.0.1"}, _Release}] = ec_dictionary:to_list(rlx_state:realized_releases(State)), + + + case os:type() of + {unix, _} -> + ?assert(ec_file:is_symlink(filename:join([OutputDir, "foo", "lib", "non_goal_1-0.0.1"]))), + ?assert(ec_file:is_symlink(filename:join([OutputDir, "foo", "lib", "non_goal_2-0.0.1"]))), + ?assert(ec_file:is_symlink(filename:join([OutputDir, "foo", "lib", "goal_app_1-0.0.1"]))), + ?assert(ec_file:is_symlink(filename:join([OutputDir, "foo", "lib", "goal_app_2-0.0.1"]))), + ?assert(ec_file:is_symlink(filename:join([OutputDir, "foo", "lib", "lib_dep_1-0.0.1"]))), + ?assert(ec_file:is_symlink(filename:join([OutputDir, "foo", "releases", "0.0.1", + "sys.config.src"]))), + ?assert(ec_file:is_symlink(filename:join([OutputDir, "foo", "releases", "0.0.1", + "vm.args.src"]))), + ?assert(not ec_file:is_symlink(filename:join([OutputDir, "foo", "releases", "0.0.1", + "sys.config"]))), + ?assert(not ec_file:is_symlink(filename:join([OutputDir, "foo", "releases", "0.0.1", + "sys.config"]))); + {win32, _} -> + ?assert(filelib:is_dir(filename:join([OutputDir, "foo", "lib", "non_goal_1-0.0.1"]))), + ?assert(filelib:is_dir(filename:join([OutputDir, "foo", "lib", "non_goal_2-0.0.1"]))), + ?assert(filelib:is_dir(filename:join([OutputDir, "foo", "lib", "goal_app_1-0.0.1"]))), + ?assert(filelib:is_dir(filename:join([OutputDir, "foo", "lib", "goal_app_2-0.0.1"]))), + ?assert(filelib:is_dir(filename:join([OutputDir, "foo", "lib", "lib_dep_1-0.0.1"]))), + ?assert(filelib:is_file(filename:join([OutputDir, "foo", "releases", "0.0.1", + "sys.config.src"]))), + ?assert(filelib:is_file(filename:join([OutputDir, "foo", "releases", "0.0.1", + "vm.args.src"]))), + ?assert(not filelib:is_file(filename:join([OutputDir, "foo", "releases", "0.0.1", + "sys.config"]))), + ?assert(not filelib:is_file(filename:join([OutputDir, "foo", "releases", "0.0.1", + "vm.args"]))) + end. %%%=================================================================== %%% Helper Functions -- cgit v1.2.3