Support OTP-20 Unicode functions

Use either optional compilation or version-safe variants of the string
functions. Prevents warnings when the switch to OTP-21 will happen.

10 files changed, 64 insertions, 28 deletions

diff --git a/priv/templates/extended_bin b/priv/templates/extended_bin
index 19d9d29..baa239c 100755
--- a/priv/templates/extended_bin
+++ b/priv/templates/extended_bin
@@ -151,7 +151,7 @@ relx_get_nodename() {
     id="longname$(relx_gen_id)-${NAME}"
     "$BINDIR/erl" -boot start_clean \
         -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
-        -eval '[H]=tl(string:tokens(atom_to_list(node()),"@")), io:format("~s~n",[H]), halt()' \
+        -eval '[_,H]=re:split(atom_to_list(node()),"@",[unicode,{return,list}]), io:format("~s~n",[H]), halt()' \
         -noshell ${NAME_TYPE} $id
 }
 
diff --git a/priv/templates/nodetool b/priv/templates/nodetool
index 1f409f5..816be9c 100644
--- a/priv/templates/nodetool
+++ b/priv/templates/nodetool
@@ -57,7 +57,7 @@ main(Args) ->
             % spaces, so this converts all of that to a single string to parse
             String = binary_to_list(
                       list_to_binary(
-                        string:join(ListOfArgs," ")
+                        join(ListOfArgs," ")
                       )
                     ),
 
@@ -126,16 +126,16 @@ epmd_path() ->
 
 
 nodename(Name) ->
-    case string:tokens(Name, "@") of
+    case re:split(Name, "@", [{return, list}, unicode]) of
         [_Node, _Host] ->
             list_to_atom(Name);
         [Node] ->
-            [_, Host] = string:tokens(atom_to_list(node()), "@"),
+            [_, Host] = re:split(atom_to_list(node()), "@", [{return, list}, unicode]),
             list_to_atom(lists:concat([Node, "@", Host]))
     end.
 
 append_node_suffix(Name, Suffix) ->
-    case string:tokens(Name, "@") of
+    case re:split(Name, "@", [{return, list}, unicode]) of
         [Node, Host] ->
             list_to_atom(lists:concat([Node, Suffix, os:getpid(), "@", Host]));
         [Node] ->
@@ -165,3 +165,13 @@ consult(Cont, Str, Acc) ->
         {more, Cont1} ->
             consult(Cont1, eof, Acc)
     end.
+
+%% string:join/2 copy; string:join/2 is getting obsoleted
+%% and replaced by lists:join/2, but lists:join/2 is too new
+%% for version support (only appeared in 19.0) so it cannot be
+%% used. Instead we just adopt join/2 locally and hope it works
+%% for most unicode use cases anyway.
+join([], Sep) when is_list(Sep) ->
+    [];
+join([H|T], Sep) ->
+    H ++ lists:append([Sep ++ X || X <- T]).
diff --git a/rebar.config b/rebar.config
index 762d6df..6686664 100644
--- a/rebar.config
+++ b/rebar.config
@@ -18,6 +18,7 @@
      {platform_define, "^1[8|9]", rand_module},
      {platform_define, "^2", rand_module},
      warnings_as_errors,
+     {platform_define, "^2", unicode_str},
      inline]}.
 
 %% Use OTP 18+ when dialyzing relx
@@ -53,6 +54,7 @@
                                          {erl_opts, [{platform_define, "^[0-9]+", namespaced_types},
                                                      {platform_define, "^R1[4|5]", deprecated_crypto},
                                                      {platform_define, "^((1[8|9])|2)", rand_module},
+                                                     {platform_define, "^2", unicode_str},
                                                      no_debug_info,
                                                      warnings_as_errors
                                                     ]},
@@ -61,7 +63,8 @@
             {override, bbmustache, [
                                    {erl_opts, [no_debug_info]},
                                    {deps, []}, {plugins, []}]},
-            {override, getopt, [{erl_opts, [no_debug_info]}]},
+            {override, getopt, [{erl_opts, [no_debug_info,
+                                            {platform_define, "^2", unicode_str}]}]},
             {override, providers, [{erl_opts, [no_debug_info]}]}
             ]}.
 
diff --git a/src/rlx_app_discovery.erl b/src/rlx_app_discovery.erl
index 0414a0a..642b887 100644
--- a/src/rlx_app_discovery.erl
+++ b/src/rlx_app_discovery.erl
@@ -40,7 +40,7 @@ do(State, LibDirs) ->
     ec_cmd_log:info(rlx_state:log(State),
                     fun() ->
                             ["Resolving OTP Applications from directories:\n",
-                             string:join([[rlx_util:indent(2), LibDir] || LibDir <- LibDirs], "\n")]
+                             rlx_string:join([[rlx_util:indent(2), LibDir] || LibDir <- LibDirs], "\n")]
                     end),
     resolve_app_metadata(State, LibDirs).
 
diff --git a/src/rlx_prv_app_discover.erl b/src/rlx_prv_app_discover.erl
index a8a40ae..d58cf71 100644
--- a/src/rlx_prv_app_discover.erl
+++ b/src/rlx_prv_app_discover.erl
@@ -132,7 +132,7 @@ add_system_lib_dir(State) ->
 add_environment_lib_dir(_State) ->
     case os:getenv("ERL_LIBS") of
         false -> [];
-        Libs -> [erlang:iolist_to_binary(L) || L <- string:tokens(Libs, ":")]
+        Libs -> [erlang:iolist_to_binary(L) || L <- rlx_string:lexemes(Libs, ":")]
     end.
 
 %% Order matters so this slow dedup needs to be used
diff --git a/src/rlx_prv_assembler.erl b/src/rlx_prv_assembler.erl
index a6bf5f8..5efa8b5 100644
--- a/src/rlx_prv_assembler.erl
+++ b/src/rlx_prv_assembler.erl
@@ -416,13 +416,13 @@ write_bin_file(State, Release, OutputDir, RelDir) ->
         _ ->
             VsnRelStartFile = case OsFamily of
                 unix -> VsnRel;
-                win32 -> string:concat(VsnRel, ".cmd")
+                win32 -> rlx_string:concat(VsnRel, ".cmd")
             end,
             ok = file:write_file(VsnRelStartFile, StartFile),
             ok = file:change_mode(VsnRelStartFile, 8#777),
             BareRelStartFile = case OsFamily of
                 unix -> BareRel;
-                win32 -> string:concat(BareRel, ".cmd")
+                win32 -> rlx_string:concat(BareRel, ".cmd")
             end,
             ok = file:write_file(BareRelStartFile, StartFile),
             ok = file:change_mode(BareRelStartFile, 8#777)
@@ -494,15 +494,15 @@ hook_filename(builtin_status) -> "hooks/builtin/status".
 hook_invocation({custom, CustomScript}) -> CustomScript;
 %% the pid builtin hook with no arguments writes to pid file
 %% at /var/run/{{ rel_name }}.pid
-hook_invocation(pid) -> string:join(["hooks/builtin/pid",
+hook_invocation(pid) -> rlx_string:join(["hooks/builtin/pid",
                                      "/var/run/$REL_NAME.pid"], "|");
-hook_invocation({pid, PidFile}) -> string:join(["hooks/builtin/pid",
+hook_invocation({pid, PidFile}) -> rlx_string:join(["hooks/builtin/pid",
                                                 PidFile], "|");
 hook_invocation(wait_for_vm_start) -> "hooks/builtin/wait_for_vm_start";
 hook_invocation({wait_for_process, Name}) ->
     %% wait_for_process takes an atom as argument
     %% which is the process name to wait for
-    string:join(["hooks/builtin/wait_for_process",
+    rlx_string:join(["hooks/builtin/wait_for_process",
                  atom_to_list(Name)], "|");
 hook_invocation(builtin_status) -> "hooks/builtin/status".
 
@@ -796,15 +796,15 @@ extended_bin_file_contents(OsFamily, RelName, RelVsn, ErtsVsn, ErlOpts, Hooks, E
         win32 -> extended_bin_windows
     end,
     %% turn all the hook lists into space separated strings
-    PreStartHooks = string:join(proplists:get_value(pre_start, Hooks, []), " "),
-    PostStartHooks = string:join(proplists:get_value(post_start, Hooks, []), " "),
-    PreStopHooks = string:join(proplists:get_value(pre_stop, Hooks, []), " "),
-    PostStopHooks = string:join(proplists:get_value(post_stop, Hooks, []), " "),
-    PreInstallUpgradeHooks = string:join(proplists:get_value(pre_install_upgrade,
+    PreStartHooks = rlx_string:join(proplists:get_value(pre_start, Hooks, []), " "),
+    PostStartHooks = rlx_string:join(proplists:get_value(post_start, Hooks, []), " "),
+    PreStopHooks = rlx_string:join(proplists:get_value(pre_stop, Hooks, []), " "),
+    PostStopHooks = rlx_string:join(proplists:get_value(post_stop, Hooks, []), " "),
+    PreInstallUpgradeHooks = rlx_string:join(proplists:get_value(pre_install_upgrade,
                                                 Hooks, []), " "),
-    PostInstallUpgradeHooks = string:join(proplists:get_value(post_install_upgrade,
+    PostInstallUpgradeHooks = rlx_string:join(proplists:get_value(post_install_upgrade,
                                                  Hooks, []), " "),
-    StatusHook = string:join(proplists:get_value(status, Hooks, []), " "),
+    StatusHook = rlx_string:join(proplists:get_value(status, Hooks, []), " "),
     {ExtensionsList1, ExtensionDeclarations1} =
         lists:foldl(fun({Name, Script},
                         {ExtensionsList0, ExtensionDeclarations0}) ->
@@ -816,10 +816,10 @@ extended_bin_file_contents(OsFamily, RelName, RelVsn, ErtsVsn, ErlOpts, Hooks, E
                     end, {[], []}, Extensions),
     % pipe separated string of extensions, to show on the start script usage
     % (eg. foo|bar)
-    ExtensionsList = string:join(ExtensionsList1 ++ ["undefined"], "|"),
+    ExtensionsList = rlx_string:join(ExtensionsList1 ++ ["undefined"], "|"),
     % command separated string of extension script declarations
     % (eg. foo_extension="path/to/foo_script")
-    ExtensionDeclarations = string:join(ExtensionDeclarations1, ";"),
+    ExtensionDeclarations = rlx_string:join(ExtensionDeclarations1, ";"),
     render(Template, [{rel_name, RelName}, {rel_vsn, RelVsn},
                       {erts_vsn, ErtsVsn}, {erl_opts, ErlOpts},
                       {pre_start_hooks, PreStartHooks},
@@ -833,7 +833,7 @@ extended_bin_file_contents(OsFamily, RelName, RelVsn, ErtsVsn, ErlOpts, Hooks, E
                       {extension_declarations, ExtensionDeclarations}]).
 
 erl_ini(OutputDir, ErtsVsn) ->
-    ErtsDirName = string:concat("erts-", ErtsVsn),
+    ErtsDirName = rlx_string:concat("erts-", ErtsVsn),
     BinDir = filename:join([OutputDir, ErtsDirName, bin]),
     render(erl_ini, [{bin_dir, BinDir}, {output_dir, OutputDir}]).
 
diff --git a/src/rlx_prv_release.erl b/src/rlx_prv_release.erl
index 2af8acd..8de1a51 100644
--- a/src/rlx_prv_release.erl
+++ b/src/rlx_prv_release.erl
@@ -202,7 +202,7 @@ set_resolved(State, Release0, Pkgs) ->
                 ErtsDir ->
                     try
                         [Erts | _] = filelib:wildcard(filename:join(ErtsDir, "erts-*")),
-                        [_, ErtsVsn] = string:tokens(filename:basename(Erts), "-"),
+                        [_, ErtsVsn] = rlx_string:lexemes(filename:basename(Erts), "-"),
                         {ok, rlx_state:add_realized_release(State, rlx_release:erts(Release1, ErtsVsn))}
                     catch
                         _:_ ->
diff --git a/src/rlx_rel_discovery.erl b/src/rlx_rel_discovery.erl
index fca416f..767ceac 100644
--- a/src/rlx_rel_discovery.erl
+++ b/src/rlx_rel_discovery.erl
@@ -42,7 +42,7 @@ do(State, LibDirs, AppMeta) ->
             ec_cmd_log:info(rlx_state:log(State),
                             fun() ->
                                     ["Resolving available OTP Releases from directories:\n",
-                                     string:join([[rlx_util:indent(2), LibDir] || LibDir <- LibDirs], "\n")]
+                                     rlx_string:join([[rlx_util:indent(2), LibDir] || LibDir <- LibDirs], "\n")]
                             end),
             resolve_rel_metadata(State, LibDirs, AppMeta)
     end.
diff --git a/src/rlx_string.erl b/src/rlx_string.erl
new file mode 100644
index 0000000..1f9cc0c
--- /dev/null
+++ b/src/rlx_string.erl
@@ -0,0 +1,23 @@
+%% Compatibility module for the string API changes between
+%% OTP-19 and OTP-21, where Unicode support means the deprecation
+%% of a lot of string functions.
+-module(rlx_string).
+-export([concat/2, lexemes/2, join/2]).
+
+-ifdef(unicode_str).
+concat(Str1, Str2) -> unicode:characters_to_list([Str1,Str2]).
+lexemes(Str, Separators) -> string:lexemes(Str, Separators).
+-else.
+concat(Str1, Str2) -> string:concat(Str1, Str2).
+lexemes(Str, Separators) -> string:tokens(Str, Separators).
+-endif.
+
+%% string:join/2 copy; string:join/2 is getting obsoleted
+%% and replaced by lists:join/2, but lists:join/2 is too new
+%% for version support (only appeared in 19.0) so it cannot be
+%% used. Instead we just adopt join/2 locally and hope it works
+%% for most unicode use cases anyway.
+join([], Sep) when is_list(Sep) ->
+    [];
+join([H|T], Sep) ->
+    H ++ lists:append([Sep ++ X || X <- T]).
diff --git a/test/rlx_depsolver_tester.erl b/test/rlx_depsolver_tester.erl
index 9b55603..9defd91 100644
--- a/test/rlx_depsolver_tester.erl
+++ b/test/rlx_depsolver_tester.erl
@@ -406,7 +406,7 @@ process_line(Device, "\n", Acc) ->
     process_line(Device, io:get_line(Device, ""),
                  Acc);
 process_line(Device, [$\s | Rest], [{Pkg, Vsn, Deps} | Acc]) ->
-    [DepPackage, Type,  DepVsn] = string:tokens(Rest, " \n"),
+    [DepPackage, Type,  DepVsn] = rlx_string:lexemes(Rest, " \n"),
     Dep =
         case Type of
             "=" ->
@@ -417,7 +417,7 @@ process_line(Device, [$\s | Rest], [{Pkg, Vsn, Deps} | Acc]) ->
     process_line(Device, io:get_line(Device, ""),
                  [{Pkg, Vsn, [Dep | Deps]} | Acc]);
 process_line(Device, Pkg, Acc) ->
-    [Package, Vsn] = string:tokens(Pkg, " \n"),
+    [Package, Vsn] = rlx_string:lexemes(Pkg, " \n"),
     process_line(Device, io:get_line(Device, ""),
                  [{Package, Vsn, []} | Acc]).
 
@@ -427,7 +427,7 @@ process_packages(Pkgs) ->
                 end, rlx_depsolver:new_graph(), Pkgs).
 
 get_constraints(ConLine) ->
-    AppVsns = string:tokens(ConLine, " \n"),
+    AppVsns = rlx_string:lexemes(ConLine, " \n"),
     lists:map(fun(AppCon) ->
                       parse_app(AppCon, [])
               end, AppVsns).