Merge branch 'bjorn/stdlib/string-tokens/OTP-12422'

* bjorn/stdlib/string-tokens/OTP-12422:
  Optimize string:tokens/2
  Modernize and strengthen the test case for string:tokens/2

1 files changed, 29 insertions, 11 deletions

diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl
index fccd1bef95..a55c710d50 100644
--- a/lib/stdlib/test/string_SUITE.erl
+++ b/lib/stdlib/test/string_SUITE.erl
@@ -217,21 +217,39 @@ substr(Config) when is_list(Config) ->
     ?line {'EXIT',_} = (catch string:substr("1234", "1")),
     ok.
 
-tokens(suite) ->
-    [];
-tokens(doc) ->
-    [];
 tokens(Config) when is_list(Config) ->
-    ?line [] = string:tokens("",""),
-    ?line [] = string:tokens("abc","abc"),
-    ?line ["abc"] = string:tokens("abc", ""),
-    ?line ["1","2 34","4","5"] = string:tokens("1,2 34,4;5", ";,"),
-    %% invalid arg type
-    ?line {'EXIT',_} = (catch string:tokens('x,y', ",")), 
+    [] = string:tokens("",""),
+    [] = string:tokens("abc","abc"),
+    ["abc"] = string:tokens("abc", ""),
+    ["1","2 34","45","5","6","7"] = do_tokens("1,2 34,45;5,;6;,7", ";,"),
+
     %% invalid arg type
-    ?line {'EXIT',_} = (catch string:tokens("x,y", ',')), 
+    {'EXIT',_} = (catch string:tokens('x,y', ",")),
+    {'EXIT',_} = (catch string:tokens("x,y", ',')),
     ok.
 
+do_tokens(S0, Sep0) ->
+    [H|T] = Sep0,
+    S = [replace_sep(C, T, H) || C <- S0],
+    Sep = [H],
+    io:format("~p ~p\n", [S0,Sep0]),
+    io:format("~p ~p\n", [S,Sep]),
+
+    Res = string:tokens(S0, Sep0),
+    Res = string:tokens(Sep0++S0, Sep0),
+    Res = string:tokens(S0++Sep0, Sep0),
+
+    Res = string:tokens(S, Sep),
+    Res = string:tokens(Sep++S, Sep),
+    Res = string:tokens(S++Sep, Sep),
+
+    Res.
+
+replace_sep(C, Seps, New) ->
+    case lists:member(C, Seps) of
+	true -> New;
+	false -> C
+    end.
 
 chars(suite) ->
     [];