Merge branch 'bjorn/stdlib/string-tokens/OTP-12422'

* bjorn/stdlib/string-tokens/OTP-12422:
  Optimize string:tokens/2
  Modernize and strengthen the test case for string:tokens/2

2 files changed, 64 insertions, 22 deletions

diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl
index f9b083a56d..f6903d1c3d 100644
--- a/lib/stdlib/src/string.erl
+++ b/lib/stdlib/src/string.erl
@@ -221,23 +221,47 @@ substr2([_|String], S) -> substr2(String, S-1).
       Tokens :: [Token :: nonempty_string()].
 
 tokens(S, Seps) ->
-    tokens1(S, Seps, []).
+    case Seps of
+	[] ->
+	    case S of
+		[] -> [];
+		[_|_] -> [S]
+	    end;
+	[C] ->
+	    tokens_single_1(reverse(S), C, []);
+	[_|_] ->
+	    tokens_multiple_1(reverse(S), Seps, [])
+    end.
 
-tokens1([C|S], Seps, Toks) ->
+tokens_single_1([Sep|S], Sep, Toks) ->
+    tokens_single_1(S, Sep, Toks);
+tokens_single_1([C|S], Sep, Toks) ->
+    tokens_single_2(S, Sep, Toks, [C]);
+tokens_single_1([], _, Toks) ->
+    Toks.
+
+tokens_single_2([Sep|S], Sep, Toks, Tok) ->
+    tokens_single_1(S, Sep, [Tok|Toks]);
+tokens_single_2([C|S], Sep, Toks, Tok) ->
+    tokens_single_2(S, Sep, Toks, [C|Tok]);
+tokens_single_2([], _Sep, Toks, Tok) ->
+    [Tok|Toks].
+
+tokens_multiple_1([C|S], Seps, Toks) ->
     case member(C, Seps) of
-	true -> tokens1(S, Seps, Toks);
-	false -> tokens2(S, Seps, Toks, [C])
+	true -> tokens_multiple_1(S, Seps, Toks);
+	false -> tokens_multiple_2(S, Seps, Toks, [C])
     end;
-tokens1([], _Seps, Toks) ->
-    reverse(Toks).
+tokens_multiple_1([], _Seps, Toks) ->
+    Toks.
 
-tokens2([C|S], Seps, Toks, Cs) ->
+tokens_multiple_2([C|S], Seps, Toks, Tok) ->
     case member(C, Seps) of
-	true -> tokens1(S, Seps, [reverse(Cs)|Toks]);
-	false -> tokens2(S, Seps, Toks, [C|Cs])
+	true -> tokens_multiple_1(S, Seps, [Tok|Toks]);
+	false -> tokens_multiple_2(S, Seps, Toks, [C|Tok])
     end;
-tokens2([], _Seps, Toks, Cs) ->
-    reverse([reverse(Cs)|Toks]).
+tokens_multiple_2([], _Seps, Toks, Tok) ->
+    [Tok|Toks].
 
 -spec chars(Character, Number) -> String when
       Character :: char(),
diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl
index fccd1bef95..a55c710d50 100644
--- a/lib/stdlib/test/string_SUITE.erl
+++ b/lib/stdlib/test/string_SUITE.erl
@@ -217,21 +217,39 @@ substr(Config) when is_list(Config) ->
     ?line {'EXIT',_} = (catch string:substr("1234", "1")),
     ok.
 
-tokens(suite) ->
-    [];
-tokens(doc) ->
-    [];
 tokens(Config) when is_list(Config) ->
-    ?line [] = string:tokens("",""),
-    ?line [] = string:tokens("abc","abc"),
-    ?line ["abc"] = string:tokens("abc", ""),
-    ?line ["1","2 34","4","5"] = string:tokens("1,2 34,4;5", ";,"),
-    %% invalid arg type
-    ?line {'EXIT',_} = (catch string:tokens('x,y', ",")), 
+    [] = string:tokens("",""),
+    [] = string:tokens("abc","abc"),
+    ["abc"] = string:tokens("abc", ""),
+    ["1","2 34","45","5","6","7"] = do_tokens("1,2 34,45;5,;6;,7", ";,"),
+
     %% invalid arg type
-    ?line {'EXIT',_} = (catch string:tokens("x,y", ',')), 
+    {'EXIT',_} = (catch string:tokens('x,y', ",")),
+    {'EXIT',_} = (catch string:tokens("x,y", ',')),
     ok.
 
+do_tokens(S0, Sep0) ->
+    [H|T] = Sep0,
+    S = [replace_sep(C, T, H) || C <- S0],
+    Sep = [H],
+    io:format("~p ~p\n", [S0,Sep0]),
+    io:format("~p ~p\n", [S,Sep]),
+
+    Res = string:tokens(S0, Sep0),
+    Res = string:tokens(Sep0++S0, Sep0),
+    Res = string:tokens(S0++Sep0, Sep0),
+
+    Res = string:tokens(S, Sep),
+    Res = string:tokens(Sep++S, Sep),
+    Res = string:tokens(S++Sep, Sep),
+
+    Res.
+
+replace_sep(C, Seps, New) ->
+    case lists:member(C, Seps) of
+	true -> New;
+	false -> C
+    end.
 
 chars(suite) ->
     [];