Fail if ':' is passed to binary_to_integer/2

Before:

1> binary_to_integer(<<":">>, 16).
3

After:

1> binary_to_integer(<<":">>, 16).
** exception error: bad argument
     in function  binary_to_integer/2
        called as binary_to_integer(<<":">>,16)

Prior to this change, both list_to_integer/2 and binary_to_integer/2
would convert strings with values between ASCII '9' up to '0'+base for
base > 10. For example, when converting in base 16, you could pass ':',
';', '<', '=', '>', and '?' without getting an exception.  This was due
to a missing check in c2int_is_invalid_char().

This change adds the missing check and a regression test for passing
':'. It also simplifies the code and tightens up an out-of-bounds check
to make it off-by-one rather than off-by-two.

2 files changed, 11 insertions, 6 deletions

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 5eaf262cd8..c5cb268f09 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -2549,12 +2549,17 @@ int term_equals_2pow32(Eterm x)
     }
 }
 
+static ERTS_INLINE int c2int_is_valid_char(byte ch, int base) {
+    if (base <= 10)
+        return (ch >= '0' && ch < ('0' + base));
+    else
+        return (ch >= '0' && ch <= '9')
+            || (ch >= 'A' && ch < ('A' + base - 10))
+            || (ch >= 'a' && ch < ('a' + base - 10));
+}
+
 static ERTS_INLINE int c2int_is_invalid_char(byte ch, int base) {
-    return (ch < '0'
-            || (ch > ('0' + base - 1)
-                && !(base > 10
-                     && ((ch >= 'a' && ch < ('a' + base - 10))
-                         || (ch >= 'A' && ch < ('A' + base - 10))))));
+    return !c2int_is_valid_char(ch, base);
 }
 
 static ERTS_INLINE byte c2int_digit_from_base(byte ch) {
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index 1c76eb8019..ac1a667185 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -491,7 +491,7 @@ t_string_to_integer(Config) when is_list(Config) ->
 				       list_to_binary(Value),Base)),
 			  {'EXIT', {badarg, _}} =
 			      (catch erlang:list_to_integer(Value,Base))
-		  end,[{" 1",1},{" 1",37},{"2",2},{"C",11},
+		  end,[{" 1",1},{" 1",37},{"2",2},{"B",11},{"b",11},{":", 16},
 		       {"1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111z",16},
 		       {"1z111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111",16},
 		       {"111z11111111",16}]),