erts: Fix bug in 'band' of two negative numbers, one big

Similar bug as for bxor fixed by abc4fd372d476821448dfb9

Ex:
1> io:format("~.16B\n", [-16#1110000000000000000 band (-1)]).
-1120000000000000000

Wrong result for

(-X bsl WS) band -Y.

where
X is any positive integer
WS is erlang:system_info(wordsize)*8*N where N is 1 or larger
Y is any positive integer smaller than (1 bsl WS)

Fix:
The subtraction of 1 (for 2-complement conversion)
must be carried along all the way to the last words.

2 files changed, 42 insertions, 4 deletions

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 323cd6c518..6497089419 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1139,8 +1139,11 @@ static dsize_t I_band(ErtsDigit* x, dsize_t xl, short xsgn,
 		*r++ = ~c1 & ~c2;
 		x++; y++;
 	    }
-	    while(xl--)
-		*r++ = ~*x++;
+	    while(xl--) {
+                DSUBb(*x,0,b1,c1);
+                *r++ = ~c1;
+		x++;
+            }
 	}
     }
     return I_btrail(r0, r, sign);
diff --git a/erts/emulator/test/big_SUITE.erl b/erts/emulator/test/big_SUITE.erl
index c5bb228bc3..4b11c4905f 100644
--- a/erts/emulator/test/big_SUITE.erl
+++ b/erts/emulator/test/big_SUITE.erl
@@ -23,7 +23,7 @@
 	 init_per_group/2,end_per_group/2]).
 -export([t_div/1, eq_28/1, eq_32/1, eq_big/1, eq_math/1, big_literals/1,
 	 borders/1, negative/1, big_float_1/1, big_float_2/1,
-         bxor_2pow/1,
+         bxor_2pow/1, band_2pow/1,
 	 shift_limit_1/1, powmod/1, system_limit/1, toobig/1, otp_6692/1]).
 
 %% Internal exports.
@@ -41,7 +41,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}].
 all() -> 
     [t_div, eq_28, eq_32, eq_big, eq_math, big_literals,
      borders, negative, {group, big_float}, shift_limit_1,
-     bxor_2pow,
+     bxor_2pow, band_2pow,
      powmod, system_limit, toobig, otp_6692].
 
 groups() -> 
@@ -476,3 +476,38 @@ my_bxor(A, B, N, Acc0) ->
                false -> Acc0 bor (1 bsl N)
           end,
     my_bxor(A bsr 1, B bsr 1, N+1, Acc1).
+
+
+%% ERL-804
+band_2pow(_Config) ->
+    IL = lists:seq(8*3, 8*16, 4),
+    JL = lists:seq(0, 64),
+    [band_2pow_1((1 bsl I), (1 bsl J))
+     || I <- IL, J <- JL],
+    ok.
+
+band_2pow_1(A, B) ->
+    for(-1,1, fun(Ad) ->
+                      for(-1,1, fun(Bd) ->
+                                        band_2pow_2(A+Ad, B+Bd),
+                                        band_2pow_2(-A+Ad, B+Bd),
+                                        band_2pow_2(A+Ad, -B+Bd),
+                                        band_2pow_2(-A+Ad, -B+Bd)
+                                end)
+              end).
+
+band_2pow_2(A, B) ->
+    Correct = my_band(A, B),
+    case A band B of
+        Correct -> ok;
+        Wrong ->
+            io:format("~.16# band ~.16#\n", [A,B]),
+            io:format("Expected ~.16#\n", [Correct]),
+            io:format("Got      ~.16#\n", [Wrong]),
+            ct:fail({failed, 'band'})
+
+    end.
+
+%% Implement band without band
+my_band(A, B) ->
+    bnot ((bnot A) bor (bnot B)).