aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/bs_instrs.tab
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/bs_instrs.tab')
-rw-r--r--erts/emulator/beam/bs_instrs.tab647
1 files changed, 573 insertions, 74 deletions
diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab
index 61eb02a7a2..9cad2b03c5 100644
--- a/erts/emulator/beam/bs_instrs.tab
+++ b/erts/emulator/beam/bs_instrs.tab
@@ -21,12 +21,57 @@
%if ARCH_64
BS_SAFE_MUL(A, B, Fail, Dst) {
- Uint64 res = ($A) * ($B);
- if (res / $B != $A) {
+ Uint a = $A;
+ Uint b = $B;
+ Uint res;
+#ifdef HAVE_OVERFLOW_CHECK_BUILTINS
+ if (__builtin_mul_overflow(a, b, &res)) {
+ $Fail;
+ }
+#else
+ res = a * b;
+ if (res / b != a) {
$Fail;
}
+#endif
$Dst = res;
}
+
+BS_GET_FIELD_SIZE(Bits, Unit, Fail, Dst) {
+ if (is_small($Bits)) {
+ Uint uint_size;
+ Sint signed_size = signed_val($Bits);
+ if (signed_size < 0) {
+ $Fail;
+ }
+ uint_size = (Uint) signed_size;
+ $BS_SAFE_MUL(uint_size, $Unit, $Fail, $Dst);
+ } else {
+ /*
+ * On a 64-bit architecture, the size of any binary
+ * that would fit in the memory fits in a small.
+ */
+ $Fail;
+ }
+}
+
+BS_GET_UNCHECKED_FIELD_SIZE(Bits, Unit, Fail, Dst) {
+ if (is_small($Bits)) {
+ Uint uint_size;
+ Sint signed_size = signed_val($Bits);
+ if (signed_size < 0) {
+ $Fail;
+ }
+ uint_size = (Uint) signed_size;
+ $Dst = uint_size * $Unit;
+ } else {
+ /*
+ * On a 64-bit architecture, the size of any binary
+ * that would fit in the memory fits in a small.
+ */
+ $Fail;
+ }
+}
%else
BS_SAFE_MUL(A, B, Fail, Dst) {
Uint64 res = (Uint64)($A) * (Uint64)($B);
@@ -35,7 +80,6 @@ BS_SAFE_MUL(A, B, Fail, Dst) {
}
$Dst = res;
}
-%endif
BS_GET_FIELD_SIZE(Bits, Unit, Fail, Dst) {
Sint signed_size;
@@ -76,6 +120,7 @@ BS_GET_UNCHECKED_FIELD_SIZE(Bits, Unit, Fail, Dst) {
}
$Dst = uint_size * $Unit;
}
+%endif
TEST_BIN_VHEAP(VNh, Nh, Live) {
Uint need = $Nh;
@@ -90,32 +135,52 @@ TEST_BIN_VHEAP(VNh, Nh, Live) {
HEAP_SPACE_VERIFIED(need);
}
-i_bs_get_binary_all2(Fail, Ms, Live, Unit, Dst) {
+i_bs_get_binary_all2 := i_bs_get_binary_all2.fetch.execute;
+
+i_bs_get_binary_all2.head() {
+ Eterm context;
+}
+
+i_bs_get_binary_all2.fetch(Ctx) {
+ context = $Ctx;
+}
+
+i_bs_get_binary_all2.execute(Fail, Live, Unit, Dst) {
ErlBinMatchBuffer *_mb;
Eterm _result;
- $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live);
- _mb = ms_matchbuffer($Ms);
+ $GC_TEST_PRESERVE(ERL_SUB_BIN_SIZE, $Live, context);
+ _mb = ms_matchbuffer(context);
if (((_mb->size - _mb->offset) % $Unit) == 0) {
LIGHT_SWAPOUT;
_result = erts_bs_get_binary_all_2(c_p, _mb);
LIGHT_SWAPIN;
HEAP_SPACE_VERIFIED(0);
ASSERT(is_value(_result));
+ $REFRESH_GEN_DEST();
$Dst = _result;
} else {
HEAP_SPACE_VERIFIED(0);
$FAIL($Fail);
}
}
+i_bs_get_binary2 := i_bs_get_binary2.fetch.execute;
-i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) {
+i_bs_get_binary2.head() {
+ Eterm context;
+}
+
+i_bs_get_binary2.fetch(Ctx) {
+ context = $Ctx;
+}
+
+i_bs_get_binary2.execute(Fail, Live, Sz, Flags, Dst) {
ErlBinMatchBuffer *_mb;
Eterm _result;
Uint _size;
$BS_GET_FIELD_SIZE($Sz, (($Flags) >> 3), $FAIL($Fail), _size);
- $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live);
- _mb = ms_matchbuffer($Ms);
+ $GC_TEST_PRESERVE(ERL_SUB_BIN_SIZE, $Live, context);
+ _mb = ms_matchbuffer(context);
LIGHT_SWAPOUT;
_result = erts_bs_get_binary_2(c_p, _size, $Flags, _mb);
LIGHT_SWAPIN;
@@ -123,15 +188,27 @@ i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) {
if (is_non_value(_result)) {
$FAIL($Fail);
} else {
+ $REFRESH_GEN_DEST();
$Dst = _result;
}
}
-i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) {
+i_bs_get_binary_imm2 := i_bs_get_binary_imm2.fetch.execute;
+
+i_bs_get_binary_imm2.head() {
+ Eterm context;
+}
+
+i_bs_get_binary_imm2.fetch(Ctx) {
+ context = $Ctx;
+}
+
+i_bs_get_binary_imm2.execute(Fail, Live, Sz, Flags, Dst) {
ErlBinMatchBuffer *_mb;
Eterm _result;
- $GC_TEST(0, heap_bin_size(ERL_ONHEAP_BIN_LIMIT), $Live);
- _mb = ms_matchbuffer($Ms);
+ $GC_TEST_PRESERVE(heap_bin_size(ERL_ONHEAP_BIN_LIMIT),
+ $Live, context);
+ _mb = ms_matchbuffer(context);
LIGHT_SWAPOUT;
_result = erts_bs_get_binary_2(c_p, $Sz, $Flags, _mb);
LIGHT_SWAPIN;
@@ -139,11 +216,21 @@ i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) {
if (is_non_value(_result)) {
$FAIL($Fail);
} else {
+ $REFRESH_GEN_DEST();
$Dst = _result;
}
}
+i_bs_get_float2 := i_bs_get_float2.fetch.execute;
-i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) {
+i_bs_get_float2.head() {
+ Eterm context;
+}
+
+i_bs_get_float2.fetch(Ctx) {
+ context = $Ctx;
+}
+
+i_bs_get_float2.execute(Fail, Live, Sz, Flags, Dst) {
ErlBinMatchBuffer *_mb;
Eterm _result;
Sint _size;
@@ -152,8 +239,8 @@ i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) {
$FAIL($Fail);
}
_size *= (($Flags) >> 3);
- $GC_TEST(0, FLOAT_SIZE_OBJECT, $Live);
- _mb = ms_matchbuffer($Ms);
+ $GC_TEST_PRESERVE(FLOAT_SIZE_OBJECT, $Live, context);
+ _mb = ms_matchbuffer(context);
LIGHT_SWAPOUT;
_result = erts_bs_get_float_2(c_p, _size, ($Flags), _mb);
LIGHT_SWAPIN;
@@ -161,17 +248,29 @@ i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) {
if (is_non_value(_result)) {
$FAIL($Fail);
} else {
+ $REFRESH_GEN_DEST();
$Dst = _result;
}
}
-i_bs_skip_bits2(Fail, Ms, Bits, Unit) {
+i_bs_skip_bits2 := i_bs_skip_bits2.fetch.execute;
+
+i_bs_skip_bits2.head() {
+ Eterm context, bits;
+}
+
+i_bs_skip_bits2.fetch(Ctx, Bits) {
+ context = $Ctx;
+ bits = $Bits;
+}
+
+i_bs_skip_bits2.execute(Fail, Unit) {
ErlBinMatchBuffer *_mb;
size_t new_offset;
Uint _size;
- _mb = ms_matchbuffer($Ms);
- $BS_GET_FIELD_SIZE($Bits, $Unit, $FAIL($Fail), _size);
+ _mb = ms_matchbuffer(context);
+ $BS_GET_FIELD_SIZE(bits, $Unit, $FAIL($Fail), _size);
new_offset = _mb->offset + _size;
if (new_offset <= _mb->size) {
_mb->offset = new_offset;
@@ -180,16 +279,6 @@ i_bs_skip_bits2(Fail, Ms, Bits, Unit) {
}
}
-i_bs_skip_bits_all2(Fail, Ms, Unit) {
- ErlBinMatchBuffer *_mb;
- _mb = ms_matchbuffer($Ms);
- if (((_mb->size - _mb->offset) % $Unit) == 0) {
- _mb->offset = _mb->size;
- } else {
- $FAIL($Fail);
- }
-}
-
i_bs_skip_bits_imm2(Fail, Ms, Bits) {
ErlBinMatchBuffer *_mb;
size_t new_offset;
@@ -203,15 +292,25 @@ i_bs_skip_bits_imm2(Fail, Ms, Bits) {
}
i_new_bs_put_binary(Fail, Sz, Flags, Src) {
+ Eterm sz = $Sz;
Sint _size;
- $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
+ $BS_GET_UNCHECKED_FIELD_SIZE(sz, (($Flags) >> 3), $BADARG($Fail), _size);
if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2(($Src), _size))) {
$BADARG($Fail);
}
}
+i_new_bs_put_binary_all := i_new_bs_put_binary_all.fetch.execute;
-i_new_bs_put_binary_all(Fail, Src, Unit) {
- if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2(($Src), ($Unit)))) {
+i_new_bs_put_binary_all.head() {
+ Eterm src;
+}
+
+i_new_bs_put_binary_all.fetch(Src) {
+ src = $Src;
+}
+
+i_new_bs_put_binary_all.execute(Fail, Unit) {
+ if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2(src, ($Unit)))) {
$BADARG($Fail);
}
}
@@ -223,9 +322,11 @@ i_new_bs_put_binary_imm(Fail, Sz, Src) {
}
i_new_bs_put_float(Fail, Sz, Flags, Src) {
+ Eterm sz = $Sz;
+ Eterm flags = $Flags;
Sint _size;
- $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
- if (!erts_new_bs_put_float(c_p, ($Src), _size, ($Flags))) {
+ $BS_GET_UNCHECKED_FIELD_SIZE(sz, (flags >> 3), $BADARG($Fail), _size);
+ if (!erts_new_bs_put_float(c_p, ($Src), _size, flags)) {
$BADARG($Fail);
}
}
@@ -237,15 +338,27 @@ i_new_bs_put_float_imm(Fail, Sz, Flags, Src) {
}
i_new_bs_put_integer(Fail, Sz, Flags, Src) {
- Sint _size;
- $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
- if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), _size, ($Flags)))) {
- $BADARG($Fail);
- }
+ Eterm sz = $Sz;
+ Eterm flags = $Flags;
+ Sint _size;
+ $BS_GET_UNCHECKED_FIELD_SIZE(sz, (flags >> 3), $BADARG($Fail), _size);
+ if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), _size, flags))) {
+ $BADARG($Fail);
+ }
+}
+
+i_new_bs_put_integer_imm := i_new_bs_put_integer_imm.fetch.execute;
+
+i_new_bs_put_integer_imm.head() {
+ Eterm src;
}
-i_new_bs_put_integer_imm(Fail, Sz, Flags, Src) {
- if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), ($Sz), ($Flags)))) {
+i_new_bs_put_integer_imm.fetch(Src) {
+ src = $Src;
+}
+
+i_new_bs_put_integer_imm.execute(Fail, Sz, Flags) {
+ if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(src, ($Sz), ($Flags)))) {
$BADARG($Fail);
}
}
@@ -724,26 +837,34 @@ bs_start_match.execute(Fail, Live, Slots, Dst) {
$FAIL($Fail);
}
header = *boxed_val(context);
- slots = $Slots;
+
+ /* Reserve a slot for the start position. */
+ slots = $Slots + 1;
live = $Live;
+
if (header_is_bin_matchstate(header)) {
ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context);
Uint actual_slots = HEADER_NUM_SLOTS(header);
+
+ /* We're not compatible with contexts created by bs_start_match3. */
+ ASSERT(actual_slots >= 1);
+
ms->save_offset[0] = ms->mb.offset;
- if (actual_slots < slots) {
- ErlBinMatchState* dst;
+ if (ERTS_UNLIKELY(actual_slots < slots)) {
+ ErlBinMatchState* expanded;
Uint live = $Live;
Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
-
$GC_TEST_PRESERVE(wordsneeded, live, context);
ms = (ErlBinMatchState *) boxed_val(context);
- dst = (ErlBinMatchState *) HTOP;
- *dst = *ms;
+ expanded = (ErlBinMatchState *) HTOP;
+ *expanded = *ms;
*HTOP = HEADER_BIN_MATCHSTATE(slots);
HTOP += wordsneeded;
HEAP_SPACE_VERIFIED(0);
- $Dst = make_matchstate(dst);
+ context = make_matchstate(expanded);
+ $REFRESH_GEN_DEST();
}
+ $Dst = context;
} else if (is_binary_header(header)) {
Eterm result;
Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
@@ -758,6 +879,7 @@ bs_start_match.execute(Fail, Live, Slots, Dst) {
if (is_non_value(result)) {
$FAIL($Fail);
}
+ $REFRESH_GEN_DEST();
$Dst = result;
} else {
$FAIL($Fail);
@@ -796,9 +918,19 @@ bs_test_unit8(Fail, Ctx) {
}
}
-i_bs_get_integer_8(Ctx, Fail, Dst) {
+i_bs_get_integer_8 := i_bs_get_integer_8.fetch.execute;
+
+i_bs_get_integer_8.head() {
+ Eterm context;
+}
+
+i_bs_get_integer_8.fetch(Ctx) {
+ context = $Ctx;
+}
+
+i_bs_get_integer_8.execute(Fail, Dst) {
Eterm _result;
- ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);
+ ErlBinMatchBuffer* _mb = ms_matchbuffer(context);
if (_mb->size - _mb->offset < 8) {
$FAIL($Fail);
@@ -812,9 +944,19 @@ i_bs_get_integer_8(Ctx, Fail, Dst) {
$Dst = _result;
}
-i_bs_get_integer_16(Ctx, Fail, Dst) {
+i_bs_get_integer_16 := i_bs_get_integer_16.fetch.execute;
+
+i_bs_get_integer_16.head() {
+ Eterm context;
+}
+
+i_bs_get_integer_16.fetch(Ctx) {
+ context = $Ctx;
+}
+
+i_bs_get_integer_16.execute(Fail, Dst) {
Eterm _result;
- ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);
+ ErlBinMatchBuffer* _mb = ms_matchbuffer(context);
if (_mb->size - _mb->offset < 16) {
$FAIL($Fail);
@@ -829,9 +971,19 @@ i_bs_get_integer_16(Ctx, Fail, Dst) {
}
%if ARCH_64
-i_bs_get_integer_32(Ctx, Fail, Dst) {
+i_bs_get_integer_32 := i_bs_get_integer_32.fetch.execute;
+
+i_bs_get_integer_32.head() {
+ Eterm context;
+}
+
+i_bs_get_integer_32.fetch(Ctx) {
+ context = $Ctx;
+}
+
+i_bs_get_integer_32.execute(Fail, Dst) {
Uint32 _integer;
- ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);
+ ErlBinMatchBuffer* _mb = ms_matchbuffer(context);
if (_mb->size - _mb->offset < 32) {
$FAIL($Fail);
@@ -881,15 +1033,23 @@ bs_get_integer.execute(Fail, Flags, Dst) {
$Dst = result;
}
-i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) {
+i_bs_get_integer := i_bs_get_integer.fetch.execute;
+
+i_bs_get_integer.head() {
+ Eterm context;
+}
+
+i_bs_get_integer.fetch(Ctx) {
+ context = $Ctx;
+}
+
+i_bs_get_integer.execute(Fail, Live, FlagsAndUnit, Sz, Dst) {
Uint flags;
Uint size;
- Eterm ms;
ErlBinMatchBuffer* mb;
Eterm result;
flags = $FlagsAndUnit;
- ms = $Ms;
$BS_GET_FIELD_SIZE($Sz, (flags >> 3), $FAIL($Fail), size);
if (size >= SMALL_BITS) {
Uint wordsneeded;
@@ -900,14 +1060,15 @@ i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) {
* Remember to re-acquire the matchbuffer after gc.
*/
- mb = ms_matchbuffer(ms);
+ mb = ms_matchbuffer(context);
if (mb->size - mb->offset < size) {
$FAIL($Fail);
}
wordsneeded = 1+WSIZE(NBYTES((Uint) size));
- $GC_TEST_PRESERVE(wordsneeded, $Live, ms);
+ $GC_TEST_PRESERVE(wordsneeded, $Live, context);
+ $REFRESH_GEN_DEST();
}
- mb = ms_matchbuffer(ms);
+ mb = ms_matchbuffer(context);
LIGHT_SWAPOUT;
result = erts_bs_get_integer_2(c_p, size, flags, mb);
LIGHT_SWAPIN;
@@ -918,9 +1079,19 @@ i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) {
$Dst = result;
}
-i_bs_get_utf8(Ctx, Fail, Dst) {
+i_bs_get_utf8 := i_bs_get_utf8.fetch.execute;
+
+i_bs_get_utf8.head() {
+ Eterm context;
+}
+
+i_bs_get_utf8.fetch(Ctx) {
+ context = $Ctx;
+}
+
+i_bs_get_utf8.execute(Fail, Dst) {
Eterm result;
- ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx);
+ ErlBinMatchBuffer* mb = ms_matchbuffer(context);
if (mb->size - mb->offset < 8) {
$FAIL($Fail);
@@ -939,21 +1110,32 @@ i_bs_get_utf8(Ctx, Fail, Dst) {
if (is_non_value(result)) {
$FAIL($Fail);
}
+ $REFRESH_GEN_DEST();
$Dst = result;
}
-i_bs_get_utf16(Ctx, Fail, Flags, Dst) {
- ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx);
+i_bs_get_utf16 := i_bs_get_utf16.fetch.execute;
+
+i_bs_get_utf16.head() {
+ Eterm context;
+}
+
+i_bs_get_utf16.fetch(Ctx) {
+ context = $Ctx;
+}
+
+i_bs_get_utf16.execute(Fail, Flags, Dst) {
+ ErlBinMatchBuffer* mb = ms_matchbuffer(context);
Eterm result = erts_bs_get_utf16(mb, $Flags);
if (is_non_value(result)) {
$FAIL($Fail);
}
+ $REFRESH_GEN_DEST();
$Dst = result;
}
bs_context_to_binary := ctx_to_bin.fetch.execute;
-i_bs_get_binary_all_reuse := ctx_to_bin.fetch_bin.execute;
ctx_to_bin.head() {
Eterm context;
@@ -976,16 +1158,6 @@ ctx_to_bin.fetch(Src) {
}
}
-ctx_to_bin.fetch_bin(Src, Fail, Unit) {
- context = $Src;
- mb = ms_matchbuffer(context);
- size = mb->size - mb->offset;
- if (size % $Unit != 0) {
- $FAIL($Fail);
- }
- offs = mb->offset;
-}
-
ctx_to_bin.execute() {
Uint hole_size;
Uint orig = mb->orig;
@@ -1029,10 +1201,337 @@ i_bs_match_string(Ctx, Fail, Bits, Ptr) {
i_bs_save2(Src, Slot) {
ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src);
+ ASSERT(HEADER_NUM_SLOTS(_ms->thing_word) > $Slot);
_ms->save_offset[$Slot] = _ms->mb.offset;
}
i_bs_restore2(Src, Slot) {
ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src);
+ ASSERT(HEADER_NUM_SLOTS(_ms->thing_word) > $Slot);
_ms->mb.offset = _ms->save_offset[$Slot];
}
+
+bs_get_tail := bs_get_tail.fetch.execute;
+
+bs_get_tail.head() {
+ Eterm context;
+}
+
+bs_get_tail.fetch(Src) {
+ context = $Src;
+}
+
+bs_get_tail.execute(Dst, Live) {
+ ErlBinMatchBuffer* mb;
+ Uint size, offs;
+ ErlSubBin* sb;
+
+ ASSERT(header_is_bin_matchstate(*boxed_val(context)));
+
+ $GC_TEST_PRESERVE(ERL_SUB_BIN_SIZE, $Live, context);
+
+ mb = ms_matchbuffer(context);
+
+ offs = mb->offset;
+ size = mb->size - offs;
+
+ sb = (ErlSubBin *) HTOP;
+ HTOP += ERL_SUB_BIN_SIZE;
+
+ sb->thing_word = HEADER_SUB_BIN;
+ sb->size = BYTE_OFFSET(size);
+ sb->bitsize = BIT_OFFSET(size);
+ sb->offs = BYTE_OFFSET(offs);
+ sb->bitoffs = BIT_OFFSET(offs);
+ sb->is_writable = 0;
+ sb->orig = mb->orig;
+
+ $REFRESH_GEN_DEST();
+ $Dst = make_binary(sb);
+}
+
+
+%if ARCH_64
+
+i_bs_start_match3_gp := i_bs_start_match3_gp.fetch.execute;
+
+i_bs_start_match3_gp.head() {
+ Eterm context;
+}
+
+i_bs_start_match3_gp.fetch(Src) {
+ context = $Src;
+}
+
+i_bs_start_match3_gp.execute(Live, Fail, Dst, Pos) {
+ Eterm header;
+ Uint position, live;
+
+ live = $Live;
+
+ if (!is_boxed(context)) {
+ $FAIL($Fail);
+ }
+
+ header = *boxed_val(context);
+
+ if (header_is_bin_matchstate(header)) {
+ ErlBinMatchBuffer *mb;
+
+ ASSERT(HEADER_NUM_SLOTS(header) == 0);
+
+ mb = ms_matchbuffer(context);
+ position = mb->offset;
+
+ $Dst = context;
+ } else if (is_binary_header(header)) {
+ ErlBinMatchState *ms;
+
+ $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(0), live, context);
+ HEAP_TOP(c_p) = HTOP;
+#ifdef DEBUG
+ c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */
+#endif
+ ms = erts_bs_start_match_3(c_p, context);
+ HTOP = HEAP_TOP(c_p);
+ HEAP_SPACE_VERIFIED(0);
+
+ if (ms == NULL) {
+ $FAIL($Fail);
+ }
+
+ $REFRESH_GEN_DEST();
+ $Dst = make_matchstate(ms);
+ position = ms->mb.offset;
+ } else {
+ $FAIL($Fail);
+ }
+
+ ASSERT(IS_USMALL(0, position));
+ $Pos = make_small(position);
+}
+
+i_bs_start_match3 := i_bs_start_match3.fetch.execute;
+
+i_bs_start_match3.head() {
+ Eterm context;
+}
+
+i_bs_start_match3.fetch(Src) {
+ context = $Src;
+}
+
+i_bs_start_match3.execute(Live, Fail, Dst) {
+ Eterm header;
+ Uint live;
+
+ live = $Live;
+
+ if (!is_boxed(context)) {
+ $FAIL($Fail);
+ }
+
+ header = *boxed_val(context);
+
+ if (header_is_bin_matchstate(header)) {
+ ASSERT(HEADER_NUM_SLOTS(header) == 0);
+ $Dst = context;
+ } else if (is_binary_header(header)) {
+ ErlBinMatchState *ms;
+
+ $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(0), live, context);
+ HEAP_TOP(c_p) = HTOP;
+#ifdef DEBUG
+ c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */
+#endif
+ ms = erts_bs_start_match_3(c_p, context);
+ HTOP = HEAP_TOP(c_p);
+ HEAP_SPACE_VERIFIED(0);
+
+ if (ms == NULL) {
+ $FAIL($Fail);
+ }
+
+ $REFRESH_GEN_DEST();
+ $Dst = make_matchstate(ms);
+ } else {
+ $FAIL($Fail);
+ }
+}
+
+bs_set_position(Ctx, Pos) {
+ ErlBinMatchBuffer* mb;
+ Eterm context;
+
+ context = $Ctx;
+ ASSERT(header_is_bin_matchstate(*boxed_val(context)));
+
+ mb = ms_matchbuffer(context);
+ mb->offset = unsigned_val($Pos);
+}
+
+i_bs_get_position(Ctx, Dst) {
+ ErlBinMatchBuffer* mb;
+ Eterm context;
+
+ context = $Ctx;
+ ASSERT(header_is_bin_matchstate(*boxed_val(context)));
+
+ mb = ms_matchbuffer(context);
+ $Dst = make_small(mb->offset);
+}
+
+%else
+
+#
+# Unlike their 64-bit counterparts, the 32-bit position instructions operate on
+# an offset from the "base position" of the context because storing raw
+# positions would lead to the creation of far too many bigints.
+#
+# When a match context is reused we check whether its position fits into an
+# immediate, and create a new match context if it does not. This means we only
+# have to allocate stuff roughly once every 16MB rather than every time we
+# match at a position beyond 16MB.
+#
+
+bs_set_position := bs_set_position.fetch.execute;
+
+bs_set_position.head() {
+ Eterm context, position;
+}
+
+bs_set_position.fetch(Ctx, Pos) {
+ context = $Ctx;
+ position = $Pos;
+}
+
+bs_set_position.execute() {
+ ErlBinMatchState *ms;
+
+ ASSERT(header_is_bin_matchstate(*boxed_val(context)));
+ ms = (ErlBinMatchState*)boxed_val(context);
+
+ if (ERTS_LIKELY(is_small(position))) {
+ ms->mb.offset = ms->save_offset[0] + unsigned_val(position);
+ } else {
+ ASSERT(is_big(position));
+ ms->mb.offset = ms->save_offset[0] + *BIG_V(big_val(position));
+ }
+}
+
+bs_get_position := bs_get_position.fetch.execute;
+
+bs_get_position.head() {
+ Eterm context;
+}
+
+bs_get_position.fetch(Ctx) {
+ context = $Ctx;
+}
+
+bs_get_position.execute(Dst, Live) {
+ ErlBinMatchState *ms;
+ Uint position;
+
+ ASSERT(header_is_bin_matchstate(*boxed_val(context)));
+ ms = (ErlBinMatchState*)boxed_val(context);
+
+ position = ms->mb.offset - ms->save_offset[0];
+
+ if (ERTS_LIKELY(IS_USMALL(0, position))) {
+ $Dst = make_small(position);
+ } else {
+ Eterm *hp;
+
+ $GC_TEST_PRESERVE(BIG_UINT_HEAP_SIZE, $Live, context);
+
+ hp = HTOP;
+ HTOP += BIG_UINT_HEAP_SIZE;
+
+ *hp = make_pos_bignum_header(1);
+ BIG_DIGIT(hp, 0) = position;
+
+ $REFRESH_GEN_DEST();
+ $Dst = make_big(hp);
+ }
+}
+
+i_bs_start_match3 := i_bs_start_match3.fetch.execute;
+
+i_bs_start_match3.head() {
+ Eterm context;
+}
+
+i_bs_start_match3.fetch(Src) {
+ context = $Src;
+}
+
+i_bs_start_match3.execute(Live, Fail, Dst) {
+ Eterm header;
+ Uint live;
+
+ live = $Live;
+
+ if (!is_boxed(context)) {
+ $FAIL($Fail);
+ }
+
+ header = *boxed_val(context);
+
+ if (header_is_bin_matchstate(header)) {
+ ErlBinMatchState *current_ms;
+ Uint position;
+
+ ASSERT(HEADER_NUM_SLOTS(header) == 1);
+
+ current_ms = (ErlBinMatchState*)boxed_val(context);
+ position = current_ms->mb.offset - current_ms->save_offset[0];
+
+ if (ERTS_LIKELY(IS_USMALL(0, position))) {
+ $Dst = context;
+ } else {
+ ErlBinMatchState *new_ms;
+
+ $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(1), live, context);
+ current_ms = (ErlBinMatchState*)boxed_val(context);
+
+ new_ms = (ErlBinMatchState*)HTOP;
+ HTOP += ERL_BIN_MATCHSTATE_SIZE(1);
+
+ new_ms->thing_word = HEADER_BIN_MATCHSTATE(1);
+ new_ms->save_offset[0] = current_ms->mb.offset;
+ new_ms->mb = current_ms->mb;
+
+ $REFRESH_GEN_DEST();
+ $Dst = make_matchstate(new_ms);
+ }
+ } else if (is_binary_header(header)) {
+ Eterm result;
+
+ $GC_TEST_PRESERVE(ERL_BIN_MATCHSTATE_SIZE(1), live, context);
+ HEAP_TOP(c_p) = HTOP;
+
+#ifdef DEBUG
+ c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */
+#endif
+
+ /* We intentionally use erts_bs_start_match_2 so that we can use
+ * save_offset as a base for all saved positions on this context,
+ * allowing us to avoid bigints for much longer. */
+ result = erts_bs_start_match_2(c_p, context, 1);
+
+ HTOP = HEAP_TOP(c_p);
+ HEAP_SPACE_VERIFIED(0);
+
+ if (is_non_value(result)) {
+ $FAIL($Fail);
+ }
+
+ $REFRESH_GEN_DEST();
+ $Dst = result;
+ } else {
+ $FAIL($Fail);
+ }
+}
+
+%endif