path: root/erts/emulator/beam/bs_instrs.tab

// -*- c -*-
//
// %CopyrightBegin%
//
// Copyright Ericsson AB 2017. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// %CopyrightEnd%
//

%if ARCH_64
BS_SAFE_MUL(A, B, Fail, Dst) {
    Uint64 res = ($A) * ($B);
    if (res / $B != $A) {
        $Fail;
    }
    $Dst = res;
}
%else
BS_SAFE_MUL(A, B, Fail, Dst) {
    Uint64 res = (Uint64)($A) * (Uint64)($B);
    if ((res >> (8*sizeof(Uint))) != 0) {
        $Fail;
    }
    $Dst = res;
}
%endif

BS_GET_FIELD_SIZE(Bits, Unit, Fail, Dst) {
    Sint signed_size;
    Uint uint_size;
    Uint temp_bits;

    if (is_small($Bits)) {
        signed_size = signed_val($Bits);
        if (signed_size < 0) {
            $Fail;
        }
        uint_size = (Uint) signed_size;
    } else {
        if (!term_to_Uint($Bits, &temp_bits)) {
            $Fail;
        }
        uint_size = temp_bits;
    }
    $BS_SAFE_MUL(uint_size, $Unit, $Fail, $Dst);
}

BS_GET_UNCHECKED_FIELD_SIZE(Bits, Unit, Fail, Dst) {
    Sint signed_size;
    Uint uint_size;
    Uint temp_bits;

    if (is_small($Bits)) {
        signed_size = signed_val($Bits);
        if (signed_size < 0) {
            $Fail;
        }
        uint_size = (Uint) signed_size;
    } else {
        if (!term_to_Uint($Bits, &temp_bits)) {
            $Fail;
        }
        uint_size = temp_bits;
    }
    $Dst = uint_size * $Unit;
}

TEST_BIN_VHEAP(VNh, Nh, Live) {
    Uint need = $Nh;
    if (E - HTOP < need || MSO(c_p).overhead + $VNh >= BIN_VHEAP_SZ(c_p)) {
        SWAPOUT;
        PROCESS_MAIN_CHK_LOCKS(c_p);
        FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS);
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
        PROCESS_MAIN_CHK_LOCKS(c_p);
        SWAPIN;
    }
    HEAP_SPACE_VERIFIED(need);
}

i_bs_get_binary_all2(Fail, Ms, Live, Unit, Dst) {
    ErlBinMatchBuffer *_mb;
    Eterm _result;

    $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live);
    _mb = ms_matchbuffer($Ms);
    if (((_mb->size - _mb->offset) % $Unit) == 0) {
        LIGHT_SWAPOUT;
        _result = erts_bs_get_binary_all_2(c_p, _mb);
        LIGHT_SWAPIN;
        HEAP_SPACE_VERIFIED(0);
        ASSERT(is_value(_result));
        $Dst = _result;
    } else {
	HEAP_SPACE_VERIFIED(0);
	$FAIL($Fail);
    }
}

i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) {
    ErlBinMatchBuffer *_mb;
    Eterm _result;
    Uint _size;
    $BS_GET_FIELD_SIZE($Sz, (($Flags) >> 3), $FAIL($Fail), _size);
    $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live);
    _mb = ms_matchbuffer($Ms);
    LIGHT_SWAPOUT;
    _result = erts_bs_get_binary_2(c_p, _size, $Flags, _mb);
    LIGHT_SWAPIN;
    HEAP_SPACE_VERIFIED(0);
    if (is_non_value(_result)) {
        $FAIL($Fail);
    } else {
        $Dst = _result;
    }
}

i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) {
    ErlBinMatchBuffer *_mb;
    Eterm _result;
    $GC_TEST(0, heap_bin_size(ERL_ONHEAP_BIN_LIMIT), $Live);
    _mb = ms_matchbuffer($Ms);
    LIGHT_SWAPOUT;
    _result = erts_bs_get_binary_2(c_p, $Sz, $Flags, _mb);
    LIGHT_SWAPIN;
    HEAP_SPACE_VERIFIED(0);
    if (is_non_value(_result)) {
        $FAIL($Fail);
    } else {
        $Dst = _result;
    }
}

i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) {
    ErlBinMatchBuffer *_mb;
    Eterm _result;
    Sint _size;

    if (!is_small($Sz) || (_size = unsigned_val($Sz)) > 64) {
        $FAIL($Fail);
    }
    _size *= (($Flags) >> 3);
    $GC_TEST(0, FLOAT_SIZE_OBJECT, $Live);
    _mb = ms_matchbuffer($Ms);
    LIGHT_SWAPOUT;
    _result = erts_bs_get_float_2(c_p, _size, ($Flags), _mb);
    LIGHT_SWAPIN;
    HEAP_SPACE_VERIFIED(0);
    if (is_non_value(_result)) {
        $FAIL($Fail);
    } else {
        $Dst = _result;
    }
}

i_bs_skip_bits2(Fail, Ms, Bits, Unit) {
    ErlBinMatchBuffer *_mb;
    size_t new_offset;
    Uint _size;

    _mb = ms_matchbuffer($Ms);
    $BS_GET_FIELD_SIZE($Bits, $Unit, $FAIL($Fail), _size);
    new_offset = _mb->offset + _size;
    if (new_offset <= _mb->size) {
        _mb->offset = new_offset;
    } else {
        $FAIL($Fail);
    }
}

i_bs_skip_bits_all2(Fail, Ms, Unit) {
    ErlBinMatchBuffer *_mb;
    _mb = ms_matchbuffer($Ms);
    if (((_mb->size - _mb->offset) % $Unit) == 0) {
        _mb->offset = _mb->size;
    } else {
        $FAIL($Fail);
    }
}

i_bs_skip_bits_imm2(Fail, Ms, Bits) {
    ErlBinMatchBuffer *_mb;
    size_t new_offset;
    _mb = ms_matchbuffer($Ms);
    new_offset = _mb->offset + ($Bits);
    if (new_offset <= _mb->size) {
        _mb->offset = new_offset;
    } else {
        $FAIL($Fail);
    }
}

i_new_bs_put_binary(Fail, Sz, Flags, Src) {
    Sint _size;
    $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
    if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2(($Src), _size))) {
        $BADARG($Fail);
    }
}

i_new_bs_put_binary_all(Fail, Src, Unit) {
    if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2(($Src), ($Unit)))) {
        $BADARG($Fail);
    }
}

i_new_bs_put_binary_imm(Fail, Sz, Src) {
    if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2(($Src), ($Sz)))) {
        $BADARG($Fail);
    }
}

i_new_bs_put_float(Fail, Sz, Flags, Src) {
    Sint _size;
    $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
    if (!erts_new_bs_put_float(c_p, ($Src), _size, ($Flags))) {
        $BADARG($Fail);
    }
}

i_new_bs_put_float_imm(Fail, Sz, Flags, Src) {
    if (!erts_new_bs_put_float(c_p, ($Src), ($Sz), ($Flags))) {
        $BADARG($Fail);
    }
}

i_new_bs_put_integer(Fail, Sz, Flags, Src) {
   Sint _size;
   $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
   if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), _size, ($Flags)))) {
       $BADARG($Fail);
   }
}

i_new_bs_put_integer_imm(Fail, Sz, Flags, Src) {
    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), ($Sz), ($Flags)))) {
        $BADARG($Fail);
    }
}

#
# i_bs_init*
#

i_bs_init_fail_heap := bs_init.fail_heap.verify.execute;
i_bs_init_fail := bs_init.fail.verify.execute;
i_bs_init := bs_init.plain.execute;
i_bs_init_heap := bs_init.heap.execute;

bs_init.head() {
    Eterm BsOp1;
    Eterm BsOp2;
}

bs_init.fail_heap(Size, HeapAlloc) {
    BsOp1 = $Size;
    BsOp2 = $HeapAlloc;
}

bs_init.fail(Size) {
    BsOp1 = $Size;
    BsOp2 = 0;
}

bs_init.plain(Size) {
    BsOp1 = $Size;
    BsOp2 = 0;
}

bs_init.heap(Size, HeapAlloc) {
    BsOp1 = $Size;
    BsOp2 = $HeapAlloc;
}

bs_init.verify(Fail) {
    if (is_small(BsOp1)) {
        Sint size = signed_val(BsOp1);
        if (size < 0) {
            $BADARG($Fail);
        }
        BsOp1 = (Eterm) size;
    } else {
        Uint bytes;

        if (!term_to_Uint(BsOp1, &bytes)) {
            c_p->freason = bytes;
            $FAIL_HEAD_OR_BODY($Fail);
        }
        if ((bytes >> (8*sizeof(Uint)-3)) != 0) {
            $SYSTEM_LIMIT($Fail);
        }
        BsOp1 = (Eterm) bytes;
    }
}

bs_init.execute(Live, Dst) {
    if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) {
        ErlHeapBin* hb;
        Uint bin_need;

        bin_need = heap_bin_size(BsOp1);
        erts_bin_offset = 0;
        erts_writable_bin = 0;
        $GC_TEST(0, bin_need+BsOp2+ERL_SUB_BIN_SIZE, $Live);
        hb = (ErlHeapBin *) HTOP;
        HTOP += bin_need;
        hb->thing_word = header_heap_bin(BsOp1);
        hb->size = BsOp1;
        erts_current_bin = (byte *) hb->data;
        $Dst = make_binary(hb);
    } else {
        Binary* bptr;
        ProcBin* pb;

        erts_bin_offset = 0;
        erts_writable_bin = 0;
        $TEST_BIN_VHEAP(BsOp1 / sizeof(Eterm),
                        BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, $Live);

        /*
         * Allocate the binary struct itself.
         */
        bptr = erts_bin_nrml_alloc(BsOp1);
        erts_current_bin = (byte *) bptr->orig_bytes;

        /*
         * Now allocate the ProcBin on the heap.
         */
        pb = (ProcBin *) HTOP;
        HTOP += PROC_BIN_SIZE;
        pb->thing_word = HEADER_PROC_BIN;
        pb->size = BsOp1;
        pb->next = MSO(c_p).first;
        MSO(c_p).first = (struct erl_off_heap_header*) pb;
        pb->val = bptr;
        pb->bytes = (byte*) bptr->orig_bytes;
        pb->flags = 0;

        OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm));

        $Dst = make_binary(pb);
    }
}

#
# i_bs_init_bits*
#

i_bs_init_bits           := bs_init_bits.plain.execute;
i_bs_init_bits_heap      := bs_init_bits.heap.execute;
i_bs_init_bits_fail      := bs_init_bits.fail.verify.execute;
i_bs_init_bits_fail_heap := bs_init_bits.fail_heap.verify.execute;

bs_init_bits.head() {
    Eterm new_binary;
    Eterm num_bits_term;
    Uint num_bits;
    Uint alloc;
    Uint num_bytes;
}

bs_init_bits.plain(NumBits) {
    num_bits = $NumBits;
    alloc = 0;
}

bs_init_bits.heap(NumBits, Alloc) {
    num_bits = $NumBits;
    alloc = $Alloc;
}

bs_init_bits.fail(NumBitsTerm) {
    num_bits_term = $NumBitsTerm;
    alloc = 0;
}

bs_init_bits.fail_heap(NumBitsTerm, Alloc) {
    num_bits_term = $NumBitsTerm;
    alloc = $Alloc;
}

bs_init_bits.verify(Fail) {
    if (is_small(num_bits_term)) {
        Sint size = signed_val(num_bits_term);
        if (size < 0) {
            $BADARG($Fail);
        }
        num_bits = (Uint) size;
    } else {
        Uint bits;

        if (!term_to_Uint(num_bits_term, &bits)) {
            c_p->freason = bits;
            $FAIL_HEAD_OR_BODY($Fail);
        }
        num_bits = (Uint) bits;
    }
}

bs_init_bits.execute(Live, Dst) {
     num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3;
     if (num_bits & 7) {
	 alloc += ERL_SUB_BIN_SIZE;
     }
     if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) {
	 alloc += heap_bin_size(num_bytes);
     } else {
	 alloc += PROC_BIN_SIZE;
     }
     $test_heap(alloc, $Live);

     /* num_bits = Number of bits to build
      * num_bytes = Number of bytes to allocate in the binary
      * alloc = Total number of words to allocate on heap
      * Operands: NotUsed NotUsed Dst
      */
     if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) {
	 ErlHeapBin* hb;

	 erts_bin_offset = 0;
	 erts_writable_bin = 0;
	 hb = (ErlHeapBin *) HTOP;
	 HTOP += heap_bin_size(num_bytes);
	 hb->thing_word = header_heap_bin(num_bytes);
	 hb->size = num_bytes;
	 erts_current_bin = (byte *) hb->data;
	 new_binary = make_binary(hb);

     do_bits_sub_bin:
	 if (num_bits & 7) {
	     ErlSubBin* sb;

	     sb = (ErlSubBin *) HTOP;
	     HTOP += ERL_SUB_BIN_SIZE;
	     sb->thing_word = HEADER_SUB_BIN;
	     sb->size = num_bytes - 1;
	     sb->bitsize = num_bits & 7;
	     sb->offs = 0;
	     sb->bitoffs = 0;
	     sb->is_writable = 0;
	     sb->orig = new_binary;
	     new_binary = make_binary(sb);
	 }
	 HEAP_SPACE_VERIFIED(0);
         $Dst = new_binary;
     } else {
	 Binary* bptr;
	 ProcBin* pb;

	 erts_bin_offset = 0;
	 erts_writable_bin = 0;

	 /*
	  * Allocate the binary struct itself.
	  */
	 bptr = erts_bin_nrml_alloc(num_bytes);
	 erts_current_bin = (byte *) bptr->orig_bytes;

	 /*
	  * Now allocate the ProcBin on the heap.
	  */
	 pb = (ProcBin *) HTOP;
	 HTOP += PROC_BIN_SIZE;
	 pb->thing_word = HEADER_PROC_BIN;
	 pb->size = num_bytes;
	 pb->next = MSO(c_p).first;
	 MSO(c_p).first = (struct erl_off_heap_header*) pb;
	 pb->val = bptr;
	 pb->bytes = (byte*) bptr->orig_bytes;
	 pb->flags = 0;
	 OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
	 new_binary = make_binary(pb);
	 goto do_bits_sub_bin;
     }
}

bs_add(Fail, Src1, Src2, Unit, Dst) {
    Eterm Op1 = $Src1;
    Eterm Op2 = $Src2;
    Uint unit = $Unit;

    if (is_both_small(Op1, Op2)) {
        Sint Arg1 = signed_val(Op1);
        Sint Arg2 = signed_val(Op2);

        if (Arg1 >= 0 && Arg2 >= 0) {
            $BS_SAFE_MUL(Arg2, unit, $SYSTEM_LIMIT($Fail), Op1);
            Op1 += Arg1;

        store_bs_add_result:
            if (Op1 <= MAX_SMALL) {
                Op1 = make_small(Op1);
            } else {
                /*
                 * May generate a heap fragment, but in this
                 * particular case it is OK, since the value will be
                 * stored into an x register (the GC will scan x
                 * registers for references to heap fragments) and
                 * there is no risk that value can be stored into a
                 * location that is not scanned for heap-fragment
                 * references (such as the heap).
                 */
                SWAPOUT;
                Op1 = erts_make_integer(Op1, c_p);
                HTOP = HEAP_TOP(c_p);
            }
            $Dst = Op1;
            $NEXT0();
        }
        $BADARG($Fail);
    } else {
        Uint a;
        Uint b;
        Uint c;

        /*
         * Now we know that one of the arguments is
         * not a small. We must convert both arguments
         * to Uints and check for errors at the same time.
         *
         * Error checking is tricky.
         *
         * If one of the arguments is not numeric or
         * not positive, the error reason is BADARG.
         *
         * Otherwise if both arguments are numeric,
         * but at least one argument does not fit in
         * an Uint, the reason is SYSTEM_LIMIT.
         */

        if (!term_to_Uint(Op1, &a)) {
            if (a == BADARG) {
                $BADARG($Fail);
            }
            if (!term_to_Uint(Op2, &b)) {
                c_p->freason = b;
                $FAIL_HEAD_OR_BODY($Fail);
            }
            $SYSTEM_LIMIT($Fail);
        } else if (!term_to_Uint(Op2, &b)) {
            c_p->freason = b;
            $FAIL_HEAD_OR_BODY($Fail);
        }

        /*
         * The arguments are now correct and stored in a and b.
         */

        $BS_SAFE_MUL(b, unit, $SYSTEM_LIMIT($Fail), c);
        Op1 = a + c;
        if (Op1 < a) {
            /*
             * If the result is less than one of the
             * arguments, there must have been an overflow.
             */
            $SYSTEM_LIMIT($Fail);
        }
        goto store_bs_add_result;
    }
    /* No fallthrough */
    ASSERT(0);
}

bs_put_string(Len, Ptr) {
    erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) $Ptr, $Len));
}

i_bs_append(Fail, ExtraHeap, Live, Unit, Size, Dst) {
    Uint live = $Live;
    Uint res;

    HEAVY_SWAPOUT;
    reg[live] = x(SCRATCH_X_REG);
    res = erts_bs_append(c_p, reg, live, $Size, $ExtraHeap, $Unit);
    HEAVY_SWAPIN;
    if (is_non_value(res)) {
        /* c_p->freason is already set (to BADARG or SYSTEM_LIMIT). */
        $FAIL_HEAD_OR_BODY($Fail);
    }
    $Dst = res;
}

i_bs_private_append(Fail, Unit, Size, Src, Dst) {
    Eterm res;

    res = erts_bs_private_append(c_p, $Src, $Size, $Unit);
    if (is_non_value(res)) {
        /* c_p->freason is already set (to BADARG or SYSTEM_LIMIT). */
        $FAIL_HEAD_OR_BODY($Fail);
    }
    $Dst = res;
}

bs_init_writable() {
    HEAVY_SWAPOUT;
    r(0) = erts_bs_init_writable(c_p, r(0));
    HEAVY_SWAPIN;
}

i_bs_utf8_size(Src, Dst) {
    Eterm arg = $Src;
    Eterm result;

    /*
     * Calculate the number of bytes needed to encode the source
     * operand to UTF-8. If the source operand is invalid (e.g. wrong
     * type or range) we return a nonsense integer result (0 or 4). We
     * can get away with that because we KNOW that bs_put_utf8 will do
     * full error checking.
     */

    if (arg < make_small(0x80UL)) {
        result = make_small(1);
    } else if (arg < make_small(0x800UL)) {
        result = make_small(2);
    } else if (arg < make_small(0x10000UL)) {
        result = make_small(3);
    } else {
        result = make_small(4);
    }
    $Dst = result;
}

i_bs_put_utf8(Fail, Src) {
    if (!erts_bs_put_utf8(ERL_BITS_ARGS_1($Src))) {
        $BADARG($Fail);
    }
}

i_bs_utf16_size(Src, Dst) {
    Eterm arg = $Src;
    Eterm result = make_small(2);

    /*
     * Calculate the number of bytes needed to encode the source
     * operarand to UTF-16. If the source operand is invalid (e.g. wrong
     * type or range) we return a nonsense integer result (2 or 4). We
     * can get away with that because we KNOW that bs_put_utf16 will do
     * full error checking.
     */

    if (arg >= make_small(0x10000UL)) {
        result = make_small(4);
    }
    $Dst = result;
}

bs_put_utf16(Fail, Flags, Src) {
    if (!erts_bs_put_utf16(ERL_BITS_ARGS_2($Src, $Flags))) {
        $BADARG($Fail);
    }
}

// Validate a value about to be stored in a binary.
i_bs_validate_unicode(Fail, Src) {
    Eterm val = $Src;

    /*
     * There is no need to untag the integer, but it IS necessary
     * to make sure it is small (if the term is a bignum, it could
     * slip through the test, and there is no further test that
     * would catch it, since bit syntax construction silently masks
     * too big numbers).
     */
    if (is_not_small(val) || val > make_small(0x10FFFFUL) ||
        (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) {
        $BADARG($Fail);
    }
}

// Validate a value that has been matched out.
i_bs_validate_unicode_retract(Fail, Src, Ms) {
    /*
     * There is no need to untag the integer, but it IS necessary
     * to make sure it is small (a bignum pointer could fall in
     * the valid range).
     */

    Eterm i = $Src;
    if (is_not_small(i) || i > make_small(0x10FFFFUL) ||
        (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) {
        Eterm ms = $Ms;		/* Match context */
        ErlBinMatchBuffer* mb;

        /* Invalid value. Retract the position in the binary. */
        mb = ms_matchbuffer(ms);
        mb->offset -= 32;
        $BADARG($Fail);
    }
}


//
// Matching of binaries.
//

i_bs_start_match2 := bs_start_match.fetch.execute;

bs_start_match.head() {
    Uint slots;
    Uint live;
    Eterm header;
    Eterm context;
}

bs_start_match.fetch(Src) {
    context = $Src;
}

bs_start_match.execute(Fail, Live, Slots, Dst) {
    if (!is_boxed(context)) {
        $FAIL($Fail);
    }
    header = *boxed_val(context);
    slots = $Slots;
    live = $Live;
    if (header_is_bin_matchstate(header)) {
        ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context);
        Uint actual_slots = HEADER_NUM_SLOTS(header);
        ms->save_offset[0] = ms->mb.offset;
        if (actual_slots < slots) {
            ErlBinMatchState* dst;
            Uint live = $Live;
            Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);

            $GC_TEST_PRESERVE(wordsneeded, live, context);
            ms = (ErlBinMatchState *) boxed_val(context);
            dst = (ErlBinMatchState *) HTOP;
            *dst = *ms;
            *HTOP = HEADER_BIN_MATCHSTATE(slots);
            HTOP += wordsneeded;
            HEAP_SPACE_VERIFIED(0);
            $Dst = make_matchstate(dst);
        }
    } else if (is_binary_header(header)) {
        Eterm result;
        Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
        $GC_TEST_PRESERVE(wordsneeded, live, context);
        HEAP_TOP(c_p) = HTOP;
#ifdef DEBUG
        c_p->stop = E;	/* Needed for checking in HeapOnlyAlloc(). */
#endif
        result = erts_bs_start_match_2(c_p, context, slots);
        HTOP = HEAP_TOP(c_p);
        HEAP_SPACE_VERIFIED(0);
        if (is_non_value(result)) {
            $FAIL($Fail);
        }
        $Dst = result;
    } else {
        $FAIL($Fail);
    }
}

bs_test_zero_tail2(Fail, Ctx) {
    ErlBinMatchBuffer *_mb;
    _mb = (ErlBinMatchBuffer*) ms_matchbuffer($Ctx);
    if (_mb->size != _mb->offset) {
        $FAIL($Fail);
    }
}

bs_test_tail_imm2(Fail, Ctx, Offset) {
    ErlBinMatchBuffer *_mb;
    _mb = ms_matchbuffer($Ctx);
    if (_mb->size - _mb->offset != $Offset) {
        $FAIL($Fail);
    }
}

bs_test_unit(Fail, Ctx, Unit) {
    ErlBinMatchBuffer *_mb;
    _mb = ms_matchbuffer($Ctx);
    if ((_mb->size - _mb->offset) % $Unit) {
        $FAIL($Fail);
    }
}

bs_test_unit8(Fail, Ctx) {
    ErlBinMatchBuffer *_mb;
    _mb = ms_matchbuffer($Ctx);
    if ((_mb->size - _mb->offset) & 7) {
        $FAIL($Fail);
    }
}

i_bs_get_integer_8(Ctx, Fail, Dst) {
    Eterm _result;
    ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);

    if (_mb->size - _mb->offset < 8) {
        $FAIL($Fail);
    }
    if (BIT_OFFSET(_mb->offset) != 0) {
        _result = erts_bs_get_integer_2(c_p, 8, 0, _mb);
    } else {
        _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
        _mb->offset += 8;
    }
    $Dst = _result;
}

i_bs_get_integer_16(Ctx, Fail, Dst) {
    Eterm _result;
    ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);

    if (_mb->size - _mb->offset < 16) {
        $FAIL($Fail);
    }
    if (BIT_OFFSET(_mb->offset) != 0) {
        _result = erts_bs_get_integer_2(c_p, 16, 0, _mb);
    } else {
        _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
        _mb->offset += 16;
    }
    $Dst = _result;
}

%if ARCH_64
i_bs_get_integer_32(Ctx, Fail, Dst) {
    Uint32 _integer;
    ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);

    if (_mb->size - _mb->offset < 32) {
        $FAIL($Fail);
    }
    if (BIT_OFFSET(_mb->offset) != 0) {
        _integer = erts_bs_get_unaligned_uint32(_mb);
    } else {
        _integer = get_int32(_mb->base + _mb->offset/8);
    }
    _mb->offset += 32;
    $Dst = make_small(_integer);
}
%endif

i_bs_get_integer_imm := bs_get_integer.fetch.execute;
i_bs_get_integer_small_imm := bs_get_integer.fetch_small.execute;

bs_get_integer.head() {
    Eterm Ms, Sz;
}

bs_get_integer.fetch(Ctx, Size, Live) {
    Uint wordsneeded;
    Ms = $Ctx;
    Sz = $Size;
    wordsneeded = 1+WSIZE(NBYTES(Sz));
    $GC_TEST_PRESERVE(wordsneeded, $Live, Ms);
}

bs_get_integer.fetch_small(Ctx, Size) {
    Ms = $Ctx;
    Sz = $Size;
}

bs_get_integer.execute(Fail, Flags, Dst) {
    ErlBinMatchBuffer* mb;
    Eterm result;

    mb = ms_matchbuffer(Ms);
    LIGHT_SWAPOUT;
    result = erts_bs_get_integer_2(c_p, Sz, $Flags, mb);
    LIGHT_SWAPIN;
    HEAP_SPACE_VERIFIED(0);
    if (is_non_value(result)) {
        $FAIL($Fail);
    }
    $Dst = result;
}

i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) {
    Uint flags;
    Uint size;
    Eterm ms;
    ErlBinMatchBuffer* mb;
    Eterm result;

    flags = $FlagsAndUnit;
    ms = $Ms;
    $BS_GET_FIELD_SIZE($Sz, (flags >> 3), $FAIL($Fail), size);
    if (size >= SMALL_BITS) {
        Uint wordsneeded;
        /* Check bits size before potential gc.
         * We do not want a gc and then realize we don't need
         * the allocated space (i.e. if the op fails).
         *
         * Remember to re-acquire the matchbuffer after gc.
         */

        mb = ms_matchbuffer(ms);
        if (mb->size - mb->offset < size) {
            $FAIL($Fail);
        }
        wordsneeded = 1+WSIZE(NBYTES((Uint) size));
        $GC_TEST_PRESERVE(wordsneeded, $Live, ms);
    }
    mb = ms_matchbuffer(ms);
    LIGHT_SWAPOUT;
    result = erts_bs_get_integer_2(c_p, size, flags, mb);
    LIGHT_SWAPIN;
    HEAP_SPACE_VERIFIED(0);
    if (is_non_value(result)) {
        $FAIL($Fail);
    }
    $Dst = result;
}

i_bs_get_utf8(Ctx, Fail, Dst) {
    ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx);
    Eterm result = erts_bs_get_utf8(mb);

    if (is_non_value(result)) {
        $FAIL($Fail);
    }
    $Dst = result;
}

i_bs_get_utf16(Ctx, Fail, Flags, Dst) {
    ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx);
    Eterm result = erts_bs_get_utf16(mb, $Flags);

    if (is_non_value(result)) {
        $FAIL($Fail);
    }
    $Dst = result;
}

bs_context_to_binary := ctx_to_bin.fetch.execute;
i_bs_get_binary_all_reuse := ctx_to_bin.fetch_bin.execute;

ctx_to_bin.head() {
     Eterm context;
     ErlBinMatchBuffer* mb;
     ErlSubBin* sb;
     Uint size;
     Uint offs;
     Uint orig;
     Uint hole_size;
}

ctx_to_bin.fetch(Src) {
    context = $Src;
    if (is_boxed(context) &&
        header_is_bin_matchstate(*boxed_val(context))) {
        ErlBinMatchState* ms;
        ms = (ErlBinMatchState *) boxed_val(context);
        mb = &ms->mb;
        offs = ms->save_offset[0];
        size = mb->size - offs;
    } else {
        $NEXT0();
    }
}

ctx_to_bin.fetch_bin(Src, Fail, Unit) {
    context = $Src;
    mb = ms_matchbuffer(context);
    size = mb->size - mb->offset;
    if (size % $Unit != 0) {
        $FAIL($Fail);
    }
    offs = mb->offset;
}

ctx_to_bin.execute() {
    orig = mb->orig;
    sb = (ErlSubBin *) boxed_val(context);
    hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE;
    sb->thing_word = HEADER_SUB_BIN;
    sb->size = BYTE_OFFSET(size);
    sb->bitsize = BIT_OFFSET(size);
    sb->offs = BYTE_OFFSET(offs);
    sb->bitoffs = BIT_OFFSET(offs);
    sb->is_writable = 0;
    sb->orig = orig;
    if (hole_size) {
        sb[1].thing_word = make_pos_bignum_header(hole_size-1);
    }
}

i_bs_match_string(Ctx, Fail, Bits, Ptr) {
    byte* bytes = (byte *) $Ptr;
    Uint bits = $Bits;
    ErlBinMatchBuffer* mb;
    Uint offs;

    mb = ms_matchbuffer($Ctx);
    if (mb->size - mb->offset < bits) {
        $FAIL($Fail);
    }
    offs = mb->offset & 7;
    if (offs == 0 && (bits & 7) == 0) {
        if (sys_memcmp(bytes, mb->base+(mb->offset>>3), bits>>3)) {
            $FAIL($Fail);
        }
    } else if (erts_cmp_bits(bytes, 0, mb->base+(mb->offset>>3), mb->offset & 7, bits)) {
        $FAIL($Fail);
    }
    mb->offset += bits;
}

i_bs_save2(Src, Slot) {
    ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src);
    _ms->save_offset[$Slot] = _ms->mb.offset;
}

i_bs_restore2(Src, Slot) {
    ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src);
    _ms->mb.offset = _ms->save_offset[$Slot];
}