Merge branch 'nox/ets-update_counter-4'

* nox/ets-update_counter-4:
  Create new BIF ets:update_counter/4
  Allow 4-ary BIFs

OTP-12563

21 files changed, 436 insertions, 94 deletions

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index b89c8b3900..e847403882 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3582,7 +3582,7 @@ get_map_elements_fail:
 	    vbf = (BifFunction) Arg(0);
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
 	    bif_nif_arity = I[-1];
-	    ASSERT(bif_nif_arity <= 3);
+            ASSERT(bif_nif_arity <= 4);
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	    reg[0] = r(0);
 	    {
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 7b69b39511..837cb017ac 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -30,10 +30,12 @@ extern Export* erts_format_cpu_topology_trap;
 #define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS
 #define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS
 #define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS
+#define BIF_ALIST_4 Process* A__p, Eterm* BIF__ARGS
 
 #define BIF_ARG_1  (BIF__ARGS[0])
 #define BIF_ARG_2  (BIF__ARGS[1])
 #define BIF_ARG_3  (BIF__ARGS[2])
+#define BIF_ARG_4  (BIF__ARGS[3])
 
 #define ERTS_IS_PROC_OUT_OF_REDS(p)		\
     ((p)->fcalls > 0				\
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 1d0d214e77..788c866e63 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -613,6 +613,7 @@ bif erlang:fun_info_mfa/1
 #
 
 bif erlang:get_keys/0
+bif ets:update_counter/4
 
 #
 # Obsolete
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 4806befd99..fff892ae54 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -805,7 +805,7 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3)
 	list = BIF_ARG_3;
     }
 
-    if (!tb->common.meth->db_lookup_dbterm(tb, BIF_ARG_2, &handle)) {
+    if (!tb->common.meth->db_lookup_dbterm(BIF_P, tb, BIF_ARG_2, THE_NON_VALUE, &handle)) {
 	cret = DB_ERROR_BADKEY;
 	goto bail_out;
     }
@@ -844,7 +844,7 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3)
     }
 
 finalize:
-    tb->common.meth->db_finalize_dbterm(&handle);
+    tb->common.meth->db_finalize_dbterm(cret, &handle);
 
 bail_out:
     UnUseTmpHeap(2,BIF_P);
@@ -863,14 +863,8 @@ bail_out:
     }
 }
 
-/* 
-** update_counter(Tab, Key, Incr) 
-** update_counter(Tab, Key, {Upop}) 
-** update_counter(Tab, Key, [{Upop}]) 
-** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo}
-** Returns new value(s) (integer or [integer])
-*/
-BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
+static BIF_RETTYPE
+do_update_counter(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Eterm arg4)
 {
     DbTable* tb;
     int cret = DB_ERROR_BADITEM;
@@ -880,7 +874,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     Eterm* ret_list_currp = NULL;
     Eterm* ret_list_prevp = NULL;
     Eterm iter;
-    DeclareTmpHeap(cell,5,BIF_P);
+    DeclareTmpHeap(cell, 5, p);
     Eterm *tuple = cell+2;
     DbUpdateHandle handle;
     Uint halloc_size = 0; /* overestimated heap usage */
@@ -888,28 +882,29 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     Eterm* hstart;
     Eterm* hend;
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
+    if ((tb = db_get_table(p, arg1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
+        BIF_ERROR(p, BADARG);
     }
 
-    UseTmpHeap(5,BIF_P);
+    UseTmpHeap(5, p);
 
     if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
 	goto bail_out;
     }
-    if (is_integer(BIF_ARG_3)) {  /* Incr */
-	upop_list = CONS(cell, TUPLE2(tuple, make_small(tb->common.keypos+1),
-				      BIF_ARG_3), NIL);
+    if (is_integer(arg3)) { /* Incr */
+        upop_list = CONS(cell,
+                         TUPLE2(tuple, make_small(tb->common.keypos+1), arg3),
+                         NIL);
     }
-    else if (is_tuple(BIF_ARG_3)) { /* {Upop} */
-	upop_list = CONS(cell, BIF_ARG_3, NIL);
+    else if (is_tuple(arg3)) { /* {Upop} */
+        upop_list = CONS(cell, arg3, NIL);
     }
     else { /* [{Upop}] (probably) */
-	upop_list = BIF_ARG_3;
+        upop_list = arg3;
 	ret_list_prevp = &ret;
     }
 
-    if (!tb->common.meth->db_lookup_dbterm(tb, BIF_ARG_2, &handle)) {
+    if (!tb->common.meth->db_lookup_dbterm(p, tb, arg2, arg4, &handle)) {
 	goto bail_out; /* key not found */
     }
 
@@ -982,13 +977,13 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     if (ret_list_prevp) { /* Prepare to return a list */
 	ret = NIL;
 	halloc_size += list_size;
-	hstart = HAlloc(BIF_P, halloc_size);
+	hstart = HAlloc(p, halloc_size);
 	ret_list_currp = hstart;
 	htop = hstart + list_size;
 	hend = hstart + halloc_size;
     }
     else {
-	hstart = htop = HAlloc(BIF_P, halloc_size);
+	hstart = htop = HAlloc(p, halloc_size);
     }
     hend = hstart + halloc_size;
 
@@ -1035,26 +1030,54 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 	   (is_list(ret) && (list_val(ret)+list_size)==ret_list_currp));
     ASSERT(htop <= hend);
 
-    HRelease(BIF_P,hend,htop);
+    HRelease(p, hend, htop);
 
 finalize:
-    tb->common.meth->db_finalize_dbterm(&handle);
+    tb->common.meth->db_finalize_dbterm(cret, &handle);
 
 bail_out:
-    UnUseTmpHeap(5,BIF_P);
+    UnUseTmpHeap(5, p);
     db_unlock(tb, LCK_WRITE_REC);
 
     switch (cret) {
     case DB_ERROR_NONE:
 	BIF_RET(ret);
     case DB_ERROR_SYSRES:
-	BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+        BIF_ERROR(p, SYSTEM_LIMIT);
     default:
-	BIF_ERROR(BIF_P, BADARG);
+        BIF_ERROR(p, BADARG);
 	break;
     }
 }
 
+/*
+** update_counter(Tab, Key, Incr)
+** update_counter(Tab, Key, Upop)
+** update_counter(Tab, Key, [{Upop}])
+** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo}
+** Returns new value(s) (integer or [integer])
+*/
+BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
+{
+    return do_update_counter(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, THE_NON_VALUE);
+}
+
+/*
+** update_counter(Tab, Key, Incr, Default)
+** update_counter(Tab, Key, Upop, Default)
+** update_counter(Tab, Key, [{Upop}], Default)
+** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo}
+** Returns new value(s) (integer or [integer])
+*/
+BIF_RETTYPE ets_update_counter_4(BIF_ALIST_4)
+{
+    if (is_not_tuple(BIF_ARG_4)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    return do_update_counter(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BIF_ARG_4);
+}
+
+
 /* 
 ** The put BIF 
 */
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index c2157457a0..8668a87ba1 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -444,8 +444,11 @@ static int db_delete_all_objects_hash(Process* p, DbTable* tbl);
 #ifdef HARDDEBUG
 static void db_check_table_hash(DbTableHash *tb);
 #endif
-static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle);
-static void db_finalize_dbterm_hash(DbUpdateHandle* handle);
+static int
+db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj,
+                      DbUpdateHandle* handle);
+static void
+db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle);
 
 static ERTS_INLINE void try_shrink(DbTableHash* tb)
 {
@@ -2796,59 +2799,129 @@ static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr,
     return NULL;
 }
 
-static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle)
+static int
+db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj,
+                      DbUpdateHandle* handle)
 {
     DbTableHash *tb = &tbl->hash;
-    HashDbTerm* b;
-    HashDbTerm** prevp;
-    int ix;
     HashValue hval;
+    HashDbTerm **bp, *b;
     erts_smp_rwmtx_t* lck;
+    int flags = 0;
+
+    ASSERT(tb->common.status & DB_SET);
 
     hval = MAKE_HASH(key);
-    lck = WLOCK_HASH(tb,hval);
-    ix = hash_to_ix(tb, hval);
-    prevp = &BUCKET(tb, ix);
-    b = *prevp;
+    lck = WLOCK_HASH(tb, hval);
+    bp = &BUCKET(tb, hash_to_ix(tb, hval));
+    b = *bp;
 
-    while (b != 0) {
-	if (has_live_key(tb,b,key,hval)) {
-	    handle->tb = tbl;
-	    handle->bp = (void**) prevp;
-	    handle->dbterm = &b->dbterm;
-	    handle->mustResize = 0;
-	    handle->new_size = b->dbterm.size;
-	#if HALFWORD_HEAP
-	    handle->abs_vec = NULL;
-	#endif
-	    handle->lck = lck;
-	    /* KEEP hval WLOCKED, db_finalize_dbterm_hash will WUNLOCK */
-	    return 1;
-	}
-	prevp = &b->next;
-	b = *prevp;
+    for (;;) {
+        if (b == NULL) {
+            break;
+        }
+        if (has_key(tb, b, key, hval)) {
+            if (b->hvalue != INVALID_HASH) {
+                goto Ldone;
+            }
+            break;
+        }
+        bp = &b->next;
+        b = *bp;
     }
-    WUNLOCK_HASH(lck);
-    return 0;
+
+    if (obj == THE_NON_VALUE) {
+        WUNLOCK_HASH(lck);
+        return 0;
+    }
+
+    {
+        Eterm *objp = tuple_val(obj);
+        int arity = arityval(*objp);
+        Eterm *htop, *hend;
+
+        ASSERT(arity >= tb->common.keypos);
+        htop = HAlloc(p, arity + 1);
+        hend = htop + arity + 1;
+        sys_memcpy(htop, objp, sizeof(Eterm) * (arity + 1));
+        htop[tb->common.keypos] = key;
+        obj = make_tuple(htop);
+
+        if (b == NULL) {
+            HashDbTerm *q = new_dbterm(tb, obj);
+
+            q->hvalue = hval;
+            q->next = NULL;
+            *bp = b = q;
+
+            {
+                int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems);
+                int nactive = NACTIVE(tb);
+
+                if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) {
+                    grow(tb, nactive);
+                }
+            }
+        } else {
+            HashDbTerm *q, *next = b->next;
+
+            ASSERT(b->hvalue == INVALID_HASH);
+            q = replace_dbterm(tb, b, obj);
+            q->next = next;
+            q->hvalue = hval;
+            *bp = b = q;
+            erts_smp_atomic_inc_nob(&tb->common.nitems);
+        }
+
+        HRelease(p, hend, htop);
+        flags |= DB_NEW_OBJECT;
+    }
+
+Ldone:
+    handle->tb = tbl;
+    handle->bp = (void **)bp;
+    handle->dbterm = &b->dbterm;
+    handle->flags = flags;
+    handle->new_size = b->dbterm.size;
+#if HALFWORD_HEAP
+    handle->abs_vec = NULL;
+#endif
+    handle->lck = lck;
+    return 1;
 }
 
 /* Must be called after call to db_lookup_dbterm
 */
-static void db_finalize_dbterm_hash(DbUpdateHandle* handle)
+static void
+db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle)
 {
     DbTable* tbl = handle->tb;
-    HashDbTerm* oldp = (HashDbTerm*) *(handle->bp);
+    DbTableHash *tb = &tbl->hash;
+    HashDbTerm **bp = (HashDbTerm **) handle->bp;
+    HashDbTerm *b = *bp;
     erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck;
 
-    ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(&tbl->hash,lck));  /* locked by db_lookup_dbterm_hash */
+    ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(tb, lck));  /* locked by db_lookup_dbterm_hash */
+
+    ASSERT((&b->dbterm == handle->dbterm) == !(tb->common.compress && handle->flags & DB_MUST_RESIZE));
 
-    ASSERT((&oldp->dbterm == handle->dbterm) == !(tbl->common.compress && handle->mustResize));
+    if (handle->flags & DB_NEW_OBJECT && cret != DB_ERROR_NONE) {
+        if (IS_FIXED(tb)) {
+            add_fixed_deletion(tb, hash_to_ix(tb, b->hvalue));
+            b->hvalue = INVALID_HASH;
+        } else {
+            *bp = b->next;
+            free_term(tb, b);
+        }
 
-    if (handle->mustResize) {
+        WUNLOCK_HASH(lck);
+        erts_smp_atomic_dec_nob(&tb->common.nitems);
+        try_shrink(tb);
+    } else if (handle->flags & DB_MUST_RESIZE) {
 	db_finalize_resize(handle, offsetof(HashDbTerm,dbterm));
 	WUNLOCK_HASH(lck);
 
-	free_term(&tbl->hash, oldp);
+	free_term(tb, b);
     }
     else {
 	WUNLOCK_HASH(lck);
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 720c0659c3..577da35b75 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -399,8 +399,11 @@ static int db_delete_all_objects_tree(Process* p, DbTable* tbl);
 #ifdef HARDDEBUG
 static void db_check_table_tree(DbTable *tbl);
 #endif
-static int db_lookup_dbterm_tree(DbTable *, Eterm key, DbUpdateHandle*);
-static void db_finalize_dbterm_tree(DbUpdateHandle*);
+static int
+db_lookup_dbterm_tree(Process *, DbTable *, Eterm key, Eterm obj,
+                      DbUpdateHandle*);
+static void
+db_finalize_dbterm_tree(int cret, DbUpdateHandle *);
 
 /*
 ** Static variables
@@ -2546,16 +2549,43 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key)
     return this;
 }
 
-static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle)
+static int
+db_lookup_dbterm_tree(Process *p, DbTable *tbl, Eterm key, Eterm obj,
+                      DbUpdateHandle* handle)
 {
     DbTableTree *tb = &tbl->tree;
     TreeDbTerm **pp = find_node2(tb, key);
-
-    if (pp == NULL) return 0;
+    int flags = 0;
+
+    if (pp == NULL) {
+        if (obj == THE_NON_VALUE) {
+            return 0;
+        } else {
+            Eterm *objp = tuple_val(obj);
+            int arity = arityval(*objp);
+            Eterm *htop, *hend;
+
+            ASSERT(arity >= tb->common.keypos);
+            htop = HAlloc(p, arity + 1);
+            hend = htop + arity + 1;
+            sys_memcpy(htop, objp, sizeof(Eterm) * (arity + 1));
+            htop[tb->common.keypos] = key;
+            obj = make_tuple(htop);
+
+            if (db_put_tree(tbl, obj, 1) != DB_ERROR_NONE) {
+                return 0;
+            }
+
+            pp = find_node2(tb, key);
+            ASSERT(pp != NULL);
+            HRelease(p, hend, htop);
+            flags |= DB_NEW_OBJECT;
+        }
+    }
 
     handle->tb = tbl;
     handle->dbterm = &(*pp)->dbterm;
-    handle->mustResize = 0;
+    handle->flags = flags;
     handle->bp = (void**) pp;
     handle->new_size = (*pp)->dbterm.size;
 #if HALFWORD_HEAP
@@ -2564,15 +2594,21 @@ static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle
     return 1;
 }
 
-static void db_finalize_dbterm_tree(DbUpdateHandle* handle)
+static void
+db_finalize_dbterm_tree(int cret, DbUpdateHandle *handle)
 {
-    if (handle->mustResize) {
-	TreeDbTerm* oldp = (TreeDbTerm*) *handle->bp;
+    DbTable *tbl = handle->tb;
+    DbTableTree *tb = &tbl->tree;
+    TreeDbTerm *bp = (TreeDbTerm *) *handle->bp;
 
+    if (handle->flags & DB_NEW_OBJECT && cret != DB_ERROR_NONE) {
+        Eterm ret;
+        db_erase_tree(tbl, GETKEY(tb, bp->dbterm.tpl), &ret);
+    } else if (handle->flags & DB_MUST_RESIZE) {
 	db_finalize_resize(handle, offsetof(TreeDbTerm,dbterm));
-	reset_static_stack(&handle->tb->tree);
+        reset_static_stack(tb);
 
-	free_term(&handle->tb->tree, oldp);
+        free_term(tb, bp);
     }
 #ifdef DEBUG
     handle->dbterm = 0;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 748be93fe3..60da35da56 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2710,10 +2710,10 @@ Wterm db_do_read_element(DbUpdateHandle* handle, Sint position)
     }
 
     ASSERT(((DbTableCommon*)handle->tb)->compress);
-    ASSERT(!handle->mustResize);
+    ASSERT(!(handle->flags & DB_MUST_RESIZE));
     handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common,
 					       handle->dbterm);
-    handle->mustResize = 1;
+    handle->flags |= DB_MUST_RESIZE;
     return handle->dbterm->tpl[position];
 }
 
@@ -2746,11 +2746,11 @@ void db_do_update_element(DbUpdateHandle* handle,
     #endif
 	return;
     }
-    if (!handle->mustResize) {
+    if (!(handle->flags & DB_MUST_RESIZE)) {
 	if (handle->tb->common.compress) {
 	    handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common,
 						       handle->dbterm);
-	    handle->mustResize = 1;
+	    handle->flags |= DB_MUST_RESIZE;
 	    oldval = handle->dbterm->tpl[position];
         #if HALFWORD_HEAP
 	    old_base = NULL;
@@ -2810,7 +2810,7 @@ both_size_set:
 
     /* write new value in old dbterm, finalize will make a flat copy */
     handle->dbterm->tpl[position] = newval;
-    handle->mustResize = 1;
+    handle->flags |= DB_MUST_RESIZE;
 
 #if HALFWORD_HEAP
     if (old_base && newval_sz > 0) {
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 5ace93c8ed..ca206c7f58 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -76,6 +76,9 @@ typedef struct db_term {
 union db_table;
 typedef union db_table DbTable;
 
+#define DB_MUST_RESIZE 1
+#define DB_NEW_OBJECT 2
+
 /* Info about a database entry while it's being updated
  * (by update_counter or update_element)
  */
@@ -84,7 +87,7 @@ typedef struct {
     DbTerm* dbterm;
     void** bp;         /* {Hash|Tree}DbTerm** */
     Uint new_size;
-    int mustResize;
+    int flags;
     void* lck;
 #if HALFWORD_HEAP
     unsigned char* abs_vec;  /* [i] true if dbterm->tpl[i] is absolute Eterm */
@@ -183,15 +186,14 @@ typedef struct db_table_method
 			       void *arg);
     void (*db_check_table)(DbTable* tb);
 
-    /* Lookup a dbterm for updating. Return false if not found.
-    */
-    int (*db_lookup_dbterm)(DbTable*, Eterm key, 
-			    DbUpdateHandle* handle); /* [out] */
+    /* Lookup a dbterm for updating. Return false if not found. */
+    int (*db_lookup_dbterm)(Process *, DbTable *, Eterm key, Eterm obj,
+                            DbUpdateHandle* handle);
 
-    /* Must be called for each db_lookup_dbterm that returned true,
-    ** even if dbterm was not updated.
-    */
-    void (*db_finalize_dbterm)(DbUpdateHandle* handle);
+    /* Must be called for each db_lookup_dbterm that returned true, even if
+    ** dbterm was not updated. If the handle was of a new object and cret is
+    ** not DB_ERROR_NONE, the object is removed from the table. */
+    void (*db_finalize_dbterm)(int cret, DbUpdateHandle* handle);
 
 } DbTableMethod;
 
diff --git a/erts/emulator/hipe/hipe_amd64_abi.txt b/erts/emulator/hipe/hipe_amd64_abi.txt
index 8a34bfa67f..72aed13995 100644
--- a/erts/emulator/hipe/hipe_amd64_abi.txt
+++ b/erts/emulator/hipe/hipe_amd64_abi.txt
@@ -45,7 +45,7 @@ The first return value from a function is placed in %rax, the second
 (if any) is placed in %rdx.
 
 Notes:
-- Currently, NR_ARG_REGS==0.
+- Currently, NR_ARG_REGS == 4.
 - C BIFs expect P in C parameter register 1: %rdi. By making Erlang
   parameter registers 1-5 coincide with C parameter registers 2-6,
   our BIF wrappers can simply move P to %rdi without having to shift
diff --git a/erts/emulator/hipe/hipe_amd64_asm.m4 b/erts/emulator/hipe/hipe_amd64_asm.m4
index b4b3c073ab..ca55d5bf3b 100644
--- a/erts/emulator/hipe/hipe_amd64_asm.m4
+++ b/erts/emulator/hipe/hipe_amd64_asm.m4
@@ -253,6 +253,10 @@ define(NBIF_ARG,`ifelse(eval($3 >= NR_ARG_REGS),0,`NBIF_REG_ARG($1,$3)',`NBIF_ST
 `/* #define NBIF_ARG_3_0	'NBIF_ARG(%rsi,3,0)` */'
 `/* #define NBIF_ARG_3_1	'NBIF_ARG(%rdx,3,1)` */'
 `/* #define NBIF_ARG_3_2	'NBIF_ARG(%rcx,3,2)` */'
+`/* #define NBIF_ARG_4_0	'NBIF_ARG(%rsi,4,0)` */'
+`/* #define NBIF_ARG_4_1	'NBIF_ARG(%rdx,4,1)` */'
+`/* #define NBIF_ARG_4_2	'NBIF_ARG(%rcx,4,2)` */'
+`/* #define NBIF_ARG_4_3	'NBIF_ARG(%r8,4,3)` */'
 `/* #define NBIF_ARG_5_0	'NBIF_ARG(%rsi,5,0)` */'
 `/* #define NBIF_ARG_5_1	'NBIF_ARG(%rdx,5,1)` */'
 `/* #define NBIF_ARG_5_2	'NBIF_ARG(%rcx,5,2)` */'
@@ -277,6 +281,7 @@ define(NBIF_RET,`NBIF_RET_N(eval(RET_POP($1)))')dnl
 `/* #define NBIF_RET_1	'NBIF_RET(1)` */'
 `/* #define NBIF_RET_2	'NBIF_RET(2)` */'
 `/* #define NBIF_RET_3	'NBIF_RET(3)` */'
+`/* #define NBIF_RET_4	'NBIF_RET(4)` */'
 `/* #define NBIF_RET_5	'NBIF_RET(5)` */'
 
 `#endif /* ASM */'
diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4
index 7a4bb30447..7d94aa05b3 100644
--- a/erts/emulator/hipe/hipe_amd64_bifs.m4
+++ b/erts/emulator/hipe/hipe_amd64_bifs.m4
@@ -51,9 +51,10 @@ define(HANDLE_GOT_MBUF,`
  * standard_bif_interface_1(nbif_name, cbif_name)
  * standard_bif_interface_2(nbif_name, cbif_name)
  * standard_bif_interface_3(nbif_name, cbif_name)
+ * standard_bif_interface_4(nbif_name, cbif_name)
  * standard_bif_interface_0(nbif_name, cbif_name)
  *
- * Generate native interface for a BIF with 0-3 parameters and
+ * Generate native interface for a BIF with 0-4 parameters and
  * standard failure mode.
  */
 define(standard_bif_interface_1,
@@ -154,6 +155,43 @@ ASYM($1):
 	TYPE_FUNCTION(ASYM($1))
 #endif')
 
+define(standard_bif_interface_4,
+`
+#ifndef HAVE_$1
+#`define' HAVE_$1
+    TEXT
+    .align  4
+    GLOBAL(ASYM($1))
+ASYM($1):
+    /* set up the parameters */
+    movq    P, %rdi
+    NBIF_ARG(%rsi,4,0)
+    NBIF_ARG(%rdx,4,1)
+    NBIF_ARG(%rcx,4,2)
+    NBIF_ARG(%r8,4,3)
+
+    /* make the call on the C stack */
+    SWITCH_ERLANG_TO_C
+    pushq   %r8
+    pushq   %rcx
+    pushq   %rdx
+    pushq   %rsi
+    movq    %rsp, %rsi  /* Eterm* BIF__ARGS */
+    sub $(8), %rsp  /* stack frame 16-byte alignment */
+    CALL_BIF($2)
+    add $(4*8 + 8), %rsp
+    TEST_GOT_MBUF
+    SWITCH_C_TO_ERLANG
+
+    /* throw exception if failure, otherwise return */
+    TEST_GOT_EXN
+    jz  nbif_4_simple_exception
+    NBIF_RET(4)
+    HANDLE_GOT_MBUF(4)
+    SET_SIZE(ASYM($1))
+    TYPE_FUNCTION(ASYM($1))
+#endif')
+
 define(standard_bif_interface_0,
 `
 #ifndef HAVE_$1
diff --git a/erts/emulator/hipe/hipe_amd64_glue.S b/erts/emulator/hipe/hipe_amd64_glue.S
index 955f7362b4..3cb0a2875b 100644
--- a/erts/emulator/hipe/hipe_amd64_glue.S
+++ b/erts/emulator/hipe/hipe_amd64_glue.S
@@ -321,6 +321,7 @@ ASYM(nbif_fail):
 	GLOBAL(nbif_1_gc_after_bif)
 	GLOBAL(nbif_2_gc_after_bif)
 	GLOBAL(nbif_3_gc_after_bif)
+	GLOBAL(nbif_4_gc_after_bif)
 	.align	4
 nbif_0_gc_after_bif:
 	xorl	%edx, %edx
@@ -336,6 +337,10 @@ nbif_2_gc_after_bif:
 	.align	4
 nbif_3_gc_after_bif:
 	movl	$3, %edx
+	jmp	.gc_after_bif
+	.align	4
+nbif_4_gc_after_bif:
+	movl	$4, %edx
 	/*FALLTHROUGH*/
 	.align	4
 .gc_after_bif:
@@ -359,6 +364,7 @@ nbif_3_gc_after_bif:
 	GLOBAL(nbif_1_simple_exception)
 	GLOBAL(nbif_2_simple_exception)
 	GLOBAL(nbif_3_simple_exception)
+	GLOBAL(nbif_4_simple_exception)
 	.align	4
 nbif_0_simple_exception:
 	xorl	%eax, %eax
@@ -374,6 +380,10 @@ nbif_2_simple_exception:
 	.align	4
 nbif_3_simple_exception:
 	movl	$3, %eax
+	jmp	.nbif_simple_exception
+	.align	4
+nbif_4_simple_exception:
+	movl	$4, %eax
 	/*FALLTHROUGH*/
 	.align	4
 .nbif_simple_exception:
diff --git a/erts/emulator/hipe/hipe_ppc_asm.m4 b/erts/emulator/hipe/hipe_ppc_asm.m4
index 4a1caa1543..e5a56de687 100644
--- a/erts/emulator/hipe/hipe_ppc_asm.m4
+++ b/erts/emulator/hipe/hipe_ppc_asm.m4
@@ -280,6 +280,10 @@ define(NBIF_ARG,`ifelse(eval($3 >= NR_ARG_REGS),0,`NBIF_REG_ARG($1,$3)',`NBIF_ST
 `/* #define NBIF_ARG_3_0	'NBIF_ARG(r3,3,0)` */'
 `/* #define NBIF_ARG_3_1	'NBIF_ARG(r3,3,1)` */'
 `/* #define NBIF_ARG_3_2	'NBIF_ARG(r3,3,2)` */'
+`/* #define NBIF_ARG_4_0	'NBIF_ARG(r3,4,0)` */'
+`/* #define NBIF_ARG_4_1	'NBIF_ARG(r3,4,1)` */'
+`/* #define NBIF_ARG_4_2	'NBIF_ARG(r3,4,2)` */'
+`/* #define NBIF_ARG_4_3	'NBIF_ARG(r3,4,3)` */'
 `/* #define NBIF_ARG_5_0	'NBIF_ARG(r3,5,0)` */'
 `/* #define NBIF_ARG_5_1	'NBIF_ARG(r3,5,1)` */'
 `/* #define NBIF_ARG_5_2	'NBIF_ARG(r3,5,2)` */'
@@ -301,6 +305,7 @@ define(NBIF_RET,`NBIF_RET_N(eval(RET_POP($1)))')dnl
 `/* #define NBIF_RET_1	'NBIF_RET(1)` */'
 `/* #define NBIF_RET_2	'NBIF_RET(2)` */'
 `/* #define NBIF_RET_3	'NBIF_RET(3)` */'
+`/* #define NBIF_RET_4	'NBIF_RET(4)` */'
 `/* #define NBIF_RET_5	'NBIF_RET(5)` */'
 
 dnl
diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4
index f53b79b52e..b173b896b8 100644
--- a/erts/emulator/hipe/hipe_ppc_bifs.m4
+++ b/erts/emulator/hipe/hipe_ppc_bifs.m4
@@ -46,9 +46,10 @@ define(HANDLE_GOT_MBUF,`
  * standard_bif_interface_1(nbif_name, cbif_name)
  * standard_bif_interface_2(nbif_name, cbif_name)
  * standard_bif_interface_3(nbif_name, cbif_name)
+ * standard_bif_interface_4(nbif_name, cbif_name)
  * standard_bif_interface_0(nbif_name, cbif_name)
  *
- * Generate native interface for a BIF with 0-3 parameters and
+ * Generate native interface for a BIF with 0-4 parameters and
  * standard failure mode.
  */
 define(standard_bif_interface_1,
@@ -144,6 +145,41 @@ ASYM($1):
 	TYPE_FUNCTION(ASYM($1))
 #endif')
 
+define(standard_bif_interface_4,
+`
+#ifndef HAVE_$1
+#`define' HAVE_$1
+	GLOBAL(ASYM($1))
+ASYM($1):
+	/* Set up C argument registers. */
+	mr	r3, P
+	NBIF_ARG(r4,4,0)
+	NBIF_ARG(r5,4,1)
+	NBIF_ARG(r6,4,2)
+	NBIF_ARG(r7,4,3)
+
+	/* Save caller-save registers and call the C function. */
+	SAVE_CONTEXT_BIF
+	STORE	r4, P_ARG0(r3)		/* Store BIF__ARGS in def_arg_reg[] */
+	STORE	r5, P_ARG1(r3)
+	STORE	r6, P_ARG2(r3)
+	STORE	r7, P_ARG3(r3)
+	addi	r4, r3, P_ARG0
+	CALL_BIF($2)
+	TEST_GOT_MBUF
+
+	/* Restore registers. Check for exception. */
+	CMPI	r3, THE_NON_VALUE
+	RESTORE_CONTEXT_BIF
+	beq-	1f
+	NBIF_RET(4)
+1:	/* workaround for bc:s small offset operand */
+	b	CSYM(nbif_4_simple_exception)
+	HANDLE_GOT_MBUF(4)
+	SET_SIZE(ASYM($1))
+	TYPE_FUNCTION(ASYM($1))
+#endif')
+
 define(standard_bif_interface_0,
 `
 #ifndef HAVE_$1
diff --git a/erts/emulator/hipe/hipe_ppc_glue.S b/erts/emulator/hipe/hipe_ppc_glue.S
index c48fb150af..b07f4bc9c8 100644
--- a/erts/emulator/hipe/hipe_ppc_glue.S
+++ b/erts/emulator/hipe/hipe_ppc_glue.S
@@ -462,10 +462,12 @@ ASYM(nbif_fail):
 	OPD(nbif_1_gc_after_bif)
 	OPD(nbif_2_gc_after_bif)
 	OPD(nbif_3_gc_after_bif)
+	OPD(nbif_4_gc_after_bif)
 	GLOBAL(CSYM(nbif_0_gc_after_bif))
 	GLOBAL(CSYM(nbif_1_gc_after_bif))
 	GLOBAL(CSYM(nbif_2_gc_after_bif))
 	GLOBAL(CSYM(nbif_3_gc_after_bif))
+	GLOBAL(CSYM(nbif_4_gc_after_bif))
 CSYM(nbif_0_gc_after_bif):
 	li	r4, 0
 	b	.gc_after_bif
@@ -477,6 +479,9 @@ CSYM(nbif_2_gc_after_bif):
 	b	.gc_after_bif
 CSYM(nbif_3_gc_after_bif):
 	li	r4, 3
+	b	.gc_after_bif
+CSYM(nbif_4_gc_after_bif):
+	li	r4, 4
 	/*FALLTHROUGH*/
 .gc_after_bif:
 	stw	r4, P_NARITY(P)		/* Note: narity is a 32-bit field */
@@ -519,6 +524,11 @@ CSYM(nbif_2_simple_exception):
 	GLOBAL(CSYM(nbif_3_simple_exception))
 CSYM(nbif_3_simple_exception):
 	li	r4, 3
+	b	.nbif_simple_exception
+	OPD(nbif_3_simple_exception)
+	GLOBAL(CSYM(nbif_4_simple_exception))
+CSYM(nbif_4_simple_exception):
+	li	r4, 4
 	/*FALLTHROUGH*/
 .nbif_simple_exception:
 	LOAD	r3, P_FREASON(P)
diff --git a/erts/emulator/hipe/hipe_sparc_asm.m4 b/erts/emulator/hipe/hipe_sparc_asm.m4
index c3c3bcb74a..8020104e40 100644
--- a/erts/emulator/hipe/hipe_sparc_asm.m4
+++ b/erts/emulator/hipe/hipe_sparc_asm.m4
@@ -176,6 +176,10 @@ define(NBIF_ARG,`ifelse(eval($3 >= NR_ARG_REGS),0,`NBIF_REG_ARG($1,$3)',`NBIF_ST
 `/* #define NBIF_ARG_3_0	'NBIF_ARG(r1,3,0)` */'
 `/* #define NBIF_ARG_3_1	'NBIF_ARG(r2,3,1)` */'
 `/* #define NBIF_ARG_3_2	'NBIF_ARG(r3,3,2)` */'
+`/* #define NBIF_ARG_4_0	'NBIF_ARG(r1,4,0)` */'
+`/* #define NBIF_ARG_4_1	'NBIF_ARG(r2,4,1)` */'
+`/* #define NBIF_ARG_4_2	'NBIF_ARG(r3,4,2)` */'
+`/* #define NBIF_ARG_4_3	'NBIF_ARG(r3,4,3)` */'
 `/* #define NBIF_ARG_5_0	'NBIF_ARG(r1,5,0)` */'
 `/* #define NBIF_ARG_5_1	'NBIF_ARG(r2,5,1)` */'
 `/* #define NBIF_ARG_5_2	'NBIF_ARG(r3,5,2)` */'
@@ -200,6 +204,7 @@ define(NBIF_RET,`NBIF_RET_N(eval(RET_POP($1)))')dnl
 `/* #define NBIF_RET_1	'NBIF_RET(1)` */'
 `/* #define NBIF_RET_2	'NBIF_RET(2)` */'
 `/* #define NBIF_RET_3	'NBIF_RET(3)` */'
+`/* #define NBIF_RET_4	'NBIF_RET(4)` */'
 `/* #define NBIF_RET_5	'NBIF_RET(5)` */'
 
 dnl
diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4
index 2bfe3a4646..8dfb28c8e0 100644
--- a/erts/emulator/hipe/hipe_sparc_bifs.m4
+++ b/erts/emulator/hipe/hipe_sparc_bifs.m4
@@ -54,9 +54,10 @@ define(HANDLE_GOT_MBUF,`
  * standard_bif_interface_1(nbif_name, cbif_name)
  * standard_bif_interface_2(nbif_name, cbif_name)
  * standard_bif_interface_3(nbif_name, cbif_name)
+ * standard_bif_interface_3(nbif_name, cbif_name)
  * standard_bif_interface_0(nbif_name, cbif_name)
  *
- * Generate native interface for a BIF with 0-3 parameters and
+ * Generate native interface for a BIF with 0-4 parameters and
  * standard failure mode.
  */
 define(standard_bif_interface_1,
@@ -146,6 +147,39 @@ $1:
 	.type	$1, #function
 #endif')
 
+define(standard_bif_interface_4,
+`
+#ifndef HAVE_$1
+#`define' HAVE_$1
+	.global $1
+$1:
+	/* Set up C argument registers. */
+	mov	P, %o0
+	NBIF_ARG(%o1,4,0)
+	NBIF_ARG(%o2,4,1)
+	NBIF_ARG(%o3,4,2)
+	NBIF_ARG(%o4,4,3)
+
+	/* Save caller-save registers and call the C function. */
+	SAVE_CONTEXT_BIF
+	st 	%o1, [%o0+P_ARG0]	! Store BIF__ARGS in def_arg_reg
+	st 	%o2, [%o0+P_ARG1]
+	st 	%o3, [%o0+P_ARG2]
+	st 	%o4, [%o0+P_ARG3]
+	add	%o0, P_ARG0, %o1
+	CALL_BIF($2)
+	nop
+	TEST_GOT_MBUF
+
+	/* Restore registers. Check for exception. */
+	TEST_GOT_EXN(4)
+	RESTORE_CONTEXT_BIF
+	NBIF_RET(4)
+	HANDLE_GOT_MBUF(4)
+	.size	$1, .-$1
+	.type	$1, #function
+#endif')
+
 define(standard_bif_interface_0,
 `
 #ifndef HAVE_$1
diff --git a/erts/emulator/hipe/hipe_sparc_glue.S b/erts/emulator/hipe/hipe_sparc_glue.S
index 6c8c841194..094a87fd58 100644
--- a/erts/emulator/hipe/hipe_sparc_glue.S
+++ b/erts/emulator/hipe/hipe_sparc_glue.S
@@ -316,6 +316,7 @@ nbif_fail:
 	.global	nbif_1_gc_after_bif
 	.global	nbif_2_gc_after_bif
 	.global	nbif_3_gc_after_bif
+	.global nbif_4_gc_after_bif
 nbif_0_gc_after_bif:
 	ba	.gc_after_bif
 	mov	0, %o1		/* delay slot */
@@ -326,7 +327,10 @@ nbif_2_gc_after_bif:
 	ba	.gc_after_bif
 	mov	2, %o1		/* delay slot */
 nbif_3_gc_after_bif:
-	mov	3, %o1
+	ba	.gc_after_bif
+	mov	3, %o1		/* delay slot */
+nbif_4_gc_after_bif:
+	mov	4, %o1
 	/*FALLTHROUGH*/
 .gc_after_bif:
 	st	%o1, [P+P_NARITY]
@@ -364,7 +368,11 @@ nbif_2_simple_exception:
 	mov	2, %o1		/* delay slot */
 	.global	nbif_3_simple_exception
 nbif_3_simple_exception:
-	mov	3, %o1
+	ba	.nbif_simple_exception
+	mov	3, %o1		/* delay slot */
+	.global	nbif_4_simple_exception
+nbif_4_simple_exception:
+	mov	4, %o1
 	/*FALLTHROUGH*/
 .nbif_simple_exception:
 	ld	[P+P_FREASON], %o0
diff --git a/erts/emulator/hipe/hipe_x86_asm.m4 b/erts/emulator/hipe/hipe_x86_asm.m4
index 39c5cb1044..436feca506 100644
--- a/erts/emulator/hipe/hipe_x86_asm.m4
+++ b/erts/emulator/hipe/hipe_x86_asm.m4
@@ -212,6 +212,7 @@ define(NBIF_COPY_NSP,`ifelse(eval($1 > NR_ARG_REGS),0,,`movl	%esp, TEMP_NSP')')d
 `/* #define NBIF_COPY_NSP_1	'NBIF_COPY_NSP(1)` */'
 `/* #define NBIF_COPY_NSP_2	'NBIF_COPY_NSP(2)` */'
 `/* #define NBIF_COPY_NSP_3	'NBIF_COPY_NSP(3)` */'
+`/* #define NBIF_COPY_NSP_4	'NBIF_COPY_NSP(4)` */'
 `/* #define NBIF_COPY_NSP_5	'NBIF_COPY_NSP(5)` */'
 
 dnl
@@ -235,6 +236,10 @@ define(NBIF_ARG_OPND,`ifelse(eval($2 >= NR_ARG_REGS),0,`ARG'$2,BASE_OFFSET(eval(
 `/* #define NBIF_ARG_OPND_3_0	'NBIF_ARG_OPND(3,0)` */'
 `/* #define NBIF_ARG_OPND_3_1	'NBIF_ARG_OPND(3,1)` */'
 `/* #define NBIF_ARG_OPND_3_2	'NBIF_ARG_OPND(3,2)` */'
+`/* #define NBIF_ARG_OPND_4_0	'NBIF_ARG_OPND(4,0)` */'
+`/* #define NBIF_ARG_OPND_4_1	'NBIF_ARG_OPND(4,1)` */'
+`/* #define NBIF_ARG_OPND_4_2	'NBIF_ARG_OPND(4,2)` */'
+`/* #define NBIF_ARG_OPND_4_3	'NBIF_ARG_OPND(4,3)` */'
 `/* #define NBIF_ARG_OPND_5_0	'NBIF_ARG_OPND(5,0)` */'
 `/* #define NBIF_ARG_OPND_5_1	'NBIF_ARG_OPND(5,1)` */'
 `/* #define NBIF_ARG_OPND_5_2	'NBIF_ARG_OPND(5,2)` */'
@@ -274,6 +279,7 @@ define(NBIF_RET,`NBIF_RET_N(eval(RET_POP($1)))')dnl
 `/* #define NBIF_RET_1	'NBIF_RET(1)` */'
 `/* #define NBIF_RET_2	'NBIF_RET(2)` */'
 `/* #define NBIF_RET_3	'NBIF_RET(3)` */'
+`/* #define NBIF_RET_4	'NBIF_RET(4)` */'
 `/* #define NBIF_RET_5	'NBIF_RET(5)` */'
 
 dnl
diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4
index a0f16efa33..b0064ee628 100644
--- a/erts/emulator/hipe/hipe_x86_bifs.m4
+++ b/erts/emulator/hipe/hipe_x86_bifs.m4
@@ -48,6 +48,7 @@ define(HANDLE_GOT_MBUF,`
  * standard_bif_interface_1(nbif_name, cbif_name)
  * standard_bif_interface_2(nbif_name, cbif_name)
  * standard_bif_interface_3(nbif_name, cbif_name)
+ * standard_bif_interface_4(nbif_name, cbif_name)
  * standard_bif_interface_0(nbif_name, cbif_name)
  *
  * Generate native interface for a BIF with 0-3 parameters and
@@ -158,6 +159,43 @@ ASYM($1):
 	TYPE_FUNCTION(ASYM($1))
 #endif')
 
+define(standard_bif_interface_4,
+`
+#ifndef HAVE_$1
+#`define' HAVE_$1
+	TEXT
+	.align	4
+	GLOBAL(ASYM($1))
+ASYM($1):
+	/* copy native stack pointer */
+	NBIF_COPY_NSP(4)
+
+	/* switch to C stack */
+	SWITCH_ERLANG_TO_C
+
+	/* make the call on the C stack */
+	NBIF_ARG_REG(0,P)
+	NBIF_ARG(2,4,0)
+	NBIF_ARG(3,4,1)
+	NBIF_ARG(4,4,2)
+	NBIF_ARG(5,4,3)
+	lea 8(%esp), %eax
+	NBIF_ARG_REG(1,%eax)	/* BIF__ARGS */
+	CALL_BIF($2)
+	TEST_GOT_MBUF
+
+	/* switch to native stack */
+	SWITCH_C_TO_ERLANG
+
+	/* throw exception if failure, otherwise return */
+	TEST_GOT_EXN
+	jz	nbif_4_simple_exception
+	NBIF_RET(4)
+	HANDLE_GOT_MBUF(4)
+	SET_SIZE(ASYM($1))
+	TYPE_FUNCTION(ASYM($1))
+#endif')
+
 define(standard_bif_interface_0,
 `
 #ifndef HAVE_$1
diff --git a/erts/emulator/hipe/hipe_x86_glue.S b/erts/emulator/hipe/hipe_x86_glue.S
index 9d38eaaafd..f124e36a26 100644
--- a/erts/emulator/hipe/hipe_x86_glue.S
+++ b/erts/emulator/hipe/hipe_x86_glue.S
@@ -299,6 +299,7 @@ ASYM(nbif_fail):
 	GLOBAL(nbif_1_gc_after_bif)
 	GLOBAL(nbif_2_gc_after_bif)
 	GLOBAL(nbif_3_gc_after_bif)
+	GLOBAL(nbif_4_gc_after_bif)
 	.align	4
 nbif_0_gc_after_bif:
 	xorl	%edx, %edx
@@ -314,6 +315,10 @@ nbif_2_gc_after_bif:
 	.align	4
 nbif_3_gc_after_bif:
 	movl	$3, %edx
+	jmp	.gc_after_bif
+	.align	4
+nbif_4_gc_after_bif:
+	movl	$4, %edx
 	/*FALLTHROUGH*/
 	.align	4
 .gc_after_bif:
@@ -337,6 +342,7 @@ nbif_3_gc_after_bif:
 	GLOBAL(nbif_1_simple_exception)
 	GLOBAL(nbif_2_simple_exception)
 	GLOBAL(nbif_3_simple_exception)
+	GLOBAL(nbif_4_simple_exception)
 	.align	4
 nbif_0_simple_exception:
 	xorl	%eax, %eax
@@ -352,6 +358,10 @@ nbif_2_simple_exception:
 	.align	4
 nbif_3_simple_exception:
 	movl	$3, %eax
+	jmp	.nbif_simple_exception
+	.align	4
+nbif_4_simple_exception:
+	movl	$4, %eax
 	/*FALLTHROUGH*/
 	.align	4
 .nbif_simple_exception: